DataImpl.java
/*
* Copyright (c) 2007-2026 MetaSolutions AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.entrystore.impl;
import com.google.common.util.concurrent.Striped;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.Strings;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.entrystore.Data;
import org.entrystore.Entry;
import org.entrystore.PrincipalManager.AccessProperty;
import org.entrystore.QuotaException;
import org.entrystore.repository.RepositoryEvent;
import org.entrystore.repository.RepositoryEventObject;
import org.entrystore.repository.config.Settings;
import org.entrystore.repository.util.FileOperations;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.channels.FileChannel;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.locks.Lock;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.commons.codec.digest.MessageDigestAlgorithms.SHA_256;
import static org.eclipse.rdf4j.model.util.Values.iri;
/**
* Data class to handle binary local resources.
*
* @author Eric Johansson
* @author Hannes Ebner
*/
public class DataImpl extends ResourceImpl implements Data {
private static final Logger log = LoggerFactory.getLogger(DataImpl.class);
public static final String SHA_256_POSTFIX = ".sha256";
public static final String STAGING_POSTFIX = ".part";
// Only the names stagingPath generates, so that an entry whose id happens to look similar keeps its file.
private static final Pattern STAGING_FILE_NAME = Pattern.compile(
"\\.\\p{XDigit}{8}-\\p{XDigit}{4}-\\p{XDigit}{4}-\\p{XDigit}{4}-\\p{XDigit}{12}" + Pattern.quote(STAGING_POSTFIX));
/** Moves files within the data folder; replaced only by tests to simulate an interruption. */
@FunctionalInterface
interface FileMover {
void move(Path source, Path target) throws IOException;
}
static volatile FileMover fileMover = DataImpl::moveReplacing;
/** Serializes the commit step of {@link #setData} and {@link #deleteFile} per entry URI. */
private static final Striped<Lock> WRITE_LOCKS = Striped.lock(64);
private File file = null;
public DataImpl(Entry entry) {
super((EntryImpl) entry, iri(entry.getResourceURI().toString()));
}
private File getFile() throws IOException {
if (file == null) {
String dataDirStr = entry.getRepositoryManager().getConfiguration().getString(Settings.DATA_FOLDER);
if (dataDirStr != null) {
File dataDir = dataFolder(dataDirStr);
if (!dataDir.exists()) {
if (!dataDir.mkdirs()) {
log.error("Unable to create data folder");
}
}
File contextDir = new File(dataDir, entry.getContext().getEntry().getId());
if (!contextDir.exists()) {
contextDir.mkdir();
}
file = new File(contextDir, entry.getId());
}
if (file == null) {
throw new IOException("Unable to get local file of resource");
}
}
return file;
}
public InputStream getData() {
this.entry.getRepositoryManager().getPrincipalManager().checkAuthenticatedUserAuthorized(entry, AccessProperty.ReadResource);
try {
if (getFile() != null) {
return Files.newInputStream(getFile().toPath());
}
} catch (IOException e) {
log.error(e.getMessage());
}
return null;
}
/**
* Streams the data to a staging file next to the data file, then, holding a per-entry lock, charges the quota
* and moves the data and its digest over the previous ones. A failed or rejected write leaves the previous
* data and digest intact, and concurrent writes to one entry cannot mix one's data with another's digest.
*/
public void setData(InputStream is) throws QuotaException, IOException {
this.entry.getRepositoryManager().getPrincipalManager().checkAuthenticatedUserAuthorized(entry, AccessProperty.WriteResource);
MessageDigest sha;
try {
sha = MessageDigest.getInstance(SHA_256);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
// Checked before getFile(), which would create the context's folder again after its removal deleted it.
if (entry.getContext().isDeleted()) {
throw new EntryRemovedException(entry.getEntryURI());
}
Path dataPath = getFile().toPath();
Path dataPart = stagingPath(dataPath);
Path digestPart = stagingPath(dataPath);
try {
long bytes;
try (OutputStream os = Files.newOutputStream(dataPart, StandardOpenOption.CREATE_NEW)) {
bytes = FileOperations.copyFile(new DigestInputStream(is, sha), os);
}
forceToDisk(dataPart);
Files.writeString(digestPart, Hex.encodeHexString(sha.digest()), UTF_8, StandardOpenOption.CREATE_NEW);
commit(dataPart, dataPath, digestPart, bytes);
} finally {
deleteLogged(dataPart);
deleteLogged(digestPart);
}
entry.getRepositoryManager().fireRepositoryEvent(new RepositoryEventObject(entry, RepositoryEvent.ResourceUpdated));
}
/**
* Moves the staged data and digest into place, holding the per-entry lock that {@link #deleteFile} takes too,
* unless the entry has been removed meanwhile, possibly recreated under the same id. The previous digest is
* removed before the data is moved, so that whatever step fails or a crash interrupts, the digest present
* either matches the data or is missing.
*/
private void commit(Path dataPart, Path dataPath, Path digestPart, long bytes) throws QuotaException, IOException {
Path digestPath = Path.of(dataPath.toFile().getCanonicalPath() + SHA_256_POSTFIX);
boolean hasQuotas = entry.getRepositoryManager().hasQuotas();
// Charged and rolled back outside the lock: entry removal holds the repository while it takes the lock,
// and the fill level is updated under the repository.
if (hasQuotas) {
entry.getContext().increaseQuotaFillLevel(bytes);
}
boolean dataMoved = false;
Lock lock = WRITE_LOCKS.get(entry.getEntryURI());
lock.lock();
try {
if (!isCurrentEntry()) {
throw new EntryRemovedException(entry.getEntryURI());
}
Files.deleteIfExists(digestPath);
fileMover.move(dataPart, dataPath);
dataMoved = true;
try {
fileMover.move(digestPart, digestPath);
} catch (IOException e) {
log.error("Failed to store the digest of entry {}, leaving it without one", entry.getEntryURI(), e);
}
} finally {
lock.unlock();
if (hasQuotas && !dataMoved) {
// Reads the fill level as the increase does, which is wrong once cached (ENTRYSTORE-1238).
entry.getContext().decreaseQuotaFillLevel(bytes);
}
}
}
/**
* Whether this entry has neither been removed nor replaced by one created under the same id. Entry objects
* may be reloaded, so the current one is recognised by its creation date rather than by identity.
*/
private boolean isCurrentEntry() {
if (entry.isDeleted()) {
return false;
}
Entry current = entry.getContext().getByEntryURI(entry.getEntryURI());
return current != null && !current.isDeleted()
&& Objects.equals(current.getCreationDate(), entry.getCreationDate());
}
/** Thrown by {@link #setData} when the entry was removed while its data was being received. */
public static final class EntryRemovedException extends IOException {
public EntryRemovedException(URI entryURI) {
super("Entry " + entryURI + " was removed while its data was being stored");
}
}
/** A staging name without the entry id, so that it stays short and is recognised by {@link #isStagingFile}. */
private static Path stagingPath(Path dataPath) {
return dataPath.resolveSibling("." + UUID.randomUUID() + STAGING_POSTFIX);
}
/**
* Tells whether a file in the data folder is a staging file of a write in progress or of one interrupted by a
* crash, which exports, backups and imports skip.
*/
public static boolean isStagingFile(String fileName) {
return STAGING_FILE_NAME.matcher(fileName).matches();
}
/** The data folder of a {@link Settings#DATA_FOLDER} value, which may carry a "file:" prefix. */
static File dataFolder(String configured) {
return new File(Strings.CS.removeStart(Strings.CS.removeStart(configured, "file://"), "file:"));
}
/**
* Deletes the staging files that writes interrupted by a crash left in the context folders of the data
* folder, given as its {@link Settings#DATA_FOLDER} value. Only safe while no write is in progress, i.e. at
* startup.
*/
public static void deleteStaleStagingFiles(String configuredDataFolder) {
Path dataFolder = dataFolder(configuredDataFolder).toPath();
if (!Files.isDirectory(dataFolder)) {
return;
}
try (Stream<Path> contextFolders = Files.list(dataFolder)) {
for (Path contextFolder : contextFolders.filter(Files::isDirectory).toList()) {
try (Stream<Path> files = Files.list(contextFolder)) {
files.filter(f -> isStagingFile(f.getFileName().toString())).forEach(f -> {
log.info("Deleting stale staging file {}", f);
deleteLogged(f);
});
}
}
} catch (IOException | UncheckedIOException e) {
log.warn("Could not delete stale staging files in {}", dataFolder, e);
}
}
/** Deletes without throwing, so that a failed cleanup cannot mask the exception being handled. */
private static void deleteLogged(Path path) {
try {
Files.deleteIfExists(path);
} catch (IOException e) {
log.warn("Could not delete {}", path, e);
}
}
private static void forceToDisk(Path path) throws IOException {
try (FileChannel channel = FileChannel.open(path, StandardOpenOption.WRITE)) {
channel.force(true);
}
}
private static void moveReplacing(Path source, Path target) throws IOException {
try {
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (AtomicMoveNotSupportedException e) {
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING);
}
}
/** Replaces the data with a copy of {@code file}, as {@link #setData} does with a stream. */
public void useData(File file) throws QuotaException, IOException {
if (file == null) {
throw new IllegalArgumentException("File must not be null");
}
try (InputStream is = Files.newInputStream(file.toPath())) {
setData(is);
}
}
@Override
public void remove(RepositoryConnection rc) throws Exception {
super.remove(rc);
deleteFile();
}
public boolean delete() {
this.entry.getRepositoryManager().getPrincipalManager().checkAuthenticatedUserAuthorized(entry, AccessProperty.WriteResource);
return deleteFile();
}
/**
* Tells whether the resource has a file on disk, without any access control check. Fails open: an
* undeterminable file state reports true, so callers attempt a deletion rather than leaving a possibly
* present file behind.
*/
protected boolean hasFile() {
try {
File f = getFile();
return f != null && f.exists();
} catch (IOException e) {
log.error("Could not determine whether entry {} has a data file, assuming it has one", entry.getEntryURI(), e);
return true;
}
}
/**
* Deletes the resource's file from disk WITHOUT any access control check (cf. ENTRYSTORE-1006). Callers
* must have verified the caller's rights themselves; user-facing deletion goes through {@link #delete()}.
*/
protected boolean deleteFile() {
boolean existed = false;
boolean success = false;
long size = 0;
// Serialized with the commit of setData; the quota is updated after the lock is released, see commit.
Lock lock = WRITE_LOCKS.get(entry.getEntryURI());
lock.lock();
try {
File f = getFile();
File digestFile = getDigestFile();
if (f != null && f.exists()) {
existed = true;
size = f.length();
success = f.delete();
if (digestFile != null && digestFile.exists()) {
digestFile.delete();
}
}
} catch (IOException ioe) {
log.error("Failed to delete data file of entry {}", entry.getEntryURI(), ioe);
} finally {
lock.unlock();
}
if (success && entry.getRepositoryManager().hasQuotas()) {
entry.getContext().decreaseQuotaFillLevel(size);
}
if (existed) {
entry.getRepositoryManager().fireRepositoryEvent(new RepositoryEventObject(entry, RepositoryEvent.ResourceDeleted));
}
return success;
}
/**
* Truncates the resource's file in place WITHOUT any access control check, as a fallback when
* {@link #deleteFile()} fails (truncation works even while another process holds the file open), so
* stale content cannot be served if the file path is later reused by another entry.
*/
protected void truncateFile() {
try {
File f = getFile();
if (f != null && f.exists()) {
try (OutputStream os = Files.newOutputStream(f.toPath(), StandardOpenOption.TRUNCATE_EXISTING)) {
// opening with TRUNCATE_EXISTING empties the file
}
}
} catch (IOException ioe) {
log.error("Failed to truncate data file of entry {}", entry.getEntryURI(), ioe);
}
}
public File getDataFile() {
this.entry.getRepositoryManager().getPrincipalManager().checkAuthenticatedUserAuthorized(entry, AccessProperty.ReadResource);
try {
File f = getFile();
if (f != null && f.exists()) {
return f;
}
} catch (IOException ioe) {
log.error(ioe.getMessage());
}
return null;
}
private File getDigestFile() {
File dataFile;
try {
dataFile = getFile();
} catch (IOException e) {
log.error("Could not resolve data file of entry {}", entry.getEntryURI(), e);
return null;
}
if (dataFile == null || !dataFile.exists()) {
return null;
}
String digestFileName;
try {
digestFileName = dataFile.getCanonicalPath() + SHA_256_POSTFIX;
} catch (IOException e) {
log.error("Could not get canonical path of: {}", dataFile.getAbsolutePath(), e);
return null;
}
return new File(digestFileName);
}
/**
* Returns the stored SHA-256 hex digest of the data, or null when there is no data or no readable digest.
*
* @throws org.entrystore.AuthorizationException if the current user may not read the resource, whether or not
* it has data, so serializing the resource reveals nothing to them
*/
public String readDigest() {
entry.getRepositoryManager().getPrincipalManager()
.checkAuthenticatedUserAuthorized(entry, AccessProperty.ReadResource);
File digestFile = getDigestFile();
if (digestFile == null) {
return null;
}
try {
return FileUtils.readFileToString(digestFile, UTF_8);
} catch (IOException e) {
return null;
}
}
}