EntryPurge.java
/*
* Copyright (c) 2007-2026 MetaSolutions AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.entrystore.impl;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Resource;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.Value;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.repository.sparql.SPARQLRepository;
import org.entrystore.Context;
import org.entrystore.Entry;
import org.entrystore.EntryType;
import org.entrystore.Group;
import org.entrystore.User;
import org.entrystore.repository.CorruptEntryException;
import org.entrystore.repository.RepositoryEvent;
import org.entrystore.repository.RepositoryEventObject;
import org.entrystore.repository.util.URISplit;
import org.entrystore.repository.util.URIType;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import static org.eclipse.rdf4j.model.util.Values.iri;
/**
* Removes entries of one context by exact quad, so that the transaction holding the repository monitor finds what it
* removes through a triple index rather than by scanning the store. {@link #plan} reads what to remove without the
* monitor. In the caller's transaction, under the monitor, {@link #refresh} reads again the entries changed since and
* {@link #apply} removes. After the commit the caller runs {@link #publish} still holding the monitor and
* {@link #afterPublish} without it; on a failure, {@link #rollBack}.
*
* <p>For each entry it removes what {@code ContextImpl.remove} does: the entry's entry, metadata, cached external
* metadata, relations and resource graphs, the relation copies its entry and metadata graphs put in other entries'
* relations graphs, the membership copies a removed list put in its remaining members' relations graphs, and its
* index triples. It records a tombstone when deleted entries are tracked, prunes the entry from the lists of the
* context that remain, stamps the context modified, and deletes the entry's data file once committed. Unlike
* {@code ContextImpl.remove}, which prunes those lists in transactions of their own before its commit, it prunes them
* in the caller's transaction and fires every event after the commit; and it keeps a relation copy that a remaining
* graph still states.
*
* <p>An entry changed after the plan is found by its modified date, which writes to its metadata and resource graphs
* and most writes to its entry graph stamp, and by its list memberships, which are read through an index. Its ACL
* statements, whose change stamps nothing, are removed by a pattern bound to their subjects. Known limit: a relation
* copy that an entry not removed writes into a removed entry's relations graph between the plan and the apply is left
* behind.
*
* <p>Refuses a context that keeps entry names and an entry whose resource is a context, user or group, whose removal
* does more than this.
*/
@Slf4j
final class EntryPurge {
private final ContextImpl context;
private final RepositoryManagerImpl repositoryManager;
private final Repository repository;
private final ExactRemoval exactRemoval;
private final boolean blankNodesByPattern;
/** The relations graphs the planned entries have by convention, whose copies go with them. */
private final Set<IRI> plannedRelations = new HashSet<>();
private final Map<IRI, Planned> planned = new LinkedHashMap<>();
/** The remaining lists of the context that lose members. */
private final Map<EntryImpl, Set<URI>> prunes = new LinkedHashMap<>();
/** The remaining lists loaded so far, by entry URI; null for one that is no local list of the context. */
private final Map<IRI, EntryImpl> listEntries = new HashMap<>();
/** {@code (list hasListMember member)} in the relations graph of each remaining member of a list it leaves. */
private final Set<Statement> memberCopies = new LinkedHashSet<>();
private final Set<IRI> unlinkedMembers = new LinkedHashSet<>();
private final List<EntryImpl> prunedLists = new ArrayList<>();
private EntryPurge(ContextImpl context) {
if (context instanceof EntryNamesContext) {
throw new IllegalArgumentException("Context " + context.id + " keeps entry names, which a purge does not "
+ "remove");
}
this.context = context;
this.repositoryManager = context.entry.repositoryManager;
this.repository = repositoryManager.getRepository();
this.exactRemoval = new ExactRemoval(repositoryManager);
this.blankNodesByPattern = repository instanceof SPARQLRepository;
}
/**
* Reads, without the monitor, what removing the entries takes.
*
* @param entries entry URIs of the context; one of another context is skipped
* @throws IllegalArgumentException if the context keeps entry names
* @throws IllegalStateException if an entry's resource is a context, user or group
*/
static EntryPurge plan(ContextImpl context, Collection<IRI> entries) {
EntryPurge purge = new EntryPurge(context);
try (RepositoryConnection reader = purge.repository.getConnection()) {
purge.read(reader, entries, false);
purge.link(reader);
}
return purge;
}
/** How many statements the planned entries' own graphs hold. */
long statementCount() {
return planned.values().stream().mapToLong(Planned::statementCount).sum();
}
/** The remaining lists {@link #apply} removed members from. */
List<EntryImpl> prunedLists() {
return List.copyOf(prunedLists);
}
/** The members of the removed lists that are not removed themselves, as last read. */
Set<IRI> unlinkedMembers() {
return Set.copyOf(unlinkedMembers);
}
/** The resources of the lists holding a planned entry, as last read; none for an entry not planned. */
Set<IRI> holdingLists(IRI entry) {
Planned p = planned.get(entry);
return p != null ? Set.copyOf(p.lists()) : Set.of();
}
/**
* Reads again the entries changed since they were read: those whose modified date or list memberships differ.
* Both are looked up through an index; the graphs of a changed entry are read again, in one store pass when many
* changed. Within the caller's transaction under the monitor before {@link #apply}, and may be called before as
* well, so that only what changes after is read under the monitor.
*
* @return how many entries were read again
*/
int refresh(RepositoryConnection rc) {
List<IRI> stale = planned.values().stream().filter(p -> isStale(rc, p)).map(Planned::entryURI).toList();
if (!stale.isEmpty()) {
read(rc, stale, true);
link(rc);
}
return stale.size();
}
/**
* Removes, within the caller's transaction under the monitor, what the plan read: each statement by exact quad,
* the index triples by a pattern the posc index answers, the ACL statements by a pattern bound to their subject,
* and the members of the remaining lists by {@link ListImpl#pruneExactly}. Stamps the context modified.
*/
void apply(RepositoryConnection rc) {
List<Statement> copies = new ArrayList<>();
Set<IRI> removedGraphs = new HashSet<>();
for (Planned p : planned.values()) {
for (List<Statement> statements : p.graphs().values()) {
ExactRemoval.removeOwn(rc, statements, blankNodesByPattern, "context " + context.id);
}
removedGraphs.addAll(p.graphs().keySet());
copies.addAll(p.copies());
}
exactRemoval.removeCopies(rc, copies, removedGraphs::contains);
rc.remove(memberCopies);
for (Planned p : planned.values()) {
rc.remove((Resource) null, RepositoryProperties.resHasEntry, p.entryURI(), context.resourceURI);
rc.remove((Resource) null, RepositoryProperties.mdHasEntry, p.entryURI(), context.resourceURI);
for (IRI subject : p.aclSubjects()) {
rc.remove(subject, RepositoryProperties.Read, null, p.entryURI());
rc.remove(subject, RepositoryProperties.Write, null, p.entryURI());
}
if (p.exists()) {
context.addTombstone(p.entryURI(), rc);
}
}
for (Map.Entry<EntryImpl, Set<URI>> prune : prunes.entrySet()) {
if (((ListImpl) prune.getKey().getResource()).pruneExactly(rc, prune.getValue())) {
prunedLists.add(prune.getKey());
}
}
((EntryImpl) context.getEntry()).updateModifiedDateSynchronized(rc, rc.getValueFactory());
}
/**
* After the commit, still under the monitor: updates the context's indexes, flags the removed entries deleted, also
* the instance the soft cache holds, evicts it, and drops the members the pruned lists hold in memory. Cannot fail
* on an IRI {@code java.net.URI} rejects.
*
* @param dropIndexes whether to drop the context's indexes, so that the next read builds them from the store, as a
* caller that changed the index graph further needs; otherwise the removed entries are taken out
* of them, unless one could not be loaded or has such an IRI, which drops them too
*/
void publish(boolean dropIndexes) {
if (dropIndexes || !unindexRemoved()) {
context.unpublishIndexes();
}
SoftCache softCache = context.getSoftCache();
for (Planned p : planned.values()) {
Entry cached = softCache.getByEntryURI(URI.create(p.entryURI().stringValue()));
if (cached instanceof EntryImpl cachedEntry) {
cachedEntry.markDeleted();
softCache.remove(cachedEntry);
}
if (p.entry() != null) {
p.entry().markDeleted();
}
}
for (EntryImpl list : prunedLists) {
if (list.resource instanceof ListImpl listResource) {
listResource.invalidateChildren();
}
}
}
/**
* Takes the removed entries out of the context's in-memory indexes, with the ops
* {@code ContextImpl.removeFromIndex} collects for one; {@link #apply} has removed their index triples.
*
* @return false, having changed nothing, if the indexes are not held in memory, where each op would only be
* buffered for a load, or if an entry could not be loaded or has an IRI {@code java.net.URI} rejects
*/
private boolean unindexRemoved() {
if (!context.indexesPublished()) {
return false;
}
List<ContextImpl.IndexOp> ops = new ArrayList<>();
try {
for (Planned p : planned.values()) {
EntryImpl entry = p.entry();
if (entry == null) {
return false;
}
if (entry.getSesameExternalMetadataURI() != null) {
ops.add(new ContextImpl.IndexOp(ContextImpl.IndexKind.EXTERNAL_METADATA, false,
entry.getExternalMetadataURI(), entry.getEntryURI()));
}
if (entry.getSesameResourceURI() != null) {
ops.add(new ContextImpl.IndexOp(ContextImpl.IndexKind.RESOURCE, false, entry.getResourceURI(),
entry.getEntryURI()));
}
}
} catch (IllegalArgumentException e) {
return false;
}
context.applyIndexOps(ops);
return true;
}
/**
* After {@link #publish}, without the monitor: fires the events and deletes the data files of the removed
* entries. Listener failures are isolated by the dispatch and a failed file deletion is logged, since the removal
* stands.
*/
void afterPublish() {
for (Planned p : planned.values()) {
if (p.entry() != null) {
fire(p.entry(), RepositoryEvent.EntryDeleted);
if (p.entry().getResource() instanceof ListImpl) {
fire(p.entry(), RepositoryEvent.ResourceDeleted);
}
}
}
for (IRI member : unlinkedMembers) {
try {
Entry entry = context.getByEntryURI(URI.create(member.stringValue()));
if (entry != null) {
fire(entry, RepositoryEvent.EntryUpdated);
}
} catch (RuntimeException e) {
log.error("Could not report the update of entry {}, which left a removed list", member, e);
}
}
for (EntryImpl list : prunedLists) {
fire(list, RepositoryEvent.ResourceUpdated);
}
for (Planned p : planned.values()) {
if (p.entry() != null && p.entry().getResource() instanceof DataImpl data) {
deleteFile(data);
}
}
}
/**
* Rolls the caller's transaction back and reloads from the store the context entry and the remaining lists,
* which {@link #apply} stamped modified in memory. Nothing else in memory changes before the commit.
*/
void rollBack(RepositoryConnection rc) {
try {
rc.rollback();
} catch (RuntimeException e) {
log.error("Failed to roll back the removal of entries of context {}", context.id, e);
}
List<EntryImpl> stamped = new ArrayList<>(prunes.keySet());
stamped.add((EntryImpl) context.getEntry());
for (EntryImpl entry : stamped) {
try {
entry.refreshFromRepository(rc);
} catch (RuntimeException e) {
log.error("Failed to refresh entry {} after a rollback, its modification date may be stale",
entry.getEntryURI(), e);
}
}
}
/**
* @param rereading whether the entries are read again, after a relation copy may have been written into the
* relations graph of another planned entry, which the first read of that graph did not see
*/
private void read(RepositoryConnection reader, Collection<IRI> entryURIs, boolean rereading) {
Map<IRI, Conventional> conventional = new LinkedHashMap<>();
for (IRI entryURI : entryURIs) {
conventionalGraphs(entryURI).ifPresentOrElse(graphs -> conventional.put(entryURI, graphs),
() -> log.warn("Not removing {}, which is no entry of context {}", entryURI, context.id));
}
conventional.values().forEach(graphs -> plannedRelations.add(graphs.relations()));
Map<IRI, List<Statement>> read = exactRemoval.read(reader,
conventional.values().stream().flatMap(graphs -> graphs.all().stream()).toList());
for (Map.Entry<IRI, Conventional> entry : conventional.entrySet()) {
planned.put(entry.getKey(), toPlanned(reader, entry.getKey(), entry.getValue(), read, rereading));
}
}
/** The graphs of an entry of this context as its URIs are made. */
private Optional<Conventional> conventionalGraphs(IRI entryURI) {
return URISplit.of(URI.create(entryURI.stringValue()), repositoryManager.getRepositoryURL())
.filter(split -> split.getUriType() == URIType.MetaMetadata && context.id.equals(split.getContextId()))
.map(split -> new Conventional(entryURI, iri(split.getMetadataURI().toString()),
iri(split.getRelationURI().toString()), iri(split.getCachedExternalMetadataURI().toString()),
iri(split.getResourceURI().toString())));
}
private Planned toPlanned(RepositoryConnection reader, IRI entryURI, Conventional conventional,
Map<IRI, List<Statement>> read, boolean rereading) {
EntryImpl entry = null;
try {
entry = context.entryFromGraph(entryURI, read.get(entryURI));
} catch (CorruptEntryException e) {
log.warn("Entry {} is corrupt, so its graphs are removed without its data file and events: {}",
entryURI, e.getMessage());
}
if (entry != null) {
refuseUnsupported(entry);
}
Map<IRI, List<Statement>> graphs = new LinkedHashMap<>();
for (IRI graph : entry != null ? ownedGraphs(entry) : conventional.all()) {
List<Statement> statements = read.get(graph);
graphs.put(graph, statements != null ? statements : ExactRemoval.read(reader, graph, Integer.MAX_VALUE));
}
IRI relations = entry != null && entry.relationURI != null ? entry.relationURI : conventional.relations();
Predicate<IRI> removedWithRelations = rereading ? object -> false
: object -> exactRemoval.relationsGraphOf(object).map(plannedRelations::contains).orElse(false);
List<Statement> copies = new ArrayList<>();
for (Map.Entry<IRI, List<Statement>> graph : graphs.entrySet()) {
boolean fromEntryGraph = graph.getKey().equals(entryURI);
if (fromEntryGraph || isMetadataGraph(graph.getKey(), entry, conventional)) {
copies.addAll(exactRemoval.relationCopies(reader, graph.getValue(), fromEntryGraph,
removedWithRelations));
}
}
List<URI> members = List.of();
if (entry != null && entry.getResource() instanceof ListImpl list) {
members = ListImpl.loadChildren(graphs.getOrDefault(list.getSesameURI(), List.of())).stream()
.filter(member -> member != null).toList();
}
return new Planned(entryURI, entry, relations, listsHolding(entryURI, graphs.get(relations)), graphs, copies,
members, aclSubjects(entryURI, entry, conventional));
}
/** The subjects the entry's ACL statements can have, as {@code EntryImpl} chooses them per access property. */
private static Set<IRI> aclSubjects(IRI entryURI, EntryImpl entry, Conventional conventional) {
Set<IRI> subjects = new LinkedHashSet<>(List.of(entryURI, conventional.metadata(), conventional.resource()));
if (entry != null && entry.getSesameLocalMetadataURI() != null) {
subjects.add(entry.getSesameLocalMetadataURI());
}
if (entry != null && entry.getSesameResourceURI() != null) {
subjects.add(entry.getSesameResourceURI());
}
return subjects;
}
/** The graphs an entry owns according to its entry graph, as {@code EntryImpl.remove} clears them. */
private static Set<IRI> ownedGraphs(EntryImpl entry) {
Set<IRI> owned = new LinkedHashSet<>();
owned.add(entry.getSesameEntryURI());
EntryType type = entry.getEntryType();
if ((type == EntryType.Local || type == EntryType.Link || type == EntryType.LinkReference)
&& entry.getSesameLocalMetadataURI() != null) {
owned.add(entry.getSesameLocalMetadataURI());
}
if ((type == EntryType.Reference || type == EntryType.LinkReference)
&& entry.getSesameCachedExternalMetadataURI() != null) {
owned.add(entry.getSesameCachedExternalMetadataURI());
}
if (entry.relationURI != null) {
owned.add(entry.relationURI);
}
if (entry.getResource() instanceof RDFResource resource && resource.isRemovable()) {
owned.add(resource.getSesameURI());
}
return owned;
}
private static boolean isMetadataGraph(IRI graph, EntryImpl entry, Conventional conventional) {
if (entry == null) {
return graph.equals(conventional.metadata()) || graph.equals(conventional.cachedExternalMetadata());
}
return graph.equals(entry.getSesameLocalMetadataURI())
|| graph.equals(entry.getSesameCachedExternalMetadataURI());
}
/** By the resource, not the graph type, which a link or reference may declare without holding such a resource. */
private static void refuseUnsupported(EntryImpl entry) {
Object resource = entry.getResource();
if (resource instanceof Context || resource instanceof User || resource instanceof Group) {
throw new IllegalStateException("Entry " + entry.getEntryURI() + " of graph type " + entry.getGraphType()
+ " cannot be purged");
}
}
/** The resources of the lists that hold the entry, as its relations graph, read with it, records them. */
private static Set<IRI> listsHolding(IRI entryURI, List<Statement> relations) {
Set<IRI> lists = new HashSet<>();
if (relations != null) {
for (Statement statement : relations) {
if (statement.getPredicate().equals(RepositoryProperties.hasListMember)
&& statement.getObject().equals(entryURI) && statement.getSubject() instanceof IRI list) {
lists.add(list);
}
}
}
return lists;
}
/**
* Derives what depends on all planned entries: which remaining lists lose which members, and the membership
* copies of the members a removed list leaves behind. Loads the remaining lists, which costs a scan for one that
* is not cached.
*/
private void link(RepositoryConnection reader) {
prunes.clear();
memberCopies.clear();
unlinkedMembers.clear();
for (Planned p : planned.values()) {
for (IRI list : p.lists()) {
listEntryOf(list).filter(listEntry -> !planned.containsKey(listEntry)).map(this::listEntry)
.ifPresent(listEntry -> prunes.computeIfAbsent(listEntry, k -> new LinkedHashSet<>())
.add(URI.create(p.entryURI().stringValue())));
}
if (p.entry() != null && p.entry().getResource() instanceof ListImpl list) {
for (URI member : p.members()) {
IRI memberIRI = iri(member.toString());
if (!planned.containsKey(memberIRI)) {
unlink(reader, list.getSesameURI(), memberIRI);
}
}
}
}
}
/** Records the membership copy of a member that stays, as {@code EntryImpl.removeReferringList} removes it. */
private void unlink(RepositoryConnection reader, IRI listResource, IRI member) {
unlinkedMembers.add(member);
exactRemoval.relationsGraphOf(member).ifPresent(relations -> {
if (reader.hasStatement(listResource, RepositoryProperties.hasListMember, member, false, relations)) {
memberCopies.add(reader.getValueFactory().createStatement(listResource,
RepositoryProperties.hasListMember, member, relations));
}
});
}
/** The entry URI of a list of this context named by its resource URI. */
private Optional<IRI> listEntryOf(IRI listResource) {
return URISplit.of(URI.create(listResource.stringValue()), repositoryManager.getRepositoryURL())
.filter(split -> context.id.equals(split.getContextId())).flatMap(URISplit::getEntryURI)
.map(entryURI -> iri(entryURI.toString()));
}
/** The remaining list entry, loaded once; null, and logged, if it is no local list of this context. */
private EntryImpl listEntry(IRI listEntryURI) {
if (listEntries.containsKey(listEntryURI)) {
return listEntries.get(listEntryURI);
}
Entry entry = context.getByEntryURI(URI.create(listEntryURI.stringValue()));
EntryImpl listEntry = entry instanceof EntryImpl impl && impl.getResource() instanceof ListImpl ? impl : null;
if (listEntry == null) {
log.warn("List {} holding a removed entry could not be resolved to a list, a dangling member reference "
+ "may remain", listEntryURI);
}
listEntries.put(listEntryURI, listEntry);
return listEntry;
}
private boolean isStale(RepositoryConnection rc, Planned p) {
Value modified = p.modified();
boolean modifiedKept = modified != null
? rc.hasStatement(p.entryURI(), RepositoryProperties.Modified, modified, false, p.entryURI())
: !rc.hasStatement(p.entryURI(), RepositoryProperties.Modified, null, false, p.entryURI());
return !modifiedKept || !ExactRemoval.listsHolding(rc, p.entryURI(), p.relations()).equals(p.lists());
}
private void fire(Entry entry, RepositoryEvent event) {
repositoryManager.fireRepositoryEvent(new RepositoryEventObject(entry, event));
}
private static void deleteFile(DataImpl data) {
try {
if (data.hasFile() && !data.deleteFile()) {
log.warn("Failed to delete the data file of removed entry {}, truncating it instead",
data.getEntry().getEntryURI());
data.truncateFile();
}
} catch (RuntimeException e) {
log.error("Failed to delete the data file of removed entry {}", data.getEntry().getEntryURI(), e);
}
}
/** The graphs of an entry as its URIs are made: entry, metadata, relations, cached external metadata, resource. */
private record Conventional(IRI entry, IRI metadata, IRI relations, IRI cachedExternalMetadata, IRI resource) {
List<IRI> all() {
return List.of(entry, metadata, relations, cachedExternalMetadata, resource);
}
}
/**
* What removing one entry takes, as read.
*
* @param entry the entry, or null if its entry graph is empty or corrupt
* @param relations its relations graph
* @param lists the resources of the lists holding it, as its relations graph records them
* @param graphs the statements of each graph it owns, its entry graph among them
* @param copies the relation copies its entry and metadata graphs put in other entries' relations graphs
* @param members its members, if it is a list
* @param aclSubjects the subjects its ACL statements can have
*/
private record Planned(IRI entryURI, EntryImpl entry, IRI relations, Set<IRI> lists,
Map<IRI, List<Statement>> graphs, List<Statement> copies, List<URI> members, Set<IRI> aclSubjects) {
boolean exists() {
return !entryGraph().isEmpty();
}
/** Its modified date as read, or null if it had none. */
Value modified() {
return entryGraph().stream()
.filter(s -> s.getSubject().equals(entryURI) && s.getPredicate().equals(RepositoryProperties.Modified))
.map(Statement::getObject).findFirst().orElse(null);
}
long statementCount() {
return graphs.values().stream().mapToLong(List::size).sum();
}
private List<Statement> entryGraph() {
return graphs.getOrDefault(entryURI, List.of());
}
}
}