RepositoryManagerImpl.java
/*
* Copyright (c) 2007-2026 MetaSolutions AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.entrystore.impl;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.solr.client.api.util.SolrVersion;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.jetty.HttpJettySolrClient;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.repository.RepositoryException;
import org.eclipse.rdf4j.repository.http.HTTPRepository;
import org.eclipse.rdf4j.repository.sail.SailRepository;
import org.eclipse.rdf4j.repository.sparql.SPARQLRepository;
import org.eclipse.rdf4j.rio.RDFFormat;
import org.eclipse.rdf4j.rio.RDFWriterFactory;
import org.eclipse.rdf4j.rio.binary.BinaryRDFWriterFactory;
import org.eclipse.rdf4j.rio.nquads.NQuadsWriterFactory;
import org.eclipse.rdf4j.rio.trig.TriGWriterFactory;
import org.eclipse.rdf4j.rio.trix.TriXWriterFactory;
import org.eclipse.rdf4j.sail.lmdb.LmdbStore;
import org.eclipse.rdf4j.sail.lmdb.config.LmdbStoreConfig;
import org.eclipse.rdf4j.sail.memory.MemoryStore;
import org.eclipse.rdf4j.sail.nativerdf.NativeStore;
import org.eclipse.rdf4j.sail.nativerdf.wal.ValueStoreWalConfig;
import org.entrystore.ContextManager;
import org.entrystore.Entry;
import org.entrystore.GraphType;
import org.entrystore.PrincipalManager;
import org.entrystore.Quota;
import org.entrystore.SearchIndex;
import org.entrystore.config.Config;
import org.entrystore.repository.RepositoryEvent;
import org.entrystore.repository.RepositoryEventObject;
import org.entrystore.repository.RepositoryListener;
import org.entrystore.repository.RepositoryManager;
import org.entrystore.repository.config.ConfigurationManager;
import org.entrystore.repository.config.Settings;
import org.entrystore.repository.util.DataCorrection;
import org.entrystore.repository.util.NS;
import org.entrystore.repository.util.SolrSearchIndex;
import org.entrystore.repository.util.StringUtils;
import org.quartz.impl.StdSchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.BooleanSupplier;
import java.util.zip.GZIPOutputStream;
public class RepositoryManagerImpl implements RepositoryManager {
private static final Logger log = LoggerFactory.getLogger(RepositoryManagerImpl.class);
private static final Map<RDFFormat, RDFWriterFactory> EXPORT_WRITER_FACTORIES = Map.of(
RDFFormat.TRIG, new TriGWriterFactory(),
RDFFormat.TRIX, new TriXWriterFactory(),
RDFFormat.NQUADS, new NQuadsWriterFactory(),
RDFFormat.BINARY, new BinaryRDFWriterFactory());
/**
* The formats {@link #exportToFile(Repository, URI, boolean, RDFFormat)} writes: those that keep named graphs.
*/
public static final Set<RDFFormat> EXPORT_FORMATS = EXPORT_WRITER_FACTORIES.keySet();
@Getter
private Repository repository;
private ContextManagerImpl contextManager;
private PrincipalManager principalManager;
private URL baseURL;
@Setter
@Getter
private boolean checkForAuthorization = true;
private final ArrayList<String> systemContextAliasList = new ArrayList<>();
private final static Map<String, RepositoryManagerImpl> instances = Collections.synchronizedMap(new HashMap<>());
private final Map<String, Class> alias2Class = new HashMap<>();
private boolean modificationLockout = false;
private boolean shutdown = false;
private final Object mutex = new Object();
@Getter
private final SoftCache softCache;
@Getter
private final Config configuration;
private final boolean quotaEnabled;
@Getter
private long defaultQuota = Quota.VALUE_UNLIMITED;
/**
* Default of {@link Settings#DATA_MAX_FILE_SIZE}, so that a mistaken upload cannot fill the data folder. It
* applies to uploads only; files already stored stay as they are. -1 removes the limit, as in 5.x.
*/
static final long DEFAULT_MAXIMUM_FILE_SIZE = 1024L * 1024 * 1024;
@Getter
private long maximumFileSize = DEFAULT_MAXIMUM_FILE_SIZE;
//ThreadPoolExecutor listenerExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(15);
private final Map<RepositoryEvent, Set<RepositoryListener>> repositoryListeners = new EnumMap<>(RepositoryEvent.class);
private SolrClient solrServer;
private SolrSearchIndex solrIndex;
@Getter
private PublicRepository publicRepository;
@Getter
private Repository provenanceRepository;
static boolean trackDeletedEntries;
/**
* Whether reading one named graph of the main store scans the whole store: a native or LMDB store without a
* configured index starting with the context. Unset indexes count, since the indexes an existing store was created
* with are unknown here, and RDF4J creates spoc,posc.
* Package-private and non-final so tests can make a memory store read as one that scans.
*/
volatile boolean readingGraphsByScan;
private static String VERSION = null;
public RepositoryManagerImpl(String baseURL, Config configuration) {
System.setProperty("org.openrdf.repository.debug", "true");
this.configuration = configuration;
String storeType = configuration.getString(Settings.STORE_TYPE, "memory").trim();
log.info("Store type: {}", storeType);
if (storeType.equalsIgnoreCase("memory")) {
log.info("Using Memory Store");
if (configuration.containsKey(Settings.STORE_PATH)) {
MemoryStore ms = new MemoryStore(new File(configuration.getURI(Settings.STORE_PATH)));
ms.setPersist(true);
ms.setSyncDelay(5000);
this.repository = new SailRepository(ms);
} else {
this.repository = new SailRepository(new MemoryStore());
}
} else if (storeType.equalsIgnoreCase("native")) {
if (!configuration.containsKey(Settings.STORE_PATH)) {
log.error("Incomplete configuration for Native store");
throw new IllegalStateException("Incomplete configuration");
} else {
File path = new File(configuration.getURI(Settings.STORE_PATH));
String indexes = configuration.getString(Settings.STORE_INDEXES);
boolean wal = configuration.getBoolean(Settings.STORE_WAL, false);
checkAndUpgradeNativeStore(path, indexes);
log.info("Main repository: using Native Store at {} with indexes {}, value-store WAL {}",
path, indexes, walState(wal));
readingGraphsByScan = !hasContextFirstIndex(indexes);
warnIfNoContextFirstIndex(indexes);
this.repository = new SailRepository(newNativeStore(path, indexes, wal));
}
} else if (storeType.equalsIgnoreCase("http")) {
if (!configuration.containsKey(Settings.STORE_URL)) {
log.error("Incomplete configuration for HTTP store");
throw new IllegalStateException("Incomplete configuration");
} else {
String url = configuration.getString(Settings.STORE_URL);
String user = configuration.getString(Settings.STORE_USER);
String password = configuration.getString(Settings.STORE_PWD);
log.info("Using HTTP repository at {}", url);
this.repository = new HTTPRepository(url);
if (user != null && password != null) {
((HTTPRepository) this.repository).setUsernameAndPassword(user, password);
}
}
} else if (storeType.equalsIgnoreCase("sparql")) {
if (!configuration.containsKey(Settings.STORE_ENDPOINT_QUERY) ||
!configuration.containsKey(Settings.STORE_ENDPOINT_UPDATE)) {
log.error("Incomplete configuration for sparql store");
throw new IllegalStateException("Incomplete configuration");
} else {
String endpointQuery = configuration.getString(Settings.STORE_ENDPOINT_QUERY);
String endpointUpdate = configuration.getString(Settings.STORE_ENDPOINT_UPDATE);
String user = configuration.getString(Settings.STORE_USER);
String password = configuration.getString(Settings.STORE_PWD);
log.info("Using SPARQL repository at {}, {}", endpointQuery, endpointUpdate);
this.repository = new SPARQLRepository(endpointQuery, endpointUpdate);
if (user != null && password != null) {
((SPARQLRepository) this.repository).setUsernameAndPassword(user, password);
}
}
} else if (storeType.equalsIgnoreCase("lmdb")) {
if (!configuration.containsKey(Settings.STORE_PATH)) {
log.error("Incomplete configuration for LMDB store");
throw new IllegalStateException("Incomplete configuration");
} else {
File path = new File(configuration.getURI(Settings.STORE_PATH));
String indexes = configuration.getString(Settings.STORE_INDEXES);
log.info("Main repository: using LMDB Store at {} with indexes {}", path, indexes);
readingGraphsByScan = !hasContextFirstIndex(indexes);
warnIfNoContextFirstIndex(indexes);
LmdbStore store;
if (indexes != null) {
store = new LmdbStore(path, new LmdbStoreConfig().setTripleIndexes(indexes));
} else {
store = new LmdbStore(path);
}
this.repository = new SailRepository(store);
}
}
if (this.repository == null) {
log.error("Failed to create SailRepository");
throw new IllegalStateException("Failed to create SailRepository");
}
// create soft cache
softCache = new SoftCache();
quotaEnabled = configuration.getBoolean(Settings.DATA_QUOTA, false);
if (quotaEnabled) {
log.info("Context quotas enabled");
String quotaValue = configuration.getString(Settings.DATA_QUOTA_DEFAULT);
if (quotaValue == null) {
log.info("Quota default set to UNLIMITED");
} else {
defaultQuota = StringUtils.convertUnitStringToByteSize(quotaValue);
log.info("Quota default set to {} bytes", defaultQuota);
}
} else {
log.info("Context quotas disabled");
}
String maxFileSizeValue = configuration.getString(Settings.DATA_MAX_FILE_SIZE);
if (maxFileSizeValue != null) {
maximumFileSize = StringUtils.convertUnitStringToByteSize(maxFileSizeValue);
}
if (maximumFileSize == Quota.VALUE_UNLIMITED) {
log.info("Maximum file size set to UNLIMITED");
} else {
log.info("Maximum file size set to {} bytes", maximumFileSize);
}
setCheckForAuthorization(false);
try {
try {
this.baseURL = new URI(baseURL).toURL();
NS.getMap().put("store", baseURL);
} catch (MalformedURLException | URISyntaxException e1) {
log.error(e1.getMessage());
}
systemContextAliasList.add("_contexts");
systemContextAliasList.add("_principals");
alias2Class.put("_contexts", ContextManagerImpl.class);
alias2Class.put("_principals", PrincipalManagerImpl.class);
try {
repository.init();
// Only once the store is ours: a second instance fails on its lock instead of deleting the
// staging files of uploads the running instance has in progress.
String dataFolderValue = configuration.getString(Settings.DATA_FOLDER);
if (dataFolderValue != null) {
DataImpl.deleteStaleStagingFiles(dataFolderValue);
}
} catch (RepositoryException e) {
log.error(e.getMessage());
}
if (configuration.getBoolean(Settings.REPOSITORY_PROVENANCE, false)) {
initializeProvenanceRepository();
}
this.initialize();
String baseURI = configuration.getString(Settings.BASE_URL);
if (instances.containsKey(baseURI) || instances.containsValue(this)) {
log.warn("This RepositoryManager instance has already been created, something is wrong");
} else {
log.info("Adding RepositoryManager instance to map: {},{}", baseURI, this);
instances.put(baseURI, this);
}
} finally {
setCheckForAuthorization(true);
}
registerGroupCacheInvalidationListener();
trackDeletedEntries = configuration.getBoolean(Settings.REPOSITORY_TRACK_DELETED, false);
log.info("Tracking of deleted entries is {}", trackDeletedEntries ? "activated" : "deactivated");
boolean cleanupDeleted = configuration.getBoolean(Settings.REPOSITORY_TRACK_DELETED_CLEANUP, false);
if (cleanupDeleted) {
DataCorrection.cleanupTrackedDeletedEntries(repository);
}
if (configuration.getBoolean(Settings.SOLR, false) && configuration.containsKey(Settings.SOLR_URL)) {
log.info("Initializing Solr");
initSolr();
registerSolrListeners();
}
if (configuration.getBoolean(Settings.REPOSITORY_PUBLIC, false)) {
log.info("Initializing public repository");
publicRepository = new PublicRepository(this);
registerPublicRepositoryListeners();
}
// after initSolr: a removal resumed without the index would skip purging the context's documents
contextManager.contextRemover.resumePending();
// after the context removals, which a tree removal in a removed context is left to
contextManager.listTreeRemover.resumePending();
log.info("Adding shutdown hook");
Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown));
}
private void initializeProvenanceRepository() {
String storeType = configuration.getString(Settings.REPOSITORY_PROVENANCE_TYPE, "memory").trim();
log.info("Provenance repository type: " + storeType);
if (storeType.equalsIgnoreCase("memory")) {
this.provenanceRepository = new SailRepository(new MemoryStore());
} else if (storeType.equalsIgnoreCase("native")) {
if (!configuration.containsKey(Settings.REPOSITORY_PROVENANCE_PATH)) {
log.error("Incomplete configuration of provenance repository");
} else {
File path = new File(configuration.getURI(Settings.REPOSITORY_PROVENANCE_PATH));
String indexes = configuration.getString(Settings.REPOSITORY_PROVENANCE_INDEXES);
boolean wal = configuration.getBoolean(Settings.REPOSITORY_PROVENANCE_WAL, false);
checkAndUpgradeNativeStore(path, indexes);
log.info("Provenance repository: using Native Store at {} with indexes {}, value-store WAL {}",
path, indexes, walState(wal));
this.provenanceRepository = new SailRepository(newNativeStore(path, indexes, wal));
}
}
try {
this.provenanceRepository.init();
} catch (RepositoryException e) {
log.error(e.getMessage());
}
}
/**
* Init all System Contexts
*/
private void initialize() {
this.contextManager = new ContextManagerImpl(this, repository);
this.contextManager.initializeSystemEntries();
}
/**
* Wires the user-to-groups authorization cache to repository events; the invalidation policy lives with the
* cache in {@link PrincipalManagerImpl#onRepositoryEvent(RepositoryEventObject)}. Registers nothing when the
* cache is disabled, so the kill switch also takes the listener out of every event dispatch.
*/
private void registerGroupCacheInvalidationListener() {
PrincipalManagerImpl principals = (PrincipalManagerImpl) getPrincipalManager();
if (!principals.isGroupCacheEnabled()) {
log.info("User-to-groups authorization cache is disabled");
return;
}
RepositoryListener invalidator = new RepositoryListener() {
@Override
public void repositoryUpdated(RepositoryEventObject eventObject) {
principals.onRepositoryEvent(eventObject);
}
};
registerListener(invalidator, RepositoryEvent.EntryCreated);
registerListener(invalidator, RepositoryEvent.EntryUpdated);
registerListener(invalidator, RepositoryEvent.ResourceUpdated);
registerListener(invalidator, RepositoryEvent.RelationsUpdated);
registerListener(invalidator, RepositoryEvent.EntryDeleted);
}
/**
* @return Returns a named instance of RepositoryManagerImpl. The method may
* return null if the RepositoryManager for the given base URI has
* not been created yet.
*/
public static RepositoryManagerImpl getInstance(String baseURI) {
// in a more complex setup (like applications running in several
// different JVMs) it is safe to use JNDI or JMX instead. For now we
// should be safe with this parametrized Singleton.
RepositoryManagerImpl rm = instances.get(baseURI);
if (rm != null) {
log.info("Instance found for {}", baseURI);
} else {
log.info("No instance found for {}", baseURI);
}
return rm;
}
/**
* Exports the whole repository, with its named graphs, to {@code file}. The file exists afterwards only if the
* export completed: a file this method opened is deleted again when the export or closing it fails.
*
* @param file File to export repository to.
* @throws IllegalArgumentException if {@code format} is not one of {@link #EXPORT_FORMATS}
* @throws IOException if the file cannot be opened, written or closed
* @throws org.eclipse.rdf4j.common.exception.RDF4JException if reading the repository or writing the RDF fails;
* a failed write to the file arrives as {@link org.eclipse.rdf4j.rio.RDFHandlerException}
*/
public void exportToFile(Repository repo, URI file, boolean gzip, RDFFormat format) throws IOException {
RDFWriterFactory rdfWriterFactory = format == null ? null : EXPORT_WRITER_FACTORIES.get(format);
if (rdfWriterFactory == null) {
throw new IllegalArgumentException("Unsupported export format: " + format);
}
long before = System.currentTimeMillis();
Path target = Path.of(file);
log.info("Exporting repository to {}", file);
try (RepositoryConnection con = repo.getConnection()) {
OutputStream fileOut = Files.newOutputStream(target);
try (fileOut; OutputStream out = new BufferedOutputStream(
gzip ? new GZIPOutputStream(fileOut, 64 * 1024) : fileOut)) {
con.export(rdfWriterFactory.getWriter(out));
} catch (Throwable e) {
try {
Files.deleteIfExists(target);
} catch (IOException deleteFailure) {
e.addSuppressed(deleteFailure);
}
throw e;
}
}
log.info("Export finished after {} ms", System.currentTimeMillis() - before);
}
@Override
public void shutdown() {
synchronized (mutex) {
if (!shutdown) {
shutdownQuietly("Quartz scheduler", () -> StdSchedulerFactory.getDefaultScheduler().shutdown());
// before Solr and the store, which a context removal still uses until it stops
if (contextManager != null) {
shutdownQuietly("context removal", contextManager.contextRemover::shutdown);
shutdownQuietly("list tree removal", contextManager.listTreeRemover::shutdown);
}
log.info("Shutting down repository listeners and executor");
//listenerExecutor.shutdown();
repositoryListeners.clear();
if (softCache != null) {
shutdownQuietly("SoftCache", softCache::shutdown);
}
if (solrIndex != null) {
shutdownQuietly("Solr support", solrIndex::shutdown);
}
if (repository != null) {
shutdownQuietly("RDF4J repository", repository::shutDown);
}
if (publicRepository != null) {
shutdownQuietly("public repository", publicRepository::shutdown);
}
if (provenanceRepository != null) {
shutdownQuietly("RDF4J provenance repository", provenanceRepository::shutDown);
}
if (solrServer != null) {
shutdownQuietly("Solr Server", solrServer::close);
}
shutdown = true;
}
}
}
@FunctionalInterface
private interface ShutdownAction {
void run() throws Exception;
}
private void shutdownQuietly(String name, ShutdownAction shutdownAction) {
log.info("Shutting down {}", name);
try {
shutdownAction.run();
} catch (Exception e) {
log.error("Error when shutting down {}: {}", name, e.getMessage(), e);
}
}
public ContextManager getContextManager() {
return this.contextManager;
}
public PrincipalManager getPrincipalManager() {
if (this.principalManager == null) {
if (this.contextManager != null) {
this.principalManager = (PrincipalManager) this.contextManager.getContext(systemContextAliasList.get(1));
}
}
return this.principalManager;
}
public URL getRepositoryURL() {
return this.baseURL;
}
public List<String> getSystemContextAliases() {
return systemContextAliasList;
}
public Class getSystemContextClassForAlias(String alias) {
return alias2Class.get(alias);
}
public Class getRegularContextClass() {
return RegularContext.class;
}
public boolean hasModificationLockOut() {
return modificationLockout;
}
public void setModificationLockOut(boolean lockout) {
log.info("Lock out set to {}", lockout);
this.modificationLockout = lockout;
}
public boolean hasQuotas() {
return quotaEnabled;
}
/**
* Dispatches {@code eventObject} synchronously to every listener registered for its event and for
* {@link RepositoryEvent#All}. Dispatch is failure-isolated against a listener's RuntimeException: it is logged
* and the remaining listeners still run, so callers never see one. An Error is not isolated — it propagates and
* abandons the rest of the dispatch, which can leave the authorization cache invalidator unreached and its
* cached group sets stale, since listener order is unspecified.
*/
public void fireRepositoryEvent(RepositoryEventObject eventObject) {
// because of concurrency problems the events are fired synchronously,
// not sure whether this event has a negative impact on performance.
synchronized (repositoryListeners) {
dispatch(repositoryListeners.get(eventObject.getEvent()), eventObject);
dispatch(repositoryListeners.get(RepositoryEvent.All), eventObject);
}
}
private void dispatch(Set<RepositoryListener> listeners, RepositoryEventObject eventObject) {
if (listeners == null) {
return;
}
for (RepositoryListener repositoryListener : listeners) {
try {
repositoryListener.repositoryUpdated(eventObject);
} catch (RuntimeException e) {
Object source = eventObject.getSource() instanceof Entry entry ? entry.getEntryURI() : eventObject.getSource();
log.error("Repository listener {} failed on {} for {}; remaining listeners still ran",
repositoryListener, eventObject.getEvent(), source, e);
}
}
}
public void registerListener(RepositoryListener listener, RepositoryEvent event) {
synchronized (repositoryListeners) {
Set<RepositoryListener> listeners = repositoryListeners.get(event);
if (listeners == null) {
listeners = new HashSet<>();
}
listeners.add(listener);
repositoryListeners.put(event, listeners);
log.info("Registered new RepositoryListener: {}", listener);
}
}
public void unregisterListener(RepositoryListener listener, RepositoryEvent event) {
synchronized (repositoryListeners) {
Set<RepositoryListener> listeners = repositoryListeners.get(event);
if (listeners != null) {
listeners.remove(listener);
repositoryListeners.put(event, listeners);
log.info("Unregistered RepositoryListener: {}", listener);
}
}
}
private void initSolr() {
try {
initSolrInternal();
} catch (RuntimeException | Error e) {
// Release RDF4J native-store file locks and any other resources before propagating;
// otherwise stale lock files block subsequent boots. Errors are included because an open
// Solr client keeps non-daemon threads alive, so the JVM would not exit after the failed
// startup. shutdown() can itself throw, so surface any cleanup failure as a suppressed
// cause to preserve the original cause.
try {
this.shutdown();
} catch (RuntimeException | Error cleanup) {
e.addSuppressed(cleanup);
}
throw e;
}
}
private void initSolrInternal() {
boolean reindex = configuration.getBoolean(Settings.SOLR_REINDEX_ON_STARTUP, false);
boolean reindexWait = configuration.getBoolean(Settings.SOLR_REINDEX_ON_STARTUP_WAIT, false);
// Memory store without persistence loses state across restarts, so the Solr index must be rebuilt.
if (!reindex && "memory".equalsIgnoreCase(configuration.getString(Settings.STORE_TYPE)) && !configuration.containsKey(Settings.STORE_PATH)) {
reindex = true;
}
String dataFolderPath = configuration.getString(Settings.DATA_FOLDER);
File dataFolder = dataFolderPath == null ? null : new File(dataFolderPath);
String currentVersion = getVersion();
boolean versionResolved = isVersionResolved(currentVersion);
if (dataFolder == null) {
log.warn("'{}' is not configured; Solr schema/version mismatch detection is disabled", Settings.DATA_FOLDER);
} else if (!versionResolved) {
// Mirror the pre-existing safety net: when the version cannot be resolved (missing
// VERSION.txt, unfiltered Maven token) force a reindex and skip marker persistence
// so the next restart re-triggers the same reindex until packaging is fixed.
log.warn("EntryStore version is unresolved ('{}'); forcing reindex and skipping Solr version-marker persistence until VERSION.txt resolution is fixed.", currentVersion);
reindex = true;
reindexWait = true;
} else {
try {
if (SolrVersionMarker.needsReindex(dataFolder, currentVersion)) {
reindex = true;
reindexWait = true;
}
} catch (IOException e) {
log.error("Failed to read Solr version markers from {}; forcing reindex as a precaution", dataFolder, e);
reindex = true;
reindexWait = true;
}
}
String solrURL = configuration.getString(Settings.SOLR_URL);
if (solrURL == null || solrURL.isBlank()) {
throw new IllegalStateException("'" + Settings.SOLR_URL + "' is not configured; set it in entrystore.properties or pass --" + Settings.SOLR_URL + "=... on the command line");
}
if (solrURL.startsWith("http://") || solrURL.startsWith("https://")) {
log.info("Using HTTP Solr server at {}", solrURL);
int requestTimeoutSeconds = configuration.getInt(Settings.SOLR_REQUEST_TIMEOUT_SECONDS, 60);
HttpJettySolrClient.Builder solrClientBuilder = new HttpJettySolrClient.Builder(solrURL)
// useHttp1_1(true) is a hard Jetty transport choice (HttpClientTransportOverHTTP), not a
// JDK-HttpClient-style preference. Structurally avoids HTTP/2 RST_STREAM frames from h2c.
.useHttp1_1(true)
.withConnectionTimeout(5, TimeUnit.SECONDS)
.withIdleTimeout(30, TimeUnit.SECONDS)
.withRequestTimeout(requestTimeoutSeconds, TimeUnit.SECONDS);
String solrUsername = configuration.getString(Settings.SOLR_AUTH_USERNAME);
String solrPassword = configuration.getString(Settings.SOLR_AUTH_PASSWORD);
if (solrUsername != null && solrPassword != null) {
solrClientBuilder.withBasicAuthCredentials(solrUsername, solrPassword);
}
solrServer = solrClientBuilder.build();
} else {
throw new IllegalStateException("Embedded Solr is no longer supported; '" + Settings.SOLR_URL + "' must be an http(s) URL");
}
if (solrServer != null) {
boolean schemaVerified = SolrSchemaCheck.requireFields(solrServer, solrURL,
SolrSearchIndex.REQUIRED_SCHEMA_FIELDS);
solrIndex = new SolrSearchIndex(this, solrServer);
boolean reindexSucceeded = false;
SearchIndex.ReindexResult reindexResult = null;
boolean reindexComplete = false;
if (reindex) {
if (reindexWait) {
if (!solrIndex.clearSolrIndex(solrServer)) {
log.error("Initial Solr full-wipe failed; skipping reindex to avoid serving a dirty index. Next restart will retry.");
} else {
long rejectedBefore = solrIndex.getRejectedDocumentCount();
reindexResult = solrIndex.reindexSync();
reindexComplete = isReindexComplete(reindexResult, solrIndex::waitForQueueDrain);
long rejected = solrIndex.getRejectedDocumentCount() - rejectedBefore;
// On a verified schema a document Solr always rejects must not block the markers, or every restart
// would wipe and reindex; unverified, rejections may be a schema mismatch the next start re-checks.
reindexSucceeded = reindexComplete && (schemaVerified || rejected == 0);
if (reindexComplete && rejected > 0 && !schemaVerified) {
log.error("Solr rejected {} documents during the initial reindex and the Solr schema could not be verified; skipping version-marker write so the next restart checks the schema again and re-triggers reindex. Check the earlier rejection logs for the reason.", rejected);
} else if (reindexComplete && rejected > 0) {
log.error("Solr rejected {} documents during the initial reindex; they are missing from the index. Check the earlier rejection logs for the reason, for example a literal too long for the field. A plain restart does not reindex, so once the cause is fixed start once with {}=on.", rejected, Settings.SOLR_REINDEX_ON_STARTUP);
}
}
} else {
log.info("Async reindex started; Solr version markers will not be persisted on this run because '{}=false' means reindex runs on every boot.",
Settings.SOLR_REINDEX_ON_STARTUP_WAIT);
solrIndex.reindex();
}
}
// The version markers record that a full reindex ran to completion, see isReindexComplete
boolean persistMarkers = dataFolder != null && reindex && reindexSucceeded && versionResolved;
if (reindexResult != null) {
if (!reindexComplete) {
log.warn("Solr {}; skipping version-marker write so the next restart re-triggers reindex.",
reindexResult.interrupted() ? "reindex was interrupted" : "submission queue did not drain");
}
if (reindexResult.hasFailures()) {
log.error("Solr reindex could not index {} context(s) and {} entries (logged above). They are"
+ " missing from the search index{}; once the cause is resolved, a reindex adds them again"
+ " (POST /management/solr with {\"command\": \"reindex\"}), and an entry is also added"
+ " again when it is modified.",
reindexResult.failedContexts(), reindexResult.failedEntries(),
persistMarkers ? " and a restart does not retry them" : "");
}
}
if (persistMarkers) {
boolean schemaWritten = SolrVersionMarker.write(SolrVersionMarker.schemaMarker(dataFolder), currentVersion);
boolean solrWritten = SolrVersionMarker.write(SolrVersionMarker.solrMarker(dataFolder), SolrVersion.LATEST.toString());
if (!schemaWritten || !solrWritten) {
log.warn("Failed to persist Solr version markers in {}; the next restart will detect them as missing and trigger another full reindex until the underlying I/O issue is resolved.", dataFolder);
}
}
} else {
log.error("Unable to initialize Solr");
this.shutdown();
}
}
/**
* Decides whether a synchronous reindex ran to completion, which is what the Solr version markers record:
* it was not interrupted and its documents left the submission queue. Contexts and entries that could not
* be indexed do not count against completion. Most of them fail because of their data and would fail again,
* and withholding the markers for them would wipe the index and reindex everything on every start. That is
* accepted for entries that fail for a possibly transient reason as well, e.g. an error of the store: they stay
* missing from the index until they are modified or reindexed (a context that fails as a whole is retried once).
*
* @param waitForQueueDrain waits until the submission queue is empty and returns false if interrupted; it is
* not called after an interrupted reindex, since it has no timeout and only an
* interruption ends it early
*/
static boolean isReindexComplete(SearchIndex.ReindexResult result, BooleanSupplier waitForQueueDrain) {
return !result.interrupted() && waitForQueueDrain.getAsBoolean();
}
private void registerSolrListeners() {
if (solrServer != null) {
RepositoryListener updater = new RepositoryListener() {
@Override
public void repositoryUpdated(RepositoryEventObject eventObject) {
if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry)) {
solrIndex.postEntry((Entry) eventObject.getSource());
}
}
};
registerListener(updater, RepositoryEvent.EntryCreated);
registerListener(updater, RepositoryEvent.EntryUpdated);
registerListener(updater, RepositoryEvent.MetadataUpdated);
registerListener(updater, RepositoryEvent.ExternalMetadataUpdated);
registerListener(updater, RepositoryEvent.ResourceUpdated);
registerListener(updater, RepositoryEvent.RelationsUpdated);
RepositoryListener remover = new RepositoryListener() {
@Override
public void repositoryUpdated(RepositoryEventObject eventObject) {
if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry)) {
solrIndex.removeEntry((Entry) eventObject.getSource());
}
}
};
registerListener(remover, RepositoryEvent.EntryDeleted);
RepositoryListener contextIndexer = new RepositoryListener() {
@Override
public void repositoryUpdated(RepositoryEventObject eventObject) {
if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry e)) {
if (GraphType.Context.equals(e.getGraphType())) {
solrIndex.submitContextForDelayedReindex(e.getEntryURI());
}
}
}
};
registerListener(contextIndexer, RepositoryEvent.EntryAclGuestUpdated);
registerListener(contextIndexer, RepositoryEvent.EntryProjectTypeUpdated);
}
}
public SearchIndex getIndex() {
return this.solrIndex;
}
private void registerPublicRepositoryListeners() {
if (publicRepository != null) {
RepositoryListener updater = new RepositoryListener() {
@Override
public void repositoryUpdated(RepositoryEventObject eventObject) {
if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry)) {
publicRepository.enqueue((Entry) eventObject.getSource());
}
}
};
registerListener(updater, RepositoryEvent.EntryCreated);
registerListener(updater, RepositoryEvent.EntryUpdated);
registerListener(updater, RepositoryEvent.MetadataUpdated);
registerListener(updater, RepositoryEvent.ExternalMetadataUpdated);
registerListener(updater, RepositoryEvent.ResourceUpdated);
registerListener(updater, RepositoryEvent.RelationsUpdated);
// delete
RepositoryListener remover = new RepositoryListener() {
@Override
public void repositoryUpdated(RepositoryEventObject eventObject) {
if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry)) {
publicRepository.remove((Entry) eventObject.getSource());
}
}
};
registerListener(remover, RepositoryEvent.EntryDeleted);
}
}
public ValueFactory getValueFactory() {
if (repository != null) {
return repository.getValueFactory();
}
return null;
}
/**
* Warns when the main store may lack a triple index starting with the context: removing a context, importing one
* or deleting a list recursively then reads graphs with a scan of the whole store.
*/
private static void warnIfNoContextFirstIndex(String indexes) {
if (indexes == null) {
log.warn("{} is not set, so the main store keeps the triple indexes it was created with; RDF4J creates "
+ "spoc,posc, which has no index starting with the context (c) and makes removing a context, "
+ "importing one or deleting a list recursively scan the whole store for the graphs they read. "
+ "Set it to cspo,posc", Settings.STORE_INDEXES);
} else if (!hasContextFirstIndex(indexes)) {
log.warn("The main store has no triple index starting with the context (c) in {}={}, which makes "
+ "removing a context, importing one or deleting a list recursively scan the whole store for "
+ "the graphs they read. Add cspo, e.g. cspo,posc", Settings.STORE_INDEXES, indexes);
}
}
/** Whether a comma-separated list of RDF4J triple indexes has one starting with the context. */
static boolean hasContextFirstIndex(String indexes) {
return indexes != null && Arrays.stream(indexes.split(",")).anyMatch(index -> index.trim().startsWith("c"));
}
/**
* Creates a Native Store with the RDF4J value-store WAL on or off. The WAL only lets RDF4J repair corrupt values
* in values.dat; commits do not depend on it. RDF4J reads an existing WAL completely into the heap whenever the
* store opens, so with the WAL on the heap needed at startup grows with the number of values in the store. A WAL
* directory left by an earlier run is reported, never deleted: it may only be removed while EntryStore is stopped.
*
* @param indexes the index specification, or null for the RDF4J default
*/
static NativeStore newNativeStore(File path, String indexes, boolean walEnabled) {
NativeStore store = indexes != null ? new NativeStore(path, indexes) : new NativeStore(path);
store.setWalEnabled(walEnabled);
File walDirectory = new File(path, ValueStoreWalConfig.DEFAULT_DIRECTORY_NAME);
if (!walEnabled && walDirectory.isDirectory()) {
log.info("Value-store WAL at {} is unused and can be deleted while EntryStore is stopped", walDirectory);
}
return store;
}
static String walState(boolean walEnabled) {
return walEnabled ? "enabled" : "disabled";
}
protected void checkAndUpgradeNativeStore(File path, String indexes) {
if (path.exists() && path.isDirectory() && (Objects.requireNonNull(path.list()).length == 0 || new File(path, "nativerdf.ver").exists())) {
// no upgrade needed
return;
}
long before = new Date().getTime();
log.warn("Unable to detect version of Native Store at {}, attempting upgrade", path);
File valuesDat = new File(path, "values.dat");
if (!valuesDat.exists()) {
throw new RuntimeException(valuesDat + " not found, is the path of the Native Store (" + path + ") configured properly?");
}
File backupPath = new File(path, "backup");
if (backupPath.exists() && backupPath.isDirectory() && Objects.requireNonNull(backupPath.list()).length > 0) {
throw new RuntimeException(("Backup path at " + backupPath + " exists and is not empty"));
}
if (!backupPath.exists() && !backupPath.mkdirs()) {
throw new RuntimeException("Backup path at " + backupPath + " could not be created");
}
try {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(path.toPath())) {
for (Path p : stream) {
if (!"backup".equals(p.getFileName().toString())) {
log.debug("Moving {} to {}", p, new File(backupPath, p.getFileName().toString()).toPath());
Files.move(p, new File(backupPath, p.getFileName().toString()).toPath());
}
}
}
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
SailRepository oldRepo = new SailRepository(newNativeStore(backupPath, indexes, false));
oldRepo.init();
SailRepository newRepo = new SailRepository(newNativeStore(path, indexes, false));
newRepo.init();
long oldTripleCount;
long newTripleCount;
long oldContextCount;
long newContextCount;
try (RepositoryConnection rcOld = oldRepo.getConnection()) {
try (RepositoryConnection rcNew = newRepo.getConnection()) {
rcNew.add(rcOld.getStatements(null, null, null, false));
oldTripleCount = rcOld.size();
newTripleCount = rcNew.size();
oldContextCount = rcOld.getContextIDs().stream().count();
newContextCount = rcNew.getContextIDs().stream().count();
} finally {
newRepo.shutDown();
}
} finally {
oldRepo.shutDown();
}
if (oldTripleCount != newTripleCount || oldContextCount != newContextCount) {
throw new RuntimeException("Amount of triples or contexts in migrated Native Store at " + path + " is not the same as in original Native Store");
}
log.info("Repository statistics for {}: {} triples, {} named graphs", path, newTripleCount, newContextCount);
try {
FileUtils.deleteDirectory(backupPath);
} catch (IOException e) {
log.error(e.getMessage());
}
log.info("Native store at {} successfully upgraded, took {} ms", path, new Date().getTime() - before);
}
public long getNamedGraphCount() {
try (RepositoryConnection rc = repository.getConnection()) {
return rc.getContextIDs().stream().count();
}
}
public long getTripleCount() {
try (RepositoryConnection rc = repository.getConnection()) {
return rc.size();
}
}
public static String getVersion() {
if (VERSION == null) {
URI versionFile = ConfigurationManager.getConfigurationURI("VERSION.txt");
if (versionFile != null) {
try {
log.debug("Reading version number from {}", versionFile);
VERSION = IOUtils.toString(versionFile, StandardCharsets.UTF_8);
VERSION = VERSION.replace("\n", "");
VERSION = VERSION.replace("\r", "");
if (VERSION.contains("@project.version@")) {
log.error("VERSION.txt at {} contains unfiltered Maven token '@project.version@'; check core-impl/pom.xml resource-filtering rule.", versionFile);
}
} catch (IOException e) {
log.error("Failed to read EntryStore version from {}", versionFile, e);
}
}
if (VERSION == null) {
VERSION = "unknown";
}
}
return VERSION;
}
// True when the version string identifies a specific build. Used to gate Solr version-marker
// persistence so a future packaging regression (missing VERSION.txt, removed resource-filtering
// rule) cannot silently disable SolrVersionMarker's drift detection by persisting an unresolved
// sentinel that would always equal itself on the next boot.
static boolean isVersionResolved(String version) {
return version != null
&& !version.isBlank()
&& !"unknown".equals(version)
&& !version.contains("@project.version@");
}
}