RepositoryManagerImpl.java

/*
 * Copyright (c) 2007-2026 MetaSolutions AB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.entrystore.impl;

import lombok.Getter;
import lombok.Setter;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.solr.client.api.util.SolrVersion;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.jetty.HttpJettySolrClient;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.repository.RepositoryException;
import org.eclipse.rdf4j.repository.http.HTTPRepository;
import org.eclipse.rdf4j.repository.sail.SailRepository;
import org.eclipse.rdf4j.repository.sparql.SPARQLRepository;
import org.eclipse.rdf4j.rio.RDFFormat;
import org.eclipse.rdf4j.rio.RDFWriterFactory;
import org.eclipse.rdf4j.rio.binary.BinaryRDFWriterFactory;
import org.eclipse.rdf4j.rio.nquads.NQuadsWriterFactory;
import org.eclipse.rdf4j.rio.trig.TriGWriterFactory;
import org.eclipse.rdf4j.rio.trix.TriXWriterFactory;
import org.eclipse.rdf4j.sail.lmdb.LmdbStore;
import org.eclipse.rdf4j.sail.lmdb.config.LmdbStoreConfig;
import org.eclipse.rdf4j.sail.memory.MemoryStore;
import org.eclipse.rdf4j.sail.nativerdf.NativeStore;
import org.eclipse.rdf4j.sail.nativerdf.wal.ValueStoreWalConfig;
import org.entrystore.ContextManager;
import org.entrystore.Entry;
import org.entrystore.GraphType;
import org.entrystore.PrincipalManager;
import org.entrystore.Quota;
import org.entrystore.SearchIndex;
import org.entrystore.config.Config;
import org.entrystore.repository.RepositoryEvent;
import org.entrystore.repository.RepositoryEventObject;
import org.entrystore.repository.RepositoryListener;
import org.entrystore.repository.RepositoryManager;
import org.entrystore.repository.config.ConfigurationManager;
import org.entrystore.repository.config.Settings;
import org.entrystore.repository.util.DataCorrection;
import org.entrystore.repository.util.NS;
import org.entrystore.repository.util.SolrSearchIndex;
import org.entrystore.repository.util.StringUtils;
import org.quartz.impl.StdSchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.BooleanSupplier;
import java.util.zip.GZIPOutputStream;

public class RepositoryManagerImpl implements RepositoryManager {

	private static final Logger log = LoggerFactory.getLogger(RepositoryManagerImpl.class);

	private static final Map<RDFFormat, RDFWriterFactory> EXPORT_WRITER_FACTORIES = Map.of(
			RDFFormat.TRIG, new TriGWriterFactory(),
			RDFFormat.TRIX, new TriXWriterFactory(),
			RDFFormat.NQUADS, new NQuadsWriterFactory(),
			RDFFormat.BINARY, new BinaryRDFWriterFactory());

	/**
	 * The formats {@link #exportToFile(Repository, URI, boolean, RDFFormat)} writes: those that keep named graphs.
	 */
	public static final Set<RDFFormat> EXPORT_FORMATS = EXPORT_WRITER_FACTORIES.keySet();

	@Getter
	private Repository repository;

	private ContextManagerImpl contextManager;

	private PrincipalManager principalManager;

	private URL baseURL;

	@Setter
	@Getter
	private boolean checkForAuthorization = true;

	private final ArrayList<String> systemContextAliasList = new ArrayList<>();

	private final static Map<String, RepositoryManagerImpl> instances = Collections.synchronizedMap(new HashMap<>());

	private final Map<String, Class> alias2Class = new HashMap<>();

	private boolean modificationLockout = false;

	private boolean shutdown = false;

	private final Object mutex = new Object();

	@Getter
	private final SoftCache softCache;

	@Getter
	private final Config configuration;

	private final boolean quotaEnabled;

	@Getter
	private long defaultQuota = Quota.VALUE_UNLIMITED;

	/**
	 * Default of {@link Settings#DATA_MAX_FILE_SIZE}, so that a mistaken upload cannot fill the data folder. It
	 * applies to uploads only; files already stored stay as they are. -1 removes the limit, as in 5.x.
	 */
	static final long DEFAULT_MAXIMUM_FILE_SIZE = 1024L * 1024 * 1024;

	@Getter
	private long maximumFileSize = DEFAULT_MAXIMUM_FILE_SIZE;

	//ThreadPoolExecutor listenerExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(15);

	private final Map<RepositoryEvent, Set<RepositoryListener>> repositoryListeners = new EnumMap<>(RepositoryEvent.class);

	private SolrClient solrServer;

	private SolrSearchIndex solrIndex;

	@Getter
	private PublicRepository publicRepository;

	@Getter
	private Repository provenanceRepository;

	static boolean trackDeletedEntries;

	/**
	 * Whether reading one named graph of the main store scans the whole store: a native or LMDB store without a
	 * configured index starting with the context. Unset indexes count, since the indexes an existing store was created
	 * with are unknown here, and RDF4J creates spoc,posc.
	 * Package-private and non-final so tests can make a memory store read as one that scans.
	 */
	volatile boolean readingGraphsByScan;

	private static String VERSION = null;

	public RepositoryManagerImpl(String baseURL, Config configuration) {
		System.setProperty("org.openrdf.repository.debug", "true");
		this.configuration = configuration;
		String storeType = configuration.getString(Settings.STORE_TYPE, "memory").trim();

		log.info("Store type: {}", storeType);

		if (storeType.equalsIgnoreCase("memory")) {
			log.info("Using Memory Store");
			if (configuration.containsKey(Settings.STORE_PATH)) {
				MemoryStore ms = new MemoryStore(new File(configuration.getURI(Settings.STORE_PATH)));
				ms.setPersist(true);
				ms.setSyncDelay(5000);
				this.repository = new SailRepository(ms);
			} else {
				this.repository = new SailRepository(new MemoryStore());
			}
		} else if (storeType.equalsIgnoreCase("native")) {
			if (!configuration.containsKey(Settings.STORE_PATH)) {
				log.error("Incomplete configuration for Native store");
				throw new IllegalStateException("Incomplete configuration");
			} else {
				File path = new File(configuration.getURI(Settings.STORE_PATH));
				String indexes = configuration.getString(Settings.STORE_INDEXES);
				boolean wal = configuration.getBoolean(Settings.STORE_WAL, false);
				checkAndUpgradeNativeStore(path, indexes);
				log.info("Main repository: using Native Store at {} with indexes {}, value-store WAL {}",
						path, indexes, walState(wal));
				readingGraphsByScan = !hasContextFirstIndex(indexes);
				warnIfNoContextFirstIndex(indexes);
				this.repository = new SailRepository(newNativeStore(path, indexes, wal));
			}
		} else if (storeType.equalsIgnoreCase("http")) {
			if (!configuration.containsKey(Settings.STORE_URL)) {
				log.error("Incomplete configuration for HTTP store");
				throw new IllegalStateException("Incomplete configuration");
			} else {
				String url = configuration.getString(Settings.STORE_URL);
				String user = configuration.getString(Settings.STORE_USER);
				String password = configuration.getString(Settings.STORE_PWD);
				log.info("Using HTTP repository at {}", url);
				this.repository = new HTTPRepository(url);
				if (user != null && password != null) {
					((HTTPRepository) this.repository).setUsernameAndPassword(user, password);
				}
			}
		} else if (storeType.equalsIgnoreCase("sparql")) {
			if (!configuration.containsKey(Settings.STORE_ENDPOINT_QUERY) ||
				!configuration.containsKey(Settings.STORE_ENDPOINT_UPDATE)) {
				log.error("Incomplete configuration for sparql store");
				throw new IllegalStateException("Incomplete configuration");
			} else {
				String endpointQuery = configuration.getString(Settings.STORE_ENDPOINT_QUERY);
				String endpointUpdate = configuration.getString(Settings.STORE_ENDPOINT_UPDATE);
				String user = configuration.getString(Settings.STORE_USER);
				String password = configuration.getString(Settings.STORE_PWD);
				log.info("Using SPARQL repository at {}, {}", endpointQuery, endpointUpdate);
				this.repository = new SPARQLRepository(endpointQuery, endpointUpdate);
				if (user != null && password != null) {
					((SPARQLRepository) this.repository).setUsernameAndPassword(user, password);
				}
			}
		} else if (storeType.equalsIgnoreCase("lmdb")) {
			if (!configuration.containsKey(Settings.STORE_PATH)) {
				log.error("Incomplete configuration for LMDB store");
				throw new IllegalStateException("Incomplete configuration");
			} else {
				File path = new File(configuration.getURI(Settings.STORE_PATH));
				String indexes = configuration.getString(Settings.STORE_INDEXES);
				log.info("Main repository: using LMDB Store at {} with indexes {}", path, indexes);
				readingGraphsByScan = !hasContextFirstIndex(indexes);
				warnIfNoContextFirstIndex(indexes);
				LmdbStore store;
				if (indexes != null) {
					store = new LmdbStore(path, new LmdbStoreConfig().setTripleIndexes(indexes));
				} else {
					store = new LmdbStore(path);
				}
				this.repository = new SailRepository(store);
			}
		}

		if (this.repository == null) {
			log.error("Failed to create SailRepository");
			throw new IllegalStateException("Failed to create SailRepository");
		}

		// create soft cache
		softCache = new SoftCache();

		quotaEnabled = configuration.getBoolean(Settings.DATA_QUOTA, false);
		if (quotaEnabled) {
			log.info("Context quotas enabled");
			String quotaValue = configuration.getString(Settings.DATA_QUOTA_DEFAULT);
			if (quotaValue == null) {
				log.info("Quota default set to UNLIMITED");
			} else {
				defaultQuota = StringUtils.convertUnitStringToByteSize(quotaValue);
				log.info("Quota default set to {} bytes", defaultQuota);
			}
		} else {
			log.info("Context quotas disabled");
		}

		String maxFileSizeValue = configuration.getString(Settings.DATA_MAX_FILE_SIZE);
		if (maxFileSizeValue != null) {
			maximumFileSize = StringUtils.convertUnitStringToByteSize(maxFileSizeValue);
		}
		if (maximumFileSize == Quota.VALUE_UNLIMITED) {
			log.info("Maximum file size set to UNLIMITED");
		} else {
			log.info("Maximum file size set to {} bytes", maximumFileSize);
		}

		setCheckForAuthorization(false);
		try {
			try {
				this.baseURL = new URI(baseURL).toURL();
				NS.getMap().put("store", baseURL);
			} catch (MalformedURLException | URISyntaxException e1) {
				log.error(e1.getMessage());
			}

			systemContextAliasList.add("_contexts");
			systemContextAliasList.add("_principals");

			alias2Class.put("_contexts", ContextManagerImpl.class);
			alias2Class.put("_principals", PrincipalManagerImpl.class);

			try {
				repository.init();
				// Only once the store is ours: a second instance fails on its lock instead of deleting the
				// staging files of uploads the running instance has in progress.
				String dataFolderValue = configuration.getString(Settings.DATA_FOLDER);
				if (dataFolderValue != null) {
					DataImpl.deleteStaleStagingFiles(dataFolderValue);
				}
			} catch (RepositoryException e) {
				log.error(e.getMessage());
			}

			if (configuration.getBoolean(Settings.REPOSITORY_PROVENANCE, false)) {
				initializeProvenanceRepository();
			}

			this.initialize();

			String baseURI = configuration.getString(Settings.BASE_URL);
			if (instances.containsKey(baseURI) || instances.containsValue(this)) {
				log.warn("This RepositoryManager instance has already been created, something is wrong");
			} else {
				log.info("Adding RepositoryManager instance to map: {},{}", baseURI, this);
				instances.put(baseURI, this);
			}
		} finally {
			setCheckForAuthorization(true);
		}

		registerGroupCacheInvalidationListener();

		trackDeletedEntries = configuration.getBoolean(Settings.REPOSITORY_TRACK_DELETED, false);
		log.info("Tracking of deleted entries is {}", trackDeletedEntries ? "activated" : "deactivated");
		boolean cleanupDeleted = configuration.getBoolean(Settings.REPOSITORY_TRACK_DELETED_CLEANUP, false);
		if (cleanupDeleted) {
			DataCorrection.cleanupTrackedDeletedEntries(repository);
		}

		if (configuration.getBoolean(Settings.SOLR, false) && configuration.containsKey(Settings.SOLR_URL)) {
			log.info("Initializing Solr");
			initSolr();
			registerSolrListeners();
		}

		if (configuration.getBoolean(Settings.REPOSITORY_PUBLIC, false)) {
			log.info("Initializing public repository");
			publicRepository = new PublicRepository(this);
			registerPublicRepositoryListeners();
		}

		// after initSolr: a removal resumed without the index would skip purging the context's documents
		contextManager.contextRemover.resumePending();
		// after the context removals, which a tree removal in a removed context is left to
		contextManager.listTreeRemover.resumePending();

		log.info("Adding shutdown hook");
		Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown));
	}

	private void initializeProvenanceRepository() {
		String storeType = configuration.getString(Settings.REPOSITORY_PROVENANCE_TYPE, "memory").trim();
		log.info("Provenance repository type: " + storeType);

		if (storeType.equalsIgnoreCase("memory")) {
			this.provenanceRepository = new SailRepository(new MemoryStore());
		} else if (storeType.equalsIgnoreCase("native")) {
			if (!configuration.containsKey(Settings.REPOSITORY_PROVENANCE_PATH)) {
				log.error("Incomplete configuration of provenance repository");
			} else {
				File path = new File(configuration.getURI(Settings.REPOSITORY_PROVENANCE_PATH));
				String indexes = configuration.getString(Settings.REPOSITORY_PROVENANCE_INDEXES);
				boolean wal = configuration.getBoolean(Settings.REPOSITORY_PROVENANCE_WAL, false);
				checkAndUpgradeNativeStore(path, indexes);
				log.info("Provenance repository: using Native Store at {} with indexes {}, value-store WAL {}",
						path, indexes, walState(wal));
				this.provenanceRepository = new SailRepository(newNativeStore(path, indexes, wal));
			}
		}
		try {
			this.provenanceRepository.init();
		} catch (RepositoryException e) {
			log.error(e.getMessage());
		}
	}

	/**
	 * Init all System Contexts
	 */
	private void initialize() {
		this.contextManager = new ContextManagerImpl(this, repository);
		this.contextManager.initializeSystemEntries();
	}

	/**
	 * Wires the user-to-groups authorization cache to repository events; the invalidation policy lives with the
	 * cache in {@link PrincipalManagerImpl#onRepositoryEvent(RepositoryEventObject)}. Registers nothing when the
	 * cache is disabled, so the kill switch also takes the listener out of every event dispatch.
	 */
	private void registerGroupCacheInvalidationListener() {
		PrincipalManagerImpl principals = (PrincipalManagerImpl) getPrincipalManager();
		if (!principals.isGroupCacheEnabled()) {
			log.info("User-to-groups authorization cache is disabled");
			return;
		}
		RepositoryListener invalidator = new RepositoryListener() {
			@Override
			public void repositoryUpdated(RepositoryEventObject eventObject) {
				principals.onRepositoryEvent(eventObject);
			}
		};
		registerListener(invalidator, RepositoryEvent.EntryCreated);
		registerListener(invalidator, RepositoryEvent.EntryUpdated);
		registerListener(invalidator, RepositoryEvent.ResourceUpdated);
		registerListener(invalidator, RepositoryEvent.RelationsUpdated);
		registerListener(invalidator, RepositoryEvent.EntryDeleted);
	}

	/**
	 * @return Returns a named instance of RepositoryManagerImpl. The method may
	 * return null if the RepositoryManager for the given base URI has
	 * not been created yet.
	 */
	public static RepositoryManagerImpl getInstance(String baseURI) {
		// in a more complex setup (like applications running in several
		// different JVMs) it is safe to use JNDI or JMX instead. For now we
		// should be safe with this parametrized Singleton.

		RepositoryManagerImpl rm = instances.get(baseURI);
		if (rm != null) {
			log.info("Instance found for {}", baseURI);
		} else {
			log.info("No instance found for {}", baseURI);
		}

		return rm;
	}

	/**
	 * Exports the whole repository, with its named graphs, to {@code file}. The file exists afterwards only if the
	 * export completed: a file this method opened is deleted again when the export or closing it fails.
	 *
	 * @param file File to export repository to.
	 * @throws IllegalArgumentException if {@code format} is not one of {@link #EXPORT_FORMATS}
	 * @throws IOException if the file cannot be opened, written or closed
	 * @throws org.eclipse.rdf4j.common.exception.RDF4JException if reading the repository or writing the RDF fails;
	 * a failed write to the file arrives as {@link org.eclipse.rdf4j.rio.RDFHandlerException}
	 */
	public void exportToFile(Repository repo, URI file, boolean gzip, RDFFormat format) throws IOException {
		RDFWriterFactory rdfWriterFactory = format == null ? null : EXPORT_WRITER_FACTORIES.get(format);
		if (rdfWriterFactory == null) {
			throw new IllegalArgumentException("Unsupported export format: " + format);
		}

		long before = System.currentTimeMillis();
		Path target = Path.of(file);
		log.info("Exporting repository to {}", file);
		try (RepositoryConnection con = repo.getConnection()) {
			OutputStream fileOut = Files.newOutputStream(target);
			try (fileOut; OutputStream out = new BufferedOutputStream(
					gzip ? new GZIPOutputStream(fileOut, 64 * 1024) : fileOut)) {
				con.export(rdfWriterFactory.getWriter(out));
			} catch (Throwable e) {
				try {
					Files.deleteIfExists(target);
				} catch (IOException deleteFailure) {
					e.addSuppressed(deleteFailure);
				}
				throw e;
			}
		}
		log.info("Export finished after {} ms", System.currentTimeMillis() - before);
	}

	@Override
	public void shutdown() {
		synchronized (mutex) {
			if (!shutdown) {
				shutdownQuietly("Quartz scheduler", () -> StdSchedulerFactory.getDefaultScheduler().shutdown());
				// before Solr and the store, which a context removal still uses until it stops
				if (contextManager != null) {
					shutdownQuietly("context removal", contextManager.contextRemover::shutdown);
					shutdownQuietly("list tree removal", contextManager.listTreeRemover::shutdown);
				}
				log.info("Shutting down repository listeners and executor");
				//listenerExecutor.shutdown();
				repositoryListeners.clear();
				if (softCache != null) {
					shutdownQuietly("SoftCache", softCache::shutdown);
				}
				if (solrIndex != null) {
					shutdownQuietly("Solr support", solrIndex::shutdown);
				}
				if (repository != null) {
					shutdownQuietly("RDF4J repository", repository::shutDown);
				}
				if (publicRepository != null) {
					shutdownQuietly("public repository", publicRepository::shutdown);
				}
				if (provenanceRepository != null) {
					shutdownQuietly("RDF4J provenance repository", provenanceRepository::shutDown);
				}
				if (solrServer != null) {
					shutdownQuietly("Solr Server", solrServer::close);
				}
				shutdown = true;
			}
		}
	}

	@FunctionalInterface
	private interface ShutdownAction {
		void run() throws Exception;
	}

	private void shutdownQuietly(String name, ShutdownAction shutdownAction) {
		log.info("Shutting down {}", name);
		try {
			shutdownAction.run();
		} catch (Exception e) {
			log.error("Error when shutting down {}: {}", name, e.getMessage(), e);
		}
	}

	public ContextManager getContextManager() {
		return this.contextManager;
	}

	public PrincipalManager getPrincipalManager() {
		if (this.principalManager == null) {
			if (this.contextManager != null) {
				this.principalManager = (PrincipalManager) this.contextManager.getContext(systemContextAliasList.get(1));
			}
		}
		return this.principalManager;
	}

	public URL getRepositoryURL() {
		return this.baseURL;
	}

	public List<String> getSystemContextAliases() {
		return systemContextAliasList;
	}

	public Class getSystemContextClassForAlias(String alias) {
		return alias2Class.get(alias);
	}

	public Class getRegularContextClass() {
		return RegularContext.class;
	}

	public boolean hasModificationLockOut() {
		return modificationLockout;
	}

	public void setModificationLockOut(boolean lockout) {
		log.info("Lock out set to {}", lockout);
		this.modificationLockout = lockout;
	}

	public boolean hasQuotas() {
		return quotaEnabled;
	}

	/**
	 * Dispatches {@code eventObject} synchronously to every listener registered for its event and for
	 * {@link RepositoryEvent#All}. Dispatch is failure-isolated against a listener's RuntimeException: it is logged
	 * and the remaining listeners still run, so callers never see one. An Error is not isolated — it propagates and
	 * abandons the rest of the dispatch, which can leave the authorization cache invalidator unreached and its
	 * cached group sets stale, since listener order is unspecified.
	 */
	public void fireRepositoryEvent(RepositoryEventObject eventObject) {
		// because of concurrency problems the events are fired synchronously,
		// not sure whether this event has a negative impact on performance.
		synchronized (repositoryListeners) {
			dispatch(repositoryListeners.get(eventObject.getEvent()), eventObject);
			dispatch(repositoryListeners.get(RepositoryEvent.All), eventObject);
		}
	}

	private void dispatch(Set<RepositoryListener> listeners, RepositoryEventObject eventObject) {
		if (listeners == null) {
			return;
		}
		for (RepositoryListener repositoryListener : listeners) {
			try {
				repositoryListener.repositoryUpdated(eventObject);
			} catch (RuntimeException e) {
				Object source = eventObject.getSource() instanceof Entry entry ? entry.getEntryURI() : eventObject.getSource();
				log.error("Repository listener {} failed on {} for {}; remaining listeners still ran",
						repositoryListener, eventObject.getEvent(), source, e);
			}
		}
	}

	public void registerListener(RepositoryListener listener, RepositoryEvent event) {
		synchronized (repositoryListeners) {
			Set<RepositoryListener> listeners = repositoryListeners.get(event);
			if (listeners == null) {
				listeners = new HashSet<>();
			}
			listeners.add(listener);
			repositoryListeners.put(event, listeners);
			log.info("Registered new RepositoryListener: {}", listener);
		}
	}

	public void unregisterListener(RepositoryListener listener, RepositoryEvent event) {
		synchronized (repositoryListeners) {
			Set<RepositoryListener> listeners = repositoryListeners.get(event);
			if (listeners != null) {
				listeners.remove(listener);
				repositoryListeners.put(event, listeners);
				log.info("Unregistered RepositoryListener: {}", listener);
			}
		}
	}

	private void initSolr() {
		try {
			initSolrInternal();
		} catch (RuntimeException | Error e) {
			// Release RDF4J native-store file locks and any other resources before propagating;
			// otherwise stale lock files block subsequent boots. Errors are included because an open
			// Solr client keeps non-daemon threads alive, so the JVM would not exit after the failed
			// startup. shutdown() can itself throw, so surface any cleanup failure as a suppressed
			// cause to preserve the original cause.
			try {
				this.shutdown();
			} catch (RuntimeException | Error cleanup) {
				e.addSuppressed(cleanup);
			}
			throw e;
		}
	}

	private void initSolrInternal() {
		boolean reindex = configuration.getBoolean(Settings.SOLR_REINDEX_ON_STARTUP, false);
		boolean reindexWait = configuration.getBoolean(Settings.SOLR_REINDEX_ON_STARTUP_WAIT, false);

		// Memory store without persistence loses state across restarts, so the Solr index must be rebuilt.
		if (!reindex && "memory".equalsIgnoreCase(configuration.getString(Settings.STORE_TYPE)) && !configuration.containsKey(Settings.STORE_PATH)) {
			reindex = true;
		}

		String dataFolderPath = configuration.getString(Settings.DATA_FOLDER);
		File dataFolder = dataFolderPath == null ? null : new File(dataFolderPath);
		String currentVersion = getVersion();
		boolean versionResolved = isVersionResolved(currentVersion);
		if (dataFolder == null) {
			log.warn("'{}' is not configured; Solr schema/version mismatch detection is disabled", Settings.DATA_FOLDER);
		} else if (!versionResolved) {
			// Mirror the pre-existing safety net: when the version cannot be resolved (missing
			// VERSION.txt, unfiltered Maven token) force a reindex and skip marker persistence
			// so the next restart re-triggers the same reindex until packaging is fixed.
			log.warn("EntryStore version is unresolved ('{}'); forcing reindex and skipping Solr version-marker persistence until VERSION.txt resolution is fixed.", currentVersion);
			reindex = true;
			reindexWait = true;
		} else {
			try {
				if (SolrVersionMarker.needsReindex(dataFolder, currentVersion)) {
					reindex = true;
					reindexWait = true;
				}
			} catch (IOException e) {
				log.error("Failed to read Solr version markers from {}; forcing reindex as a precaution", dataFolder, e);
				reindex = true;
				reindexWait = true;
			}
		}

		String solrURL = configuration.getString(Settings.SOLR_URL);
		if (solrURL == null || solrURL.isBlank()) {
			throw new IllegalStateException("'" + Settings.SOLR_URL + "' is not configured; set it in entrystore.properties or pass --" + Settings.SOLR_URL + "=... on the command line");
		}
		if (solrURL.startsWith("http://") || solrURL.startsWith("https://")) {
			log.info("Using HTTP Solr server at {}", solrURL);

			int requestTimeoutSeconds = configuration.getInt(Settings.SOLR_REQUEST_TIMEOUT_SECONDS, 60);
			HttpJettySolrClient.Builder solrClientBuilder = new HttpJettySolrClient.Builder(solrURL)
					// useHttp1_1(true) is a hard Jetty transport choice (HttpClientTransportOverHTTP), not a
					// JDK-HttpClient-style preference. Structurally avoids HTTP/2 RST_STREAM frames from h2c.
					.useHttp1_1(true)
					.withConnectionTimeout(5, TimeUnit.SECONDS)
					.withIdleTimeout(30, TimeUnit.SECONDS)
					.withRequestTimeout(requestTimeoutSeconds, TimeUnit.SECONDS);

			String solrUsername = configuration.getString(Settings.SOLR_AUTH_USERNAME);
			String solrPassword = configuration.getString(Settings.SOLR_AUTH_PASSWORD);
			if (solrUsername != null && solrPassword != null) {
				solrClientBuilder.withBasicAuthCredentials(solrUsername, solrPassword);
			}

			solrServer = solrClientBuilder.build();
		} else {
			throw new IllegalStateException("Embedded Solr is no longer supported; '" + Settings.SOLR_URL + "' must be an http(s) URL");
		}
		if (solrServer != null) {
			boolean schemaVerified = SolrSchemaCheck.requireFields(solrServer, solrURL,
					SolrSearchIndex.REQUIRED_SCHEMA_FIELDS);
			solrIndex = new SolrSearchIndex(this, solrServer);
			boolean reindexSucceeded = false;
			SearchIndex.ReindexResult reindexResult = null;
			boolean reindexComplete = false;
			if (reindex) {
				if (reindexWait) {
					if (!solrIndex.clearSolrIndex(solrServer)) {
						log.error("Initial Solr full-wipe failed; skipping reindex to avoid serving a dirty index. Next restart will retry.");
					} else {
						long rejectedBefore = solrIndex.getRejectedDocumentCount();
						reindexResult = solrIndex.reindexSync();
						reindexComplete = isReindexComplete(reindexResult, solrIndex::waitForQueueDrain);
						long rejected = solrIndex.getRejectedDocumentCount() - rejectedBefore;
						// On a verified schema a document Solr always rejects must not block the markers, or every restart
						// would wipe and reindex; unverified, rejections may be a schema mismatch the next start re-checks.
						reindexSucceeded = reindexComplete && (schemaVerified || rejected == 0);
						if (reindexComplete && rejected > 0 && !schemaVerified) {
							log.error("Solr rejected {} documents during the initial reindex and the Solr schema could not be verified; skipping version-marker write so the next restart checks the schema again and re-triggers reindex. Check the earlier rejection logs for the reason.", rejected);
						} else if (reindexComplete && rejected > 0) {
							log.error("Solr rejected {} documents during the initial reindex; they are missing from the index. Check the earlier rejection logs for the reason, for example a literal too long for the field. A plain restart does not reindex, so once the cause is fixed start once with {}=on.", rejected, Settings.SOLR_REINDEX_ON_STARTUP);
						}
					}
				} else {
					log.info("Async reindex started; Solr version markers will not be persisted on this run because '{}=false' means reindex runs on every boot.",
							Settings.SOLR_REINDEX_ON_STARTUP_WAIT);
					solrIndex.reindex();
				}
			}
			// The version markers record that a full reindex ran to completion, see isReindexComplete
			boolean persistMarkers = dataFolder != null && reindex && reindexSucceeded && versionResolved;
			if (reindexResult != null) {
				if (!reindexComplete) {
					log.warn("Solr {}; skipping version-marker write so the next restart re-triggers reindex.",
							reindexResult.interrupted() ? "reindex was interrupted" : "submission queue did not drain");
				}
				if (reindexResult.hasFailures()) {
					log.error("Solr reindex could not index {} context(s) and {} entries (logged above). They are"
							+ " missing from the search index{}; once the cause is resolved, a reindex adds them again"
							+ " (POST /management/solr with {\"command\": \"reindex\"}), and an entry is also added"
							+ " again when it is modified.",
							reindexResult.failedContexts(), reindexResult.failedEntries(),
							persistMarkers ? " and a restart does not retry them" : "");
				}
			}
			if (persistMarkers) {
				boolean schemaWritten = SolrVersionMarker.write(SolrVersionMarker.schemaMarker(dataFolder), currentVersion);
				boolean solrWritten = SolrVersionMarker.write(SolrVersionMarker.solrMarker(dataFolder), SolrVersion.LATEST.toString());
				if (!schemaWritten || !solrWritten) {
					log.warn("Failed to persist Solr version markers in {}; the next restart will detect them as missing and trigger another full reindex until the underlying I/O issue is resolved.", dataFolder);
				}
			}
		} else {
			log.error("Unable to initialize Solr");
			this.shutdown();
		}
	}

	/**
	 * Decides whether a synchronous reindex ran to completion, which is what the Solr version markers record:
	 * it was not interrupted and its documents left the submission queue. Contexts and entries that could not
	 * be indexed do not count against completion. Most of them fail because of their data and would fail again,
	 * and withholding the markers for them would wipe the index and reindex everything on every start. That is
	 * accepted for entries that fail for a possibly transient reason as well, e.g. an error of the store: they stay
	 * missing from the index until they are modified or reindexed (a context that fails as a whole is retried once).
	 *
	 * @param waitForQueueDrain waits until the submission queue is empty and returns false if interrupted; it is
	 *                          not called after an interrupted reindex, since it has no timeout and only an
	 *                          interruption ends it early
	 */
	static boolean isReindexComplete(SearchIndex.ReindexResult result, BooleanSupplier waitForQueueDrain) {
		return !result.interrupted() && waitForQueueDrain.getAsBoolean();
	}

	private void registerSolrListeners() {
		if (solrServer != null) {
			RepositoryListener updater = new RepositoryListener() {
				@Override
				public void repositoryUpdated(RepositoryEventObject eventObject) {
					if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry)) {
						solrIndex.postEntry((Entry) eventObject.getSource());
					}
				}
			};
			registerListener(updater, RepositoryEvent.EntryCreated);
			registerListener(updater, RepositoryEvent.EntryUpdated);
			registerListener(updater, RepositoryEvent.MetadataUpdated);
			registerListener(updater, RepositoryEvent.ExternalMetadataUpdated);
			registerListener(updater, RepositoryEvent.ResourceUpdated);
			registerListener(updater, RepositoryEvent.RelationsUpdated);

			RepositoryListener remover = new RepositoryListener() {
				@Override
				public void repositoryUpdated(RepositoryEventObject eventObject) {
					if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry)) {
						solrIndex.removeEntry((Entry) eventObject.getSource());
					}
				}
			};
			registerListener(remover, RepositoryEvent.EntryDeleted);

			RepositoryListener contextIndexer = new RepositoryListener() {
				@Override
				public void repositoryUpdated(RepositoryEventObject eventObject) {
					if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry e)) {
						if (GraphType.Context.equals(e.getGraphType())) {
							solrIndex.submitContextForDelayedReindex(e.getEntryURI());
						}
					}
				}
			};
			registerListener(contextIndexer, RepositoryEvent.EntryAclGuestUpdated);
			registerListener(contextIndexer, RepositoryEvent.EntryProjectTypeUpdated);
		}
	}

	public SearchIndex getIndex() {
		return this.solrIndex;
	}

	private void registerPublicRepositoryListeners() {
		if (publicRepository != null) {
			RepositoryListener updater = new RepositoryListener() {
				@Override
				public void repositoryUpdated(RepositoryEventObject eventObject) {
					if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry)) {
						publicRepository.enqueue((Entry) eventObject.getSource());
					}
				}
			};
			registerListener(updater, RepositoryEvent.EntryCreated);
			registerListener(updater, RepositoryEvent.EntryUpdated);
			registerListener(updater, RepositoryEvent.MetadataUpdated);
			registerListener(updater, RepositoryEvent.ExternalMetadataUpdated);
			registerListener(updater, RepositoryEvent.ResourceUpdated);
			registerListener(updater, RepositoryEvent.RelationsUpdated);

			// delete
			RepositoryListener remover = new RepositoryListener() {
				@Override
				public void repositoryUpdated(RepositoryEventObject eventObject) {
					if ((eventObject.getSource() != null) && (eventObject.getSource() instanceof Entry)) {
						publicRepository.remove((Entry) eventObject.getSource());
					}
				}
			};
			registerListener(remover, RepositoryEvent.EntryDeleted);
		}
	}

	public ValueFactory getValueFactory() {
		if (repository != null) {
			return repository.getValueFactory();
		}
		return null;
	}

	/**
	 * Warns when the main store may lack a triple index starting with the context: removing a context, importing one
	 * or deleting a list recursively then reads graphs with a scan of the whole store.
	 */
	private static void warnIfNoContextFirstIndex(String indexes) {
		if (indexes == null) {
			log.warn("{} is not set, so the main store keeps the triple indexes it was created with; RDF4J creates "
					+ "spoc,posc, which has no index starting with the context (c) and makes removing a context, "
					+ "importing one or deleting a list recursively scan the whole store for the graphs they read. "
					+ "Set it to cspo,posc", Settings.STORE_INDEXES);
		} else if (!hasContextFirstIndex(indexes)) {
			log.warn("The main store has no triple index starting with the context (c) in {}={}, which makes "
					+ "removing a context, importing one or deleting a list recursively scan the whole store for "
					+ "the graphs they read. Add cspo, e.g. cspo,posc", Settings.STORE_INDEXES, indexes);
		}
	}

	/** Whether a comma-separated list of RDF4J triple indexes has one starting with the context. */
	static boolean hasContextFirstIndex(String indexes) {
		return indexes != null && Arrays.stream(indexes.split(",")).anyMatch(index -> index.trim().startsWith("c"));
	}

	/**
	 * Creates a Native Store with the RDF4J value-store WAL on or off. The WAL only lets RDF4J repair corrupt values
	 * in values.dat; commits do not depend on it. RDF4J reads an existing WAL completely into the heap whenever the
	 * store opens, so with the WAL on the heap needed at startup grows with the number of values in the store. A WAL
	 * directory left by an earlier run is reported, never deleted: it may only be removed while EntryStore is stopped.
	 *
	 * @param indexes the index specification, or null for the RDF4J default
	 */
	static NativeStore newNativeStore(File path, String indexes, boolean walEnabled) {
		NativeStore store = indexes != null ? new NativeStore(path, indexes) : new NativeStore(path);
		store.setWalEnabled(walEnabled);
		File walDirectory = new File(path, ValueStoreWalConfig.DEFAULT_DIRECTORY_NAME);
		if (!walEnabled && walDirectory.isDirectory()) {
			log.info("Value-store WAL at {} is unused and can be deleted while EntryStore is stopped", walDirectory);
		}
		return store;
	}

	static String walState(boolean walEnabled) {
		return walEnabled ? "enabled" : "disabled";
	}

	protected void checkAndUpgradeNativeStore(File path, String indexes) {
		if (path.exists() && path.isDirectory() && (Objects.requireNonNull(path.list()).length == 0 || new File(path, "nativerdf.ver").exists())) {
			// no upgrade needed
			return;
		}
		long before = new Date().getTime();
		log.warn("Unable to detect version of Native Store at {}, attempting upgrade", path);

		File valuesDat = new File(path, "values.dat");
		if (!valuesDat.exists()) {
			throw new RuntimeException(valuesDat + " not found, is the path of the Native Store (" + path + ") configured properly?");
		}

		File backupPath = new File(path, "backup");
		if (backupPath.exists() && backupPath.isDirectory() && Objects.requireNonNull(backupPath.list()).length > 0) {
			throw new RuntimeException(("Backup path at " + backupPath + " exists and is not empty"));
		}

		if (!backupPath.exists() && !backupPath.mkdirs()) {
			throw new RuntimeException("Backup path at " + backupPath + " could not be created");
		}

		try {
			try (DirectoryStream<Path> stream = Files.newDirectoryStream(path.toPath())) {
				for (Path p : stream) {
					if (!"backup".equals(p.getFileName().toString())) {
						log.debug("Moving {} to {}", p, new File(backupPath, p.getFileName().toString()).toPath());
						Files.move(p, new File(backupPath, p.getFileName().toString()).toPath());
					}
				}
			}
		} catch (IOException ioe) {
			throw new RuntimeException(ioe);
		}

		SailRepository oldRepo = new SailRepository(newNativeStore(backupPath, indexes, false));
		oldRepo.init();
		SailRepository newRepo = new SailRepository(newNativeStore(path, indexes, false));
		newRepo.init();

		long oldTripleCount;
		long newTripleCount;
		long oldContextCount;
		long newContextCount;

		try (RepositoryConnection rcOld = oldRepo.getConnection()) {
			try (RepositoryConnection rcNew = newRepo.getConnection()) {
				rcNew.add(rcOld.getStatements(null, null, null, false));
				oldTripleCount = rcOld.size();
				newTripleCount = rcNew.size();
				oldContextCount = rcOld.getContextIDs().stream().count();
				newContextCount = rcNew.getContextIDs().stream().count();
			} finally {
				newRepo.shutDown();
			}
		} finally {
			oldRepo.shutDown();
		}

		if (oldTripleCount != newTripleCount || oldContextCount != newContextCount) {
			throw new RuntimeException("Amount of triples or contexts in migrated Native Store at " + path + " is not the same as in original Native Store");
		}

		log.info("Repository statistics for {}: {} triples, {} named graphs", path, newTripleCount, newContextCount);

		try {
			FileUtils.deleteDirectory(backupPath);
		} catch (IOException e) {
			log.error(e.getMessage());
		}

		log.info("Native store at {} successfully upgraded, took {} ms", path, new Date().getTime() - before);
	}

	public long getNamedGraphCount() {
		try (RepositoryConnection rc = repository.getConnection()) {
			return rc.getContextIDs().stream().count();
		}
	}

	public long getTripleCount() {
		try (RepositoryConnection rc = repository.getConnection()) {
			return rc.size();
		}
	}

	public static String getVersion() {
		if (VERSION == null) {
			URI versionFile = ConfigurationManager.getConfigurationURI("VERSION.txt");
			if (versionFile != null) {
				try {
					log.debug("Reading version number from {}", versionFile);
					VERSION = IOUtils.toString(versionFile, StandardCharsets.UTF_8);
					VERSION = VERSION.replace("\n", "");
					VERSION = VERSION.replace("\r", "");
					if (VERSION.contains("@project.version@")) {
						log.error("VERSION.txt at {} contains unfiltered Maven token '@project.version@'; check core-impl/pom.xml resource-filtering rule.", versionFile);
					}
				} catch (IOException e) {
					log.error("Failed to read EntryStore version from {}", versionFile, e);
				}
			}
			if (VERSION == null) {
				VERSION = "unknown";
			}
		}
		return VERSION;
	}

	// True when the version string identifies a specific build. Used to gate Solr version-marker
	// persistence so a future packaging regression (missing VERSION.txt, removed resource-filtering
	// rule) cannot silently disable SolrVersionMarker's drift detection by persisting an unresolved
	// sentinel that would always equal itself on the next boot.
	static boolean isVersionResolved(String version) {
		return version != null
				&& !version.isBlank()
				&& !"unknown".equals(version)
				&& !version.contains("@project.version@");
	}

}