SolrSearchIndex.java
/*
* Copyright (c) 2007-2026 MetaSolutions AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.entrystore.repository.util;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.solr.client.solrj.RemoteSolrException;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.SolrQuery;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Literal;
import org.eclipse.rdf4j.model.Model;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.model.vocabulary.RDF;
import org.entrystore.AuthorizationException;
import org.entrystore.Context;
import org.entrystore.ContextManager;
import org.entrystore.Entry;
import org.entrystore.EntryType;
import org.entrystore.GraphType;
import org.entrystore.PrincipalManager;
import org.entrystore.PrincipalManager.AccessProperty;
import org.entrystore.SearchIndex;
import org.entrystore.User;
import org.entrystore.impl.ContextManagerImpl;
import org.entrystore.impl.LocalMetadataWrapper;
import org.entrystore.impl.RegularContext;
import org.entrystore.repository.CorruptEntryException;
import org.entrystore.repository.RepositoryManager;
import org.entrystore.repository.config.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.datatype.DatatypeConstants;
import javax.xml.datatype.XMLGregorianCalendar;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.StringJoiner;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
/**
* @author Hannes Ebner
*/
public class SolrSearchIndex implements SearchIndex {
private static final Logger log = LoggerFactory.getLogger(SolrSearchIndex.class);
private static final int BATCH_SIZE_ADD = 100;
private static final int BATCH_SIZE_DELETE = 100;
/**
* Documents the post queue may hold before a reindex waits for the submitter, so that the heap does not grow
* with the number of entries. Solr's ingest limits the reindex anyway; ten batches keep the submitter supplied
* and the queue above the threshold for the longer commitWithin.
*/
private static final int MAX_REINDEX_QUEUE_SIZE = BATCH_SIZE_ADD * 10;
private static final long REINDEX_QUEUE_POLL_MS = 20;
private static final int SOLR_COMMIT_WITHIN = 1000;
private static final int SOLR_COMMIT_WITHIN_MAX = 10000;
private static final long EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS = 10;
private static final long MAX_PURGE_WAIT_NANOS = TimeUnit.MINUTES.toNanos(5);
private static final long MAX_PURGE_TOTAL_WAIT_NANOS = TimeUnit.HOURS.toNanos(1);
private static final long PURGE_POLL_MILLIS = 5000;
static final int SENT_CHECK_CHUNK_SIZE = 1000;
/**
* Number of distinct entry failure causes per context reindex that are logged with a stack trace. Entries
* of one context tend to fail for a shared cause, and one trace per entry would flood the log; later
* failures with an already traced cause, and all failures beyond this limit, are logged as one line.
*/
private static final int MAX_ENTRY_FAILURE_TRACES = 5;
/**
* Number of attempts to reindex a context whose reindex fails as a whole during a synchronous reindex.
*/
private static final int MAX_CONTEXT_REINDEX_ATTEMPTS = 2;
private static final DateTimeFormatter SOLR_DATE_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC);
private final String defaultSortLang;
private final boolean extractFulltext;
private boolean related;
private Map<IRI, Boolean> relatedProperties = null;
private boolean relatedContainsGlobal = false;
private final RepositoryManager rm;
private final SolrClient solrServer;
private final Thread documentSubmitter;
private final Thread delayedContextIndexer;
private final Cache<URI, SolrInputDocument> postQueue = Caffeine.newBuilder().build();
private final int maxReindexQueueSize;
private volatile boolean shuttingDown;
/**
* Entries whose documents were taken from the submission queue by the submitter but whose add batch Solr has
* neither accepted nor rejected yet; a failed batch puts them back into the queue, unless their deletion was
* queued meanwhile, in which case they are dropped. Guarded by the submission queue's monitor, so that an entry
* is always in the queue, in flight, or settled.
*/
private final Set<URI> inFlightAdds = new HashSet<>();
/** How long a purge after a reindex waits without Solr answering any document; a field so tests can shorten it. */
long purgeStallNanos = MAX_PURGE_WAIT_NANOS;
/** How often a waiting purge checks whether its documents were sent; a field so tests can shorten it. */
long purgePollMillis = PURGE_POLL_MILLIS;
/**
* Opened by {@link #shutdown()} once the submitter has stopped, so that a waiting purge checks a last time
* instead of sleeping until the purge executor is forced down.
*/
private final CountDownLatch submitterStopped = new CountDownLatch(1);
/**
* Entries whose documents are to be removed from the index, in the order in which they were queued. A set, so
* that dropping the pending deletion of an entry when its document is queued does not scan all pending
* deletions.
*/
private final Set<URI> deleteQueue = Collections.synchronizedSet(new LinkedHashSet<>());
private final Map<URI, Future> reindexing = Collections.synchronizedMap(new HashMap<>());
private final ExecutorService reindexExecutor = Executors.newSingleThreadExecutor();
private final ExecutorService purgeExecutor = Executors.newVirtualThreadPerTaskExecutor();
// True while the submitter is between drain and process/requeue — observers awaiting a quiescent
// queue must wait for this to clear too, otherwise an in-flight batch can be missed.
private final AtomicBoolean submitterInFlight = new AtomicBoolean();
// Documents Solr rejected on an add batch. Startup withholds the version markers over rejections only when
// the schema could not be verified, so an unchecked schema mismatch is retried on the next start.
private final AtomicLong rejectedDocuments = new AtomicLong();
// Documents Solr has accepted or rejected; growing while a purge waits means the submitter is not stalled.
private final AtomicLong settledDocuments = new AtomicLong();
private final AtomicLong readFilterMismatches = new AtomicLong();
/** Whether the entry has an ACL of its own ({@link Entry#hasAllowedPrincipals()}), which replaces its context's. */
public static final String FIELD_ACL_EXPLICIT = "acl.explicit";
/** Entry URI of a reference's local target, whose ACL governs the reference's indexed metadata. */
public static final String FIELD_REFERENCED_ENTRY = "referencedEntry";
/**
* Fields and dynamic field patterns this version writes that an older {@code schema.xml} lacks; the startup
* schema check refuses a schema without them, since Solr would reject every document.
*/
public static final List<String> REQUIRED_SCHEMA_FIELDS = List.of(FIELD_ACL_EXPLICIT, FIELD_REFERENCED_ENTRY,
LangFacetValue.FIELD_PREFIX + "*", "related." + LangFacetValue.FIELD_PREFIX + "*");
/** Total {@code facet.query} clauses one {@link #facetCountsForLabels} request may carry, across all fields. */
public static final int MAX_COUNT_CLAUSES = 1600;
private final Set<String> fieldsKnownToHaveTerms = ConcurrentHashMap.newKeySet();
/** Context entry URI to the {@link System#nanoTime()} of its latest submission, immune to wall-clock steps. */
private final Map<URI, Long> delayedReindex = Collections.synchronizedMap(new HashMap<>());
private final ValueFactory valueFactory;
public class SolrInputDocumentSubmitter extends Thread {
private static final int MAX_RETRIES = 3;
private static final long RETRY_DELAY_MS = 2000;
private static final long FAILURE_COOLDOWN_MS = 10000;
@Override
public void run() {
while (!interrupted()) {
try {
postQueue.cleanUp();
boolean batchFailed = false;
if (postQueue.estimatedSize() > 0 || !deleteQueue.isEmpty()) {
if (!deleteQueue.isEmpty()) {
batchFailed = processDeleteBatch();
}
if (postQueue.estimatedSize() > 0 && !Thread.currentThread().isInterrupted()) {
batchFailed = processAddBatch() || batchFailed;
}
if (batchFailed) {
sleepOrShutdown(FAILURE_COOLDOWN_MS, "cooldown");
}
} else {
sleepOrShutdown(500, "idle wait");
}
} catch (ShutdownRequestedException e) {
return;
} catch (Exception e) {
log.error("Unexpected error in Solr document submitter, will retry on next iteration", e);
try {
sleepOrShutdown(FAILURE_COOLDOWN_MS, "post-error cooldown");
} catch (ShutdownRequestedException sre) {
return;
}
}
}
}
/**
* Sleeps for the given duration; on interruption, logs and throws
* {@link ShutdownRequestedException} to unwind the run loop cleanly.
*/
private void sleepOrShutdown(long millis, String phase) {
try {
Thread.sleep(millis);
} catch (InterruptedException ie) {
log.info("Solr document submitter got interrupted during {}, shutting down", phase);
throw new ShutdownRequestedException();
}
}
/**
* @return true if the batch failed permanently and a cooldown is needed
*/
private boolean processDeleteBatch() {
submitterInFlight.set(true);
try {
List<URI> deleteBatch = drainDeleteQueue();
if (deleteBatch.isEmpty()) {
return false;
}
StringJoiner joiner = new StringJoiner(" OR ", "uri:(", ")");
deleteBatch.forEach(uri -> joiner.add(ClientUtils.escapeQueryChars(uri.toString())));
UpdateRequest delReq = new UpdateRequest();
delReq.deleteByQuery(joiner.toString());
delReq.setCommitWithin(SOLR_COMMIT_WITHIN);
for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
log.info("Sending request to delete {} entries from Solr (attempt {}/{}), {} entries remaining in delete queue",
deleteBatch.size(), attempt, MAX_RETRIES, deleteQueue.size());
delReq.process(solrServer);
return false;
} catch (RuntimeException e) {
requeueDeletes(deleteBatch);
if (e.getCause() instanceof InterruptedException) {
log.info("Solr document submitter got interrupted during delete, re-queuing and shutting down");
Thread.currentThread().interrupt();
return true;
}
throw e;
} catch (SolrServerException | IOException e) {
log.warn("Failed to delete {} entries from Solr (attempt {}/{}): {}",
deleteBatch.size(), attempt, MAX_RETRIES, e.getMessage());
if (attempt < MAX_RETRIES && sleepForRetry(attempt)) {
requeueDeletes(deleteBatch);
return true;
}
}
}
log.error("Permanently failed to delete {} entries from Solr after {} attempts, re-queuing", deleteBatch.size(), MAX_RETRIES);
requeueDeletes(deleteBatch);
return true;
} finally {
submitterInFlight.set(false);
}
}
/**
* @return true if the batch failed permanently and a cooldown is needed
*/
private boolean processAddBatch() {
submitterInFlight.set(true);
Map<URI, SolrInputDocument> addBatch = Map.of();
try {
addBatch = drainPostQueue();
if (addBatch.isEmpty()) {
return false;
}
UpdateRequest addReq = new UpdateRequest();
addBatch.values().forEach(addReq::add);
postQueue.cleanUp();
if (postQueue.estimatedSize() > BATCH_SIZE_ADD * 5) {
addReq.setCommitWithin(SOLR_COMMIT_WITHIN_MAX);
} else {
addReq.setCommitWithin(SOLR_COMMIT_WITHIN);
}
for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
log.info("Sending {} entries to Solr (attempt {}/{}), {} entries remaining in post queue",
addBatch.size(), attempt, MAX_RETRIES, postQueue.estimatedSize());
addReq.process(solrServer);
settleAdds(addBatch);
return false;
} catch (RemoteSolrException e) {
log.error("Solr rejected {} entries (HTTP {}, discarding batch). URIs: {}",
addBatch.size(), e.code(), addBatch.keySet(), e);
rejectedDocuments.addAndGet(addBatch.size());
settleAdds(addBatch);
return false;
} catch (RuntimeException e) {
if (e.getCause() instanceof InterruptedException) {
log.info("Solr document submitter got interrupted during send, re-queuing and shutting down");
Thread.currentThread().interrupt();
return true;
}
throw e;
} catch (SolrServerException | IOException e) {
log.warn("Failed to send {} entries to Solr (attempt {}/{}): {}",
addBatch.size(), attempt, MAX_RETRIES, e.getMessage());
if (attempt < MAX_RETRIES && sleepForRetry(attempt)) {
return true;
}
}
}
log.error("Permanently failed to send {} entries to Solr after {} attempts, re-queuing", addBatch.size(), MAX_RETRIES);
return true;
} finally {
// Also for an Error escaping the send, which must not leave the batch in flight forever
requeueAdds(addBatch);
submitterInFlight.set(false);
}
}
/**
* Drains up to {@link SolrSearchIndex#BATCH_SIZE_DELETE} entries from the delete queue.
*/
private List<URI> drainDeleteQueue() {
List<URI> batch = new ArrayList<>();
synchronized (deleteQueue) {
Iterator<URI> it = deleteQueue.iterator();
while (batch.size() < BATCH_SIZE_DELETE && it.hasNext()) {
batch.add(it.next());
it.remove();
}
}
return batch;
}
/**
* Drains up to {@link SolrSearchIndex#BATCH_SIZE_ADD} entries from the post queue; they are in flight until
* {@link #settleAdds} or {@link #requeueAdds}.
*/
private Map<URI, SolrInputDocument> drainPostQueue() {
Map<URI, SolrInputDocument> batch = new HashMap<>();
synchronized (postQueue) {
ConcurrentMap<URI, SolrInputDocument> postQueueMap = postQueue.asMap();
Iterator<URI> it = postQueueMap.keySet().iterator();
while (batch.size() < BATCH_SIZE_ADD && it.hasNext()) {
URI key = it.next();
SolrInputDocument doc = postQueueMap.get(key);
postQueueMap.remove(key, doc);
if (doc == null) {
log.warn("Value for key {} is null in Solr submit queue", key);
continue;
}
batch.put(key, doc);
inFlightAdds.add(key);
}
}
return batch;
}
/**
* Marks the entries of a batch that Solr accepted or rejected as no longer in flight.
*/
private void settleAdds(Map<URI, SolrInputDocument> batch) {
synchronized (postQueue) {
inFlightAdds.removeAll(batch.keySet());
}
settledDocuments.addAndGet(batch.size());
}
/**
* Puts the deletions of a failed batch back into the delete queue, except those of entries with a queued
* document: that document was queued after the deletion, since queueing a deletion drops the entry's
* queued document, so the deletion must not remove it.
*/
private void requeueDeletes(List<URI> batch) {
synchronized (postQueue) {
synchronized (deleteQueue) {
for (URI entryURI : batch) {
if (postQueue.getIfPresent(entryURI) == null) {
deleteQueue.add(entryURI);
}
}
}
}
}
/**
* Puts the documents of a failed batch that are still in flight back into the post queue, except those of
* entries with a queued deletion or document: either was queued after the batch was drained, since queueing
* a document drops the entry's queued deletion, and supersedes the batch's document. Documents already
* settled or requeued are left alone, so the call is safe on every exit of {@link #processAddBatch()}.
*/
private void requeueAdds(Map<URI, SolrInputDocument> batch) {
synchronized (postQueue) {
synchronized (deleteQueue) {
batch.forEach((entryURI, document) -> {
if (inFlightAdds.remove(entryURI) && !deleteQueue.contains(entryURI)) {
postQueue.asMap().putIfAbsent(entryURI, document);
}
});
}
}
}
/**
* Sleeps for a retry backoff delay.
*
* @return true if the sleep was interrupted (caller should re-queue and shut down)
*/
private boolean sleepForRetry(int attempt) {
try {
Thread.sleep(RETRY_DELAY_MS * attempt);
return false;
} catch (InterruptedException ie) {
log.info("Solr document submitter got interrupted during retry backoff, shutting down");
Thread.currentThread().interrupt();
return true;
}
}
/**
* Signals that the submitter thread should shut down (used to exit the run loop on interrupt).
*/
private static class ShutdownRequestedException extends RuntimeException {
}
}
public static class FacetSettings {
public String fields;
public int minCount;
public int limit;
public String matches;
public boolean missing;
/** Optional BCP 47 language range: literal facets keep only labels occurring in it, or untagged. */
public String lang;
}
public class DelayedContextIndexer extends Thread {
@Override
public void run() {
while (!interrupted()) {
reindexDue(System.nanoTime());
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
log.info("Solr delayed context indexer got interrupted, shutting down thread");
return;
}
}
}
}
public SolrSearchIndex(RepositoryManager rm, SolrClient solrServer) {
this(rm, solrServer, MAX_REINDEX_QUEUE_SIZE);
}
SolrSearchIndex(RepositoryManager rm, SolrClient solrServer, int maxReindexQueueSize) {
this.maxReindexQueueSize = maxReindexQueueSize;
this.rm = rm;
valueFactory = this.rm.getValueFactory();
this.solrServer = solrServer;
extractFulltext = rm.getConfiguration().getBoolean(Settings.SOLR_EXTRACT_FULLTEXT, false);
related = rm.getConfiguration().getBoolean(Settings.SOLR_RELATED, false);
defaultSortLang = rm.getConfiguration().getString(Settings.SOLR_DEFAULT_SORTING_LANG);
if (related) {
List<String> relPropsSetting = rm.getConfiguration().getStringList(Settings.SOLR_RELATED_PROPERTIES, new ArrayList<>());
if (relPropsSetting.isEmpty()) {
related = false;
} else {
relatedProperties = new HashMap<>();
for (String relProp : rm.getConfiguration().getStringList(Settings.SOLR_RELATED_PROPERTIES, new ArrayList<>())) {
if (relProp.endsWith(",global")) {
relatedProperties.put(valueFactory.createIRI(relProp.substring(0, relProp.indexOf(","))), true);
} else {
relatedProperties.put(valueFactory.createIRI(relProp), false);
}
}
relatedContainsGlobal = relatedProperties.containsValue(true);
}
}
documentSubmitter = new SolrInputDocumentSubmitter();
documentSubmitter.setDaemon(true);
documentSubmitter.start();
delayedContextIndexer = new DelayedContextIndexer();
delayedContextIndexer.setDaemon(true);
delayedContextIndexer.start();
}
public void shutdown() {
shuttingDown = true;
if (documentSubmitter != null) {
documentSubmitter.interrupt();
}
if (delayedContextIndexer != null) {
delayedContextIndexer.interrupt();
}
reindexExecutor.shutdown();
// `&=` (not `&&`) so we always join both threads even if the first didn't terminate cleanly.
boolean threadsClean = true;
if (documentSubmitter != null) {
threadsClean &= joinOrWarn(documentSubmitter, "Solr document submitter");
}
if (delayedContextIndexer != null) {
threadsClean &= joinOrWarn(delayedContextIndexer, "Delayed context indexer");
}
awaitOrForceShutdown(reindexExecutor, "Reindex executor");
submitterStopped.countDown();
// Purge executor shuts down only after reindex tasks have fully drained, so any
// purgeExecutor.submit(...) calls they make complete before shutdown begins.
purgeExecutor.shutdown();
awaitOrForceShutdown(purgeExecutor, "Purge executor");
if (threadsClean) {
try {
log.debug("Sending commit to Solr");
solrServer.commit(true, false);
} catch (SolrServerException | IOException e) {
log.error(e.getMessage());
}
} else {
log.warn("Skipping final Solr commit because background threads did not terminate cleanly within {}s",
EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS);
}
}
private boolean joinOrWarn(Thread thread, String name) {
try {
thread.join(TimeUnit.SECONDS.toMillis(EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS));
if (thread.isAlive()) {
log.warn("{} did not terminate within {}s after interrupt; subsequent Solr operations may fail",
name, EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS);
return false;
}
return true;
} catch (InterruptedException e) {
log.warn("Interrupted while joining {}", name);
Thread.currentThread().interrupt();
return false;
}
}
private void awaitOrForceShutdown(ExecutorService executor, String name) {
try {
if (!executor.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
forceShutdown(executor, name);
}
} catch (InterruptedException e) {
forceShutdown(executor, name);
Thread.currentThread().interrupt();
}
}
private void forceShutdown(ExecutorService executor, String name) {
List<Runnable> dropped = executor.shutdownNow();
log.warn("{} did not terminate within {}s; forced shutdownNow (running tasks interrupted, {} queued tasks dropped)",
name, EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS, dropped.size());
try {
if (!executor.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
log.warn("{} still has running tasks after shutdownNow; tasks may be ignoring interrupts or blocked in non-interruptible I/O",
name);
}
} catch (InterruptedException e) {
log.warn("Interrupted while awaiting forced termination of {}", name);
Thread.currentThread().interrupt();
}
}
public boolean clearSolrIndex(SolrClient solrServer) {
UpdateRequest req = new UpdateRequest();
req.deleteByQuery("*:*");
req.setCommitWithin(SOLR_COMMIT_WITHIN);
try {
req.process(solrServer);
return true;
} catch (SolrServerException | IOException e) {
log.error("clearSolrIndex (full-wipe *:*) failed: {}", e.getMessage(), e);
return false;
}
}
/**
* Also drops the queued documents of the context's entries, which the submitter would otherwise send after
* the delete-by-query, and settles those being sent, so a failed batch does not queue them again; a batch
* already being sent can still land after it.
*/
@Override
public boolean removeContext(URI contextURI) {
String entryURIPrefix = contextURI + "/";
synchronized (postQueue) {
postQueue.asMap().keySet().removeIf(entryURI -> entryURI.toString().startsWith(entryURIPrefix));
inFlightAdds.removeIf(entryURI -> entryURI.toString().startsWith(entryURIPrefix));
}
return clearSolrIndex(solrServer, null, contextURI);
}
public boolean clearSolrIndex(SolrClient solrServer, Date expirationDate, URI contextURI) {
if (solrServer == null || (expirationDate == null && contextURI == null)) {
throw new IllegalArgumentException("Too many parameters are null");
}
UpdateRequest req = new UpdateRequest();
String deleteQuery = "";
if (expirationDate != null) {
String solrExpirationDate = ClientUtils.escapeQueryChars(SOLR_DATE_FORMATTER.format(expirationDate.toInstant()));
deleteQuery += "indexedAt:[* TO " + solrExpirationDate + "}";
}
if (contextURI != null) {
if (!deleteQuery.isEmpty()) {
deleteQuery += " AND ";
}
deleteQuery += "context:" + ClientUtils.escapeQueryChars(contextURI.toString());
}
req.deleteByQuery(deleteQuery);
req.setCommitWithin(SOLR_COMMIT_WITHIN);
try {
req.process(solrServer);
return true;
} catch (SolrServerException | IOException e) {
log.error("clearSolrIndex failed for query {}: {}", deleteQuery, e.getMessage(), e);
return false;
}
}
/**
* Re-indexes every context in Solr, one indexing task per context, cancelling a task already
* running for the same context. Failures are logged by the tasks and not reported to the caller;
* see {@link #reindex(URI)}.
*/
public void reindex() {
for (URI contextURI : listContextsAsAdmin()) {
reindex(contextURI);
}
}
/**
* Re-indexes a context and all of its entries in Solr. Starts a new indexing task and cancels an
* eventually existing one for the same context. Expired documents are purged after the context
* has been reposted; see {@link #reindexSync(URI)}.
*
* @param contextURI The URI of the context to be re-indexed, never null.
*/
public void reindex(URI contextURI) {
if (contextURI == null) {
throw new IllegalArgumentException("Context URI must not be null");
}
synchronized (reindexing) {
if (reindexing.containsKey(contextURI)) {
Future existingIndexer = reindexing.get(contextURI);
if (!existingIndexer.isDone()) {
log.info("Cancelling existing indexer thread for {}", contextURI);
existingIndexer.cancel(true);
}
reindexing.remove(contextURI);
}
AtomicReference<Future<?>> self = new AtomicReference<>();
Future<?> indexer = reindexExecutor.submit(() -> {
try {
reindexSync(contextURI);
} catch (RuntimeException | Error e) {
// The Future is never inspected, so without this the failure would vanish without a trace.
log.error("Reindexing of context {} failed", contextURI, e);
} finally {
// Remove only this task's mapping: a cancelled task that is still unwinding must not remove
// the mapping of its replacement. Acquire the monitor before reading self, so the submitter
// has assigned the future and installed its mapping even if this task finishes immediately.
synchronized (reindexing) {
reindexing.remove(contextURI, self.get());
}
}
});
self.set(indexer);
reindexing.put(contextURI, indexer);
}
}
/**
* Re-indexes all contexts in the calling thread, so that a single corrupt context does not prevent the
* remaining contexts from being indexed:
* <ul>
* <li>A context whose reindex fails as a whole is logged and retried once after all other contexts, since
* the failure may be transient, e.g. an I/O error of the store. If it fails again, it is counted as failed.
* Besides runtime exceptions this covers {@link StackOverflowError}, which corrupt data can cause through
* unbounded recursion and after which the JVM is usable again; other errors are propagated.</li>
* <li>A context that cannot be resolved and entries that cannot be loaded or indexed are logged and
* counted.</li>
* </ul>
* An interrupted reindex stops, and the interrupt status of the calling thread is kept.
*
* @return the number of contexts and entries that could not be indexed, and whether the reindex was
* interrupted before all contexts were processed
*/
public ReindexResult reindexSync() {
if (solrServer == null) {
log.warn("Ignoring request as Solr is not used by this instance");
return new ReindexResult(0, 0, false);
}
int failedContexts = 0;
int failedEntries = 0;
if (Thread.currentThread().isInterrupted()) {
return new ReindexResult(0, 0, true);
}
Collection<URI> pendingContexts = listContextsAsAdmin();
for (int attempt = 1; attempt <= MAX_CONTEXT_REINDEX_ATTEMPTS && !pendingContexts.isEmpty(); attempt++) {
List<URI> abortedContexts = new ArrayList<>();
for (URI contextURI : pendingContexts) {
if (Thread.currentThread().isInterrupted()) {
return new ReindexResult(failedContexts + abortedContexts.size(), failedEntries, true);
}
ContextPostResult posted;
try {
posted = reindexContext(contextURI);
} catch (RuntimeException | StackOverflowError e) {
abortedContexts.add(contextURI);
if (Thread.currentThread().isInterrupted()) {
log.info("Reindexing of context {} was interrupted; stopping without retry", contextURI);
return new ReindexResult(failedContexts + abortedContexts.size(), failedEntries, true);
}
if (attempt < MAX_CONTEXT_REINDEX_ATTEMPTS) {
log.error("Reindexing of context {} failed; it is retried after the remaining contexts",
contextURI, e);
} else {
log.error("Reindexing of context {} failed again, giving up", contextURI, e);
}
continue;
}
failedEntries += posted.notIndexedEntries();
boolean interrupted = switch (posted.outcome()) {
case COMPLETED -> false;
case UNRESOLVED -> {
failedContexts++;
yield false;
}
case INTERRUPTED -> true;
};
if (interrupted || Thread.currentThread().isInterrupted()) {
return new ReindexResult(failedContexts + abortedContexts.size(), failedEntries, true);
}
}
pendingContexts = abortedContexts;
}
return new ReindexResult(failedContexts + pendingContexts.size(), failedEntries,
Thread.currentThread().isInterrupted());
}
private Set<URI> listContextsAsAdmin() {
PrincipalManager pm = rm.getPrincipalManager();
URI currentUser = pm.getAuthenticatedUserURI();
try {
pm.setAuthenticatedUserURI(pm.getAdminUser().getURI());
return rm.getContextManager().getEntries();
} finally {
pm.setAuthenticatedUserURI(currentUser);
}
}
/**
* Re-indexes a context: every entry is queued for reposting on the calling thread, which waits while
* the queue is full, and a purge of the documents this reindex did not repost is scheduled to run once
* Solr has accepted or rejected every queued document, so search never sees a gap. The method returns
* before the queue drains or the purge runs. Shutdown, and a stopped submitter while the queue is full,
* end the reindex as an interruption does. The conditions under which no purge happens are listed at
* {@link #purgeExpiredDocumentsAfterReindex}; the purge is abandoned once Solr has answered no document for
* MAX_PURGE_WAIT_NANOS, after MAX_PURGE_TOTAL_WAIT_NANOS in all, or on shutdown.
*
* @param contextURI The URI of the context to be re-indexed, never null.
*/
public void reindexSync(URI contextURI) {
if (solrServer == null) {
log.warn("Ignoring request as Solr is not used by this instance");
return;
}
reindexContext(contextURI);
}
private ContextPostResult reindexContext(URI contextURI) {
if (contextURI == null) {
throw new IllegalArgumentException("Context URI must not be null");
}
// Reindex tasks still queued on the executor run after shutdown(); they end here without logging
if (shuttingDown) {
return ContextPostResult.interrupted();
}
log.info("Starting Solr reindexing of context " + contextURI);
Entry contextEntry = rm.getContextManager().getByEntryURI(contextURI);
// no purge on an incomplete URI index: it would delete the unlisted entries (ENTRYSTORE-1095)
boolean purgeIsSafe = indexIsComplete(contextURI);
Date reindexStart = new Date();
PrincipalManager pm = rm.getPrincipalManager();
URI currentUser = pm.getAuthenticatedUserURI();
try {
pm.setAuthenticatedUserURI(pm.getAdminUser().getURI());
ContextPostResult posted = postContextEntriesToQueue(contextURI);
switch (posted.outcome()) {
case COMPLETED -> log.info("Finished Solr reindexing of context {}, took {} ms; the Solr submission"
+ " queue may still contain yet to be processed documents",
contextURI, new Date().getTime() - reindexStart.getTime());
case UNRESOLVED -> log.warn("Context {} could not be resolved; its entries were not reindexed and its"
+ " documents in the index were left unchanged", contextURI);
case INTERRUPTED -> log.info(
"Reindexing of context {} was interrupted; expired documents are not purged", contextURI);
}
purgeExpiredDocumentsAfterReindex(contextURI, contextEntry, reindexStart, purgeIsSafe, posted);
return posted;
} finally {
pm.setAuthenticatedUserURI(currentUser);
}
}
/**
* Removes the documents of a context that were indexed before its reindex started, as soon as none of the
* documents the reindex posted is in the submission queue or in an add batch that Solr has not answered yet:
* the submitter drains the queue in no particular order, and a failed batch is put back into the queue. A
* document the reindex posted whose entry is deleted meanwhile counts as sent, since its deletion is queued
* and the purge may remove the old document too; one that is replaced by a newer document of the same entry
* is awaited as that document. Documents Solr rejects are discarded and count as sent; the purge then warns that
* entries of the context may be missing from search. The wait is given up when Solr has answered no document at
* all for MAX_PURGE_WAIT_NANOS, so a large backlog ahead of the context is awaited but a stalled submitter is
* not, and after MAX_PURGE_TOTAL_WAIT_NANOS in all. On shutdown, a waiting purge checks once more after the
* submitter has stopped and ends if documents are still unsent. Nothing is purged if
* <ul>
* <li>the context's URI index is incomplete, because the purge would remove the documents of the entries
* missing from its listing (ENTRYSTORE-1095),</li>
* <li>the context could not be resolved or its reindex was interrupted,</li>
* <li>entries of the context could not be loaded because of a failure that is not caused by their data,
* e.g. of the store, or loaded but could not be indexed, because their previous documents would be removed
* without being replaced although the entries may still be loadable,</li>
* <li>the context lists no entries, which saves a Solr request for each empty context, or</li>
* <li>the context entry could not be loaded, because the delete query would then not be restricted to the
* context.</li>
* </ul>
* Entries that are corrupt or cannot be found do not prevent the purge: they cannot be returned by a search, and
* the purge removes the documents of entries that cannot be found, see {@link #postContextEntriesToQueue(URI)}.
*/
private void purgeExpiredDocumentsAfterReindex(URI contextURI, Entry contextEntry, Date reindexStart,
boolean purgeIsSafe, ContextPostResult posted) {
if (!purgeIsSafe || posted.outcome() != ContextPostOutcome.COMPLETED) {
return;
}
if (posted.keptEntries() > 0) {
log.warn("Not purging expired documents of context {} because {} of its entries could not be indexed;"
+ " their previously indexed documents are kept", contextURI, posted.keptEntries());
return;
}
if (posted.listedEntries() == 0) {
return;
}
if (contextEntry == null) {
log.warn("Not purging expired documents of context {} because its context entry could not be loaded",
contextURI);
return;
}
Set<URI> pending = new HashSet<>(posted.queuedEntryURIs());
purgeExecutor.submit(() -> {
long giveUp = System.nanoTime() + MAX_PURGE_TOTAL_WAIT_NANOS;
long deadline = System.nanoTime() + purgeStallNanos;
long settledBefore = settledDocuments.get();
long rejectedBefore = rejectedDocuments.get();
boolean stopped = false;
try {
removeSentDocuments(pending);
while (!pending.isEmpty()
&& !stopped
&& System.nanoTime() < deadline
&& System.nanoTime() < giveUp
&& !Thread.currentThread().isInterrupted()) {
log.debug("{} documents of context {} are still queued or being sent, checking again in {} ms"
+ " before attempting new purge of expired entries", pending.size(), contextURI,
purgePollMillis);
stopped = submitterStopped.await(purgePollMillis, TimeUnit.MILLISECONDS);
removeSentDocuments(pending);
// The wait is bounded by a stalled submitter, not by the size of the backlog ahead of the context
long settled = settledDocuments.get();
if (settled != settledBefore) {
settledBefore = settled;
deadline = System.nanoTime() + purgeStallNanos;
}
}
if (Thread.currentThread().isInterrupted()) {
log.warn("Delayed purge of context {} interrupted (shutdown); expired entries were NOT removed",
contextURI);
return;
}
if (!pending.isEmpty()) {
if (stopped) {
log.warn("Delayed purge of context {} ended by shutdown; {} of its documents were not sent and"
+ " expired entries were NOT removed", contextURI, pending.size());
} else if (System.nanoTime() < giveUp) {
log.warn("Delayed purge of context {} aborted after {} s without Solr answering any document;"
+ " {} of its documents are still queued or being sent", contextURI,
TimeUnit.NANOSECONDS.toSeconds(purgeStallNanos), pending.size());
} else {
log.warn("Delayed purge of context {} aborted after {} h; the submission backlog did not drain"
+ " in time and {} of its documents are still queued or being sent", contextURI,
TimeUnit.NANOSECONDS.toHours(MAX_PURGE_TOTAL_WAIT_NANOS), pending.size());
}
return;
}
long rejected = rejectedDocuments.get() - rejectedBefore;
if (rejected > 0) {
log.warn("Solr rejected {} documents while the purge of context {} waited; entries of the context"
+ " may be missing from search results until a reindex after the cause is fixed",
rejected, contextURI);
}
if (clearSolrIndex(solrServer, reindexStart, contextEntry.getResourceURI())) {
log.info("Expired entries of context {} have been purged from the index", contextURI);
} else {
log.warn("Delayed purge of context {} did not clear expired entries; rerun the reindex if needed",
contextURI);
}
} catch (InterruptedException e) {
log.warn("Delayed purge of context {} interrupted before clearSolrIndex completed; expired entries were NOT removed",
contextURI, e);
Thread.currentThread().interrupt();
} catch (RuntimeException e) {
log.error("Unexpected {} during delayed purge of context {}; expired entries were NOT removed",
e.getClass().getSimpleName(), contextURI, e);
} catch (Error e) {
// Not re-thrown: with submit() the Error is captured by FutureTask.run() and the
// discarded Future means rethrow has no observable effect. Logging is the only signal.
log.error("Fatal {} during delayed purge of context {}; expired entries were NOT removed",
e.getClass().getSimpleName(), contextURI, e);
}
});
}
/**
* Removes from the set the entries whose document is neither in the submission queue nor in flight. The monitor
* is held per chunk only, so that posting documents is not blocked for long by a large set; this is safe because
* a document only leaves the queue into flight, and only leaves flight by settling, by going back into the queue
* or by being dropped for a queued deletion, all under the monitor, so an entry seen in neither stays sent.
*/
private void removeSentDocuments(Set<URI> entryURIs) {
Iterator<URI> it = entryURIs.iterator();
while (it.hasNext()) {
synchronized (postQueue) {
for (int i = 0; i < SENT_CHECK_CHUNK_SIZE && it.hasNext(); i++) {
URI entryURI = it.next();
if (!postQueue.asMap().containsKey(entryURI) && !inFlightAdds.contains(entryURI)) {
it.remove();
}
}
}
}
}
private boolean hasPendingAdds() {
synchronized (postQueue) {
postQueue.cleanUp();
return postQueue.estimatedSize() > 0 || !inFlightAdds.isEmpty();
}
}
public boolean isIndexing() {
return !reindexing.isEmpty();
}
public boolean isIndexing(URI contextURI) {
return reindexing.containsKey(contextURI);
}
public Set<URI> getIndexingContexts() {
// Snapshot under the map's monitor: Collections.synchronizedMap synchronizes
// individual ops but iteration over a live keySet view is not thread-safe,
// and the reindex executor mutates this map from another thread.
synchronized (reindexing) {
return Set.copyOf(reindexing.keySet());
}
}
public long getPostQueueSize() {
postQueue.cleanUp();
return postQueue.estimatedSize();
}
/** Documents Solr has rejected since startup; a growing value means documents are silently missing from the index. */
public long getRejectedDocumentCount() {
return rejectedDocuments.get();
}
/**
* Search hits since startup that the read-access filter admitted but the per-hit access check denied. A short lag
* after an ACL change explains a few; a steadily growing value means the filter and the access check disagree, so
* counts and facets include entries their callers cannot read.
*/
public long getReadFilterMismatchCount() {
return readFilterMismatches.get();
}
public long getDeleteQueueSize() {
return deleteQueue.size();
}
/**
* Blocks until both queues are empty and no batch is in flight. Returns false only on thread
* interruption (typically JVM shutdown). Trade-off: a sustained Solr outage with a poison-pill
* batch re-queued by the {@code SolrServerException | IOException} retry path will block this
* call indefinitely; for EntryStore's single-instance ops model the operator's recourse is to
* kill the JVM and investigate.
*
* @return true if the submitter became quiescent, false if interrupted
*/
public boolean waitForQueueDrain() {
while (hasPendingAdds() || !deleteQueue.isEmpty() || submitterInFlight.get()) {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
}
return true;
}
@Override
public boolean ping() {
try {
SolrPingResponse pingResponse = this.solrServer.ping();
if (pingResponse.getStatus() == 0) {
return true;
}
} catch (SolrServerException | IOException e) {
log.error(e.getMessage());
}
return false;
}
@Override
public boolean isUp() {
return ping() && documentSubmitter.isAlive() && delayedContextIndexer.isAlive() && !reindexExecutor.isShutdown();
}
/**
* Queues the context's entries for a reindex after a changed guest ACL or project type of the context, both of
* which the entries' documents depend on. Every submission restarts the 10 second wait, so a burst of changes
* reindexes the context once. A change reverted within the wait still reindexes.
*/
public void submitContextForDelayedReindex(URI contextEntryURI) {
log.info("Enqueueing context {} for delayed reindexing", contextEntryURI);
delayedReindex.put(contextEntryURI, System.nanoTime());
}
/**
* Reindexes and dequeues every context whose latest submission is at least 10 seconds older than
* {@code nowNanos}, a {@link System#nanoTime()} reading; one pass of the {@link DelayedContextIndexer}. A failed
* reindex is logged and its context dequeued, so that it stops neither the other contexts nor the indexer thread.
*/
void reindexDue(long nowNanos) {
synchronized (delayedReindex) {
Iterator<Map.Entry<URI, Long>> it = delayedReindex.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<URI, Long> queued = it.next();
if (nowNanos - queued.getValue() >= TimeUnit.SECONDS.toNanos(10)) {
log.info("Submitting context {} for reindexing after 10 seconds delay", queued.getKey());
try {
reindex(queued.getKey());
} catch (RuntimeException e) {
log.error("Delayed reindex of context {} failed: {}", queued.getKey(), e.getMessage(), e);
}
it.remove();
}
}
}
}
/**
* Whether the context's URI index carries every entry, so a purge keyed on "not reposted by this
* reindex" is safe. Answers false rather than propagating if the context cannot be resolved at all,
* since a purge is not safe in that case either.
*/
private boolean indexIsComplete(URI contextURI) {
String id = URISplit.getLastSegment(contextURI.toString());
Context context = rm.getContextManager().getContext(id);
if (context == null) {
log.warn("Context {} could not be resolved; skipping the reindex purge", contextURI);
return false;
}
if (context.isIndexComplete()) {
return true;
}
log.error("Context {} has an incomplete URI index, so entries missing from its listing would be "
+ "deleted from the Solr index by the post-reindex purge. Reindexing without purging; "
+ "expired documents may remain until the underlying data is repaired", contextURI);
return false;
}
private enum ContextPostOutcome {
/** All entries of the context were processed; some of them may have failed. */
COMPLETED,
/** The context could not be resolved, so none of its entries were processed. */
UNRESOLVED,
/** The reindex was interrupted before all entries of the context were processed. */
INTERRUPTED
}
/**
* Result of posting the entries of a context to the submission queue. All entries that could not be posted
* count as not indexed. The documents of corrupt entries are removed from the index right away and those of
* unresolved entries by the purge of the context's expired documents, while unloadable and unindexable entries
* keep their previous documents and prevent that purge.
*
* @param outcome whether all entries of the context were processed
* @param queuedEntryURIs the entries whose documents were posted
* @param loadedEntries entries that were loaded, whether they were then posted, skipped as deleted or
* could not be indexed
* @param corruptEntries entries whose loading failed because of their data, see
* {@link #isCausedByCorruptData(Throwable)}
* @param unresolvedEntries entries that are listed in the context but were not found when loading them
* @param unloadableEntries entries whose loading failed for another reason, e.g. an error of the store
* @param unindexableEntries entries that were loaded but whose Solr document could not be constructed
*/
private record ContextPostResult(ContextPostOutcome outcome, Set<URI> queuedEntryURIs, int loadedEntries,
int corruptEntries, int unresolvedEntries, int unloadableEntries, int unindexableEntries) {
static ContextPostResult unresolved() {
return new ContextPostResult(ContextPostOutcome.UNRESOLVED, Set.of(), 0, 0, 0, 0, 0);
}
static ContextPostResult interrupted() {
return new ContextPostResult(ContextPostOutcome.INTERRUPTED, Set.of(), 0, 0, 0, 0, 0);
}
/**
* @return the entries that were processed, whether they could be loaded or not
*/
int listedEntries() {
return loadedEntries + corruptEntries + unresolvedEntries + unloadableEntries;
}
/**
* @return the entries that could not be indexed and whose documents are removed from the index
*/
int removedEntries() {
return corruptEntries + unresolvedEntries;
}
/**
* @return the entries that could not be indexed and keep their previous documents
*/
int keptEntries() {
return unloadableEntries + unindexableEntries;
}
int notIndexedEntries() {
return removedEntries() + keptEntries();
}
}
/**
* Posts the entries of a context to the submission queue. The documents of entries that cannot be loaded
* because of their data are removed from the index: they would otherwise keep matching queries although their
* entries cannot be returned, and a reindex restores them once the data is repaired. Entries that cannot be
* found are left to the purge of expired documents, which only removes documents indexed before the reindex
* started: an entry that is listed but not yet committed while it is being created cannot be found either, and
* removing its document right away could remove the document that its creation posts. Entries whose loading
* fails for another reason, e.g. a transient error of the store, keep their documents.
* <p>
* Loading runs as admin, so a failure caused by corrupt data is attributed by its cause chain alone: no
* principals are loaded, and the context entry was already loaded to list the entries. Building a document
* checks guest access, which loads all principals, so a corrupt principal makes every entry unindexable until it
* is repaired.
*/
private ContextPostResult postContextEntriesToQueue(URI contextURI) {
String id = URISplit.getLastSegment(contextURI.toString());
ContextManager cm = rm.getContextManager();
Context context = cm.getContext(id);
if (context == null) {
return ContextPostResult.unresolved();
}
Set<URI> queuedEntryURIs = new HashSet<>();
int loadedEntries = 0;
int corruptEntries = 0;
int unresolvedEntries = 0;
int unloadableEntries = 0;
int unindexableEntries = 0;
Set<String> tracedFailureCauses = new HashSet<>();
for (URI entryURI : context.getEntries()) {
if (Thread.currentThread().isInterrupted() || !awaitPostQueueCapacity()) {
return new ContextPostResult(ContextPostOutcome.INTERRUPTED, queuedEntryURIs, loadedEntries,
corruptEntries, unresolvedEntries, unloadableEntries, unindexableEntries);
}
if (entryURI == null) {
continue;
}
Entry entry;
try {
entry = cm.getEntry(entryURI);
} catch (Exception | StackOverflowError e) {
if (isCausedByCorruptData(e)) {
corruptEntries++;
logEntryFailure("Unable to load corrupt entry", entryURI, e, tracedFailureCauses);
removeCorruptEntryDocument(entryURI);
} else {
unloadableEntries++;
logEntryFailure("Unable to load entry", entryURI, e, tracedFailureCauses);
}
continue;
}
if (entry == null) {
unresolvedEntries++;
log.warn("Unable to load entry with URI {}", entryURI);
continue;
}
loadedEntries++;
synchronized (postQueue) {
try {
// Checked while holding the monitor that the removal of a deleted entry's document takes, so
// that a concurrent deletion is either seen here or removes the document queued here
if (entry.isDeleted() || entry.getContext().isDeleted()) {
log.debug("Not adding deleted entry to post queue: {}", entryURI);
continue;
}
log.info("Adding entry to Solr post queue: {}", entryURI);
queueDocument(entryURI, constructSolrInputDocument(entry, extractFulltext));
queuedEntryURIs.add(entryURI);
} catch (Exception | StackOverflowError e) {
unindexableEntries++;
logEntryFailure("Not indexing entry", entryURI, e, tracedFailureCauses);
}
}
}
ContextPostResult posted = new ContextPostResult(ContextPostOutcome.COMPLETED, queuedEntryURIs,
loadedEntries, corruptEntries, unresolvedEntries, unloadableEntries, unindexableEntries);
if (posted.notIndexedEntries() > 0) {
log.error("{} entries of context {} could not be indexed: {} are corrupt and {} could not be found, their"
+ " documents are removed from the index unless the purge is prevented; {} could not be loaded"
+ " and {} could not be indexed, they keep the documents they had in the index, if any",
posted.notIndexedEntries(),
contextURI, corruptEntries, unresolvedEntries, unloadableEntries, unindexableEntries);
}
return posted;
}
/**
* Waits while the post queue holds {@link #maxReindexQueueSize} documents or more. The caller must not hold the
* queue's monitor, which the submitter takes to drain it. Entry writes do not wait, so the queue may exceed the
* bound by the documents they queue.
*
* @return false if the index is shutting down, or if the thread was interrupted or the submitter has stopped
* while waiting, since the queue would then not drain
*/
private boolean awaitPostQueueCapacity() {
while (!shuttingDown && getPostQueueSize() >= maxReindexQueueSize) {
if (!documentSubmitter.isAlive()) {
log.error("Stopping the reindex because the Solr document submitter has stopped and would not drain"
+ " the post queue; a restart is required to resume indexing");
return false;
}
try {
Thread.sleep(REINDEX_QUEUE_POLL_MS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
}
return !shuttingDown;
}
/**
* Removes the document of an entry that failed to load because of its data, unless a document of the entry is
* queued: then the entry loaded after it failed to load here, e.g. because its data was repaired in between,
* and the queued document must not be dropped. A queued document from before the data became corrupt is kept
* as well, which is harmless because a search skips hits whose entries are corrupt.
*/
private void removeCorruptEntryDocument(URI entryURI) {
synchronized (postQueue) {
if (postQueue.getIfPresent(entryURI) != null) {
log.info("Not removing the document of corrupt entry {} because a document of it is queued", entryURI);
return;
}
removeEntryDocument(entryURI);
}
}
/**
* Tells whether loading an entry failed because of its data, so that it fails the same way until the data is
* repaired: the data is recognizably corrupt, or references in it form a cycle and cause unbounded recursion.
* Other failures, e.g. of the store, may be transient.
*/
private static boolean isCausedByCorruptData(Throwable e) {
return ExceptionUtils.indexOfType(e, CorruptEntryException.class) >= 0
|| ExceptionUtils.indexOfType(e, StackOverflowError.class) >= 0;
}
/**
* Logs an entry failure with its stack trace if it is the first failure of this kind (see
* {@link #failureKind(String, Throwable)}) in the current context reindex and the trace budget is not exhausted,
* otherwise as one line.
*/
private void logEntryFailure(String message, URI entryURI, Throwable e, Set<String> tracedFailureCauses) {
if (tracedFailureCauses.size() < MAX_ENTRY_FAILURE_TRACES
&& tracedFailureCauses.add(failureKind(message, e))) {
log.error("{} {}", message, entryURI, e);
} else {
log.error("{} {}: {}", message, entryURI, ExceptionUtils.getRootCauseMessage(e));
}
}
/**
* @return the kind of an entry failure: the log message and the classes of the exception and its causes. Not
* the exception messages, which contain the URI of the failing entry, so that the same failure of many entries
* is one kind.
*/
static String failureKind(String message, Throwable e) {
return message + ": " + ExceptionUtils.getThrowableList(e).stream()
.map(t -> t.getClass().getName())
.collect(Collectors.joining(" <- "));
}
private void storeLiteralsWithLanguages(SolrInputDocument doc, Map<String, Set<String>> literals, String literalType) {
final String missingLanguageString = "nolang";
final String defaultString = "default";
Set<String> alreadySetLanguages = new HashSet<>();
for (String literal : literals.keySet()) {
doc.addField(literalType, literal);
// we also store title.{lang} as dynamic field to be able to
// sort after titles in a specific language
Set<String> literalLanguages = literals.get(literal);
if (literalLanguages.isEmpty()) {
if (!alreadySetLanguages.contains(missingLanguageString)) {
doc.addField(String.format("%s.%s", literalType, missingLanguageString), literal);
alreadySetLanguages.add(missingLanguageString);
}
} else {
for (String language : literalLanguages) {
if (language != null && language.equalsIgnoreCase(defaultSortLang) && !alreadySetLanguages.contains(defaultString)) {
// if a default sorting language is configured, we create a field for that
doc.addField(String.format("%s.%s", literalType, defaultString), literal);
alreadySetLanguages.add(defaultString);
}
// we only want one title per language, otherwise sorting will not work
if (!alreadySetLanguages.contains(language)) {
doc.addField(String.format("%s.%s", literalType, language == null ? missingLanguageString : language), literal);
alreadySetLanguages.add(language);
}
}
}
}
}
public SolrInputDocument constructSolrInputDocument(Entry entry, boolean extractFulltext) {
Model mdGraph = entry.getMetadataGraph();
Model entryGraph = entry.getGraph();
URI resourceURI = entry.getResourceURI();
SolrInputDocument doc = new SolrInputDocument();
// URI
doc.setField("uri", entry.getEntryURI().toString());
// resource URI
doc.setField("resource", resourceURI.toString());
// resource URI of the surrounding context
doc.setField("context", entry.getContext().getEntry().getResourceURI().toString());
// RDF type
for (Statement value : entryGraph.filter(valueFactory.createIRI(resourceURI.toString()), RDF.TYPE, null)) {
doc.addField("rdfType", value.getObject().stringValue());
}
for (Statement statement : mdGraph.filter(valueFactory.createIRI(resourceURI.toString()), RDF.TYPE, null)) {
doc.addField("rdfType", statement.getObject().stringValue());
}
// creation date
Date creationDate = entry.getCreationDate();
if (creationDate != null) {
doc.setField("created", creationDate);
}
// modification date
Date modificationDate = entry.getModifiedDate();
if (modificationDate != null) {
doc.setField("modified", modificationDate);
}
// types
doc.setField("graphType", entry.getGraphType().name());
doc.setField("entryType", entry.getEntryType().name());
doc.setField("resourceType", entry.getResourceType().name());
// profile
if (entry.getLocalMetadataURI() != null) {
Set<IRI> profilePreds = new HashSet<>();
profilePreds.add(valueFactory.createIRI("http://entryscape.com/terms/entityType"));
profilePreds.add(valueFactory.createIRI("http://entrystore.org/terms/profile"));
for (String profileURI : EntryUtil.getResourceValues(entryGraph, entry.getLocalMetadataURI(), profilePreds)) {
doc.setField("profile", profileURI);
break; // we only need the first match
}
} else {
log.warn("Local metadata URI of entry is null: {}", entry.getEntryURI());
}
// project type
// FIXME this can be optimized by not checking the context for every single indexed entry, perhaps we can
// remember the context status for a limited amount of time instead of fetching and querying the context graph
// every time
Model graphWithProjectType;
URI resourceUriForProjectType;
if (GraphType.Context.equals(entry.getGraphType())) {
graphWithProjectType = entryGraph;
resourceUriForProjectType = resourceURI;
} else {
Entry contextEntry = entry.getContext().getEntry();
graphWithProjectType = contextEntry.getGraph();
resourceUriForProjectType = contextEntry.getResourceURI();
}
for (String projectTypeURI : EntryUtil.getResourceValues(
graphWithProjectType,
resourceUriForProjectType,
Collections.singleton(valueFactory.createIRI("http://entryscape.com/terms/projectType")))) {
doc.setField("projectType", projectTypeURI);
break; // we only need the first match
}
// creator
URI creator = entry.getCreator();
if (creator != null) {
doc.setField("creator", creator.toString());
}
// contributors
for (URI c : entry.getContributors()) {
doc.addField("contributors", c.toString());
}
// lists
for (URI l : entry.getReferringListsInSameContext()) {
doc.addField("lists", l.toString());
}
// ACL: admin, metadata r/w, resource r/w
try {
for (URI p : entry.getAllowedPrincipalsFor(AccessProperty.Administer)) {
doc.addField("acl.admin", p.toString());
}
for (URI p : entry.getAllowedPrincipalsFor(AccessProperty.ReadMetadata)) {
doc.addField("acl.metadata.r", p.toString());
}
for (URI p : entry.getAllowedPrincipalsFor(AccessProperty.WriteMetadata)) {
doc.addField("acl.metadata.rw", p.toString());
}
for (URI p : entry.getAllowedPrincipalsFor(AccessProperty.ReadResource)) {
doc.addField("acl.resource.r", p.toString());
}
for (URI p : entry.getAllowedPrincipalsFor(AccessProperty.WriteResource)) {
doc.addField("acl.resource.rw", p.toString());
}
} catch (IllegalArgumentException iae) {
log.warn("Unable to index ACL for entry {}: {}", entry.getEntryURI().toString(), iae.getMessage());
}
doc.setField(FIELD_ACL_EXPLICIT, entry.hasAllowedPrincipals());
if ((entry.getEntryType() == EntryType.Reference || entry.getEntryType() == EntryType.LinkReference)
&& entry.getCachedExternalMetadata() instanceof LocalMetadataWrapper wrapper
&& wrapper.getReferencedEntryURI() instanceof URI referencedEntryURI) {
doc.setField(FIELD_REFERENCED_ENTRY, referencedEntryURI.toString());
}
// status
URI status = entry.getStatus();
if (status != null) {
doc.setField("status", status.toString());
}
// titles
Map<String, Set<String>> titles = EntryUtil.getTitles(entry);
if (titles != null && !titles.isEmpty()) {
storeLiteralsWithLanguages(doc, titles, "title");
}
String firstName = EntryUtil.getFirstName(entry);
String lastName = EntryUtil.getLastName(entry);
String name = "";
if (firstName != null) {
name += firstName;
}
if (lastName != null) {
name += " " + lastName;
}
if (!name.isEmpty()) {
doc.addField("title", name);
}
// user name
if (GraphType.User.equals(entry.getGraphType())) {
User user = (org.entrystore.User) entry.getResource();
if (user != null) {
String username = user.getName();
if (username != null) {
doc.addField("username", username);
}
} else {
log.warn("User resource of {} is null", entry.getEntryURI().toString());
}
}
// context name
if (GraphType.Context.equals(entry.getGraphType()) || GraphType.SystemContext.equals(entry.getGraphType())) {
String contextName = rm.getContextManager().getName(entry.getResource().getURI());
if (contextName != null) {
doc.addField("contextname", contextName);
}
}
// description
Map<String, Set<String>> descriptions = EntryUtil.getDescriptions(entry);
if (descriptions != null && !descriptions.isEmpty()) {
storeLiteralsWithLanguages(doc, descriptions, "description");
}
// tag.literal[.*]
Map<String, Set<String>> tagLiterals = EntryUtil.getTagLiterals(entry);
if (tagLiterals != null) {
storeLiteralsWithLanguages(doc, tagLiterals, "tag.literal");
}
// tag.uri
for (String s : EntryUtil.getTagResources(entry)) {
doc.addField("tag.uri", s);
}
// language of the resource
String dcLang = EntryUtil.getLabel(mdGraph, resourceURI, valueFactory.createIRI(NS.dc + "language"), null);
if (dcLang != null) {
doc.addField("lang", dcLang);
}
String dctLang = EntryUtil.getLabel(mdGraph, resourceURI, valueFactory.createIRI(NS.dcterms + "language"), null);
if (dctLang != null) {
doc.addField("lang", dctLang);
}
// email (foaf:mbox)
String email = EntryUtil.getEmail(entry);
if (email != null) {
doc.addField("email", email);
}
// publicly viewable metadata?
boolean guestReadable = false;
PrincipalManager pm = entry.getRepositoryManager().getPrincipalManager();
URI currentUser = pm.getAuthenticatedUserURI();
try {
pm.setAuthenticatedUserURI(pm.getGuestUser().getURI());
try {
pm.checkAuthenticatedUserAuthorized(entry, AccessProperty.ReadMetadata);
guestReadable = true;
} catch (AuthorizationException ignored) {
} catch (IllegalArgumentException iae) {
log.warn(iae.getMessage());
}
} finally {
pm.setAuthenticatedUserURI(currentUser);
}
doc.setField("public", guestReadable);
addGenericMetadataFields(doc, mdGraph, false);
if (related) {
addRelatedFields(doc, entry);
}
// Full text extraction using Apache Tika
/* if (extractFulltext && EntryType.Local.equals(entry.getEntryType())
&& ResourceType.InformationResource.equals(entry.getResourceType())
&& entry.getResource() instanceof Data) {
Data d = (Data) entry.getResource();
File f = d.getDataFile();
if (f != null && f.exists()) {
String textContent = extractFulltext(f);
if (textContent != null) {
doc.addField("fulltext", textContent);
}
}
} */
return doc;
}
/**
* Indexes the IRI and literal objects of the graph (blank-node objects are skipped) into the dynamic
* {@code metadata.predicate.*} fields, {@code related.}-prefixed for the related graph, plus the flat
* {@code metadata.object.*} fields for the primary graph. Package-private so the literal field layout can be
* tested without a Solr server.
*/
void addGenericMetadataFields(SolrInputDocument doc, Model metadata, boolean related) {
if (doc == null || metadata == null) {
throw new IllegalArgumentException("Neither SolrInputDocument nor Graph must be null");
}
String prefix = "";
if (related) {
prefix = "related.";
}
// All subject, predicates and objects in the metadata graph
//
// We also provide an index for all predicate-object tuples, stored in dynamic fields.
for (Statement s : metadata) {
// predicate
String predString = s.getPredicate().stringValue();
String predMD5Trunc8 = Hashing.hash(predString, HashType.MD5).substring(0, 8);
// object
if (s.getObject() instanceof IRI) {
String objString = s.getObject().stringValue();
if (!related) {
addFieldValueOnce(doc,prefix + "metadata.object.uri", objString);
}
// predicate value is included in the parameter name, the object value is the field value
addFieldValueOnce(doc,prefix + "metadata.predicate.uri." + predMD5Trunc8, objString);
} else if (s.getObject() instanceof Literal l) {
if (!related) {
if (MetadataUtil.isStringLiteral(l)) { // we only index plain literals (human-readable text)
addFieldValueOnce(doc,prefix + "metadata.object.literal", l.getLabel());
}
}
// predicate value is included in the parameter name, the object value is the field value
addFieldValueOnce(doc,prefix + "metadata.predicate.literal_s." + predMD5Trunc8, l.getLabel());
if (!LangFacetValue.exceedsLabelCap(l.getLabel())) {
addFieldValueOnce(doc, prefix + LangFacetValue.FIELD_PREFIX + predMD5Trunc8, LangFacetValue.encode(l));
}
// special handling of integer values, to be used for e.g., sorting
if (MetadataUtil.isIntegerLiteral(l)) {
try {
// it's a single-value field, so we call setField instead of addField just in case there should be
doc.setField(prefix + "metadata.predicate.integer." + predMD5Trunc8, l.longValue());
} catch (NumberFormatException nfe) {
log.warn("Unable to index integer literal: {}. (Subject: {}, Predicate: {}, Object: {})", nfe.getMessage(), s.getSubject(), predString, l.getLabel());
}
}
if (MetadataUtil.isDateLiteral(l)) {
try {
doc.setField(prefix + "metadata.predicate.date." + predMD5Trunc8, dateToSolrDateString(l.calendarValue()));
} catch (IllegalArgumentException iae) {
log.warn("Unable to index date literal: {}. (Subject: {}, Predicate: {}, Object: {})", iae.getMessage(), s.getSubject(), predString, l.getLabel());
}
}
// special handling of decimal/float/double values, to be used for e.g., numeric range queries and sorting
if (MetadataUtil.isDecimalLiteral(l)) {
try {
doc.setField(prefix + "metadata.predicate.decimal." + predMD5Trunc8, l.doubleValue());
} catch (NumberFormatException nfe) {
log.warn("Unable to index decimal literal: {}. (Subject: {}, Predicate: {}, Object: {})", nfe.getMessage(), s.getSubject(), predString, l.getLabel());
}
}
}
}
}
private void addRelatedFields(SolrInputDocument doc, Entry entry) {
if (doc == null || entry == null) {
throw new IllegalArgumentException("Neither SolrInputDocument nor Entry must be null");
}
Set<Context> contexts = new HashSet<>();
if (relatedContainsGlobal) {
// TODO there is room for optimization, the contexts are loaded now once per indexed entry.
// possible solution: provide getter method and load only if set is null; set to null when there are
// relevant changes, e.g. context removed or added (ContextManager needs to tell SolrSearchIndex that
// contexts have changed)
ContextManager cm = entry.getRepositoryManager().getContextManager();
for (URI contextURI : cm.getEntries()) {
Context c = cm.getContext(contextURI);
if (c instanceof RegularContext) {
contexts.add(c);
}
}
}
Set<Entry> relatedEntries = new HashSet<>();
for (IRI relProp : relatedProperties.keySet()) {
List<String> relatedURIs = EntryUtil.getResourceValues(entry, Collections.singleton(relProp));
if (relatedURIs.isEmpty()) {
continue;
}
if (relatedContainsGlobal && relatedProperties.get(relProp)) {
for (Context context : contexts) {
for (String relEntURI : relatedURIs) {
relatedEntries.addAll(context.getByResourceURI(URI.create(relEntURI)));
}
}
} else {
for (String relEntURI : relatedURIs) {
relatedEntries.addAll(entry.getContext().getByResourceURI(URI.create(relEntURI)));
}
}
}
if (!relatedEntries.isEmpty()) {
Set<URI> mainEntryACL = entry.getAllowedPrincipalsFor(AccessProperty.ReadMetadata);
for (Entry relE : relatedEntries) {
if (mainEntryACL.equals(relE.getAllowedPrincipalsFor(AccessProperty.ReadMetadata))) {
log.debug("Adding " + relE.getEntryURI() + " to related property index of " + entry.getEntryURI());
addGenericMetadataFields(doc, relE.getMetadataGraph(), true);
} else {
log.debug("ACLs of " + entry.getEntryURI() + " and " + relE.getEntryURI() + " do not match, not adding to related property index");
}
}
}
}
private void addFieldValueOnce(SolrInputDocument doc, String name, Object value) {
Collection<Object> fieldValues = doc.getFieldValues(name);
if (fieldValues == null || !fieldValues.contains(value)) {
doc.addField(name, value);
}
}
private String dateToSolrDateString(XMLGregorianCalendar c) {
if (c.getTimezone() == DatatypeConstants.FIELD_UNDEFINED) {
c.setTimezone(0);
}
return SOLR_DATE_FORMATTER.format(c.toGregorianCalendar().toInstant());
}
public void postEntry(Entry entry) {
PrincipalManager pm = entry.getRepositoryManager().getPrincipalManager();
URI currentUser = pm.getAuthenticatedUserURI();
try {
pm.setAuthenticatedUserURI(pm.getAdminUser().getURI());
URI entryURI = entry.getEntryURI();
synchronized (postQueue) {
if (postQueue.getIfPresent(entryURI) != null) {
log.debug("Entry {} already exists in post queue, attempting replacement", entryURI);
}
if (!entry.isDeleted() && !entry.getContext().isDeleted()) {
log.info("Adding document to Solr post queue: {}", entryURI);
try {
queueDocument(entryURI, constructSolrInputDocument(entry, extractFulltext));
} catch (Exception e) {
log.error("Not indexing {}", entryURI, e);
}
} else {
log.debug("Not adding deleted entry to post queue: {}", entryURI);
}
}
} finally {
pm.setAuthenticatedUserURI(currentUser);
}
}
public void removeEntry(Entry entry) {
removeEntryDocument(entry.getEntryURI());
// if entry is a context, also remove all entries inside
if (GraphType.Context.equals(entry.getGraphType())) {
if (!removeContext(entry.getResourceURI())) {
log.warn("Context-removal purge for context {} failed; expired Solr documents may remain", entry.getEntryURI());
}
}
}
/**
* Queues the document of an entry for submission and drops a pending deletion of the entry, so that the
* deletion cannot remove the newer document: the submitter does not necessarily send all queued deletions
* before the queued documents. The caller must hold the submission queue's monitor, see
* {@link #removeEntryDocument(URI)}.
*/
private void queueDocument(URI entryURI, SolrInputDocument document) {
synchronized (deleteQueue) {
deleteQueue.remove(entryURI);
}
postQueue.put(entryURI, document);
}
/**
* Removes the document of an entry from the index and drops a document of the entry that is still waiting in
* the submission queue, so that it is not added again after the deletion. Both happen while holding the
* submission queue's monitor, as does {@link #queueDocument(URI, SolrInputDocument)}, so that of a deletion
* and a concurrently queued document of the same entry the later one wins.
*/
private void removeEntryDocument(URI entryURI) {
synchronized (postQueue) {
postQueue.invalidate(entryURI);
synchronized (deleteQueue) {
log.info("Adding entry to Solr delete queue: {}", entryURI);
deleteQueue.add(entryURI);
}
}
}
/**
* Exact counts for candidate labels of several facet fields, in one bounded {@code rows=0} request restricted by
* the caller's {@link SolrReadAccessFilter}, as {@link #sendQuery} restricts the search the candidates come from:
* one {@code facet.query} per (field, label), at most {@link #MAX_COUNT_CLAUSES} in total, shared
* round-robin across the fields so each keeps the head of its ranked candidate list. The field query parser takes
* the label literally, {@code cache=false} keeps the clauses out of the filterCache that every other search
* depends on, and macro expansion is switched off because labels are user-written text in which {@code ${…}}
* would otherwise be expanded. The request goes out as POST because labels can be long. Labels the field does not
* hold are absent from the result rather than present with zero.
*
* @param baseQuery the query whose result set the counts apply to; only its query and client filters are reused
* @param candidatesByField per client-facing facet field, its candidate labels in ranked order
* @return per field, the counted labels; a field whose candidates were all cut by the budget maps to an empty map
* @throws SolrException when Solr rejects the request, or with {@code SERVICE_UNAVAILABLE} when it cannot be
* reached; a failure is never reported as an empty facet
*/
public Map<String, Map<String, Long>> facetCountsForLabels(SolrQuery baseQuery,
Map<String, List<String>> candidatesByField) {
Map<String, Map<String, Long>> counts = new LinkedHashMap<>();
candidatesByField.keySet().forEach(field -> counts.put(field, new LinkedHashMap<>()));
Map<String, Map.Entry<String, String>> fieldAndLabelByQuery = new LinkedHashMap<>();
List<Iterator<String>> cursors = new ArrayList<>();
List<String> cursorFields = new ArrayList<>();
candidatesByField.forEach((field, labels) -> {
cursors.add(labels.iterator());
cursorFields.add(field);
});
boolean progressed = true;
while (progressed && fieldAndLabelByQuery.size() < MAX_COUNT_CLAUSES) {
progressed = false;
for (int i = 0; i < cursors.size() && fieldAndLabelByQuery.size() < MAX_COUNT_CLAUSES; i++) {
if (cursors.get(i).hasNext()) {
String field = cursorFields.get(i);
String label = cursors.get(i).next();
fieldAndLabelByQuery.put("{!field cache=false f=" + field + "}" + label, Map.entry(field, label));
progressed = true;
}
}
}
if (fieldAndLabelByQuery.isEmpty()) {
return counts;
}
SolrQuery countQuery = facetOnlyQuery(baseQuery.getQuery(), baseQuery.getFilterQueries());
fieldAndLabelByQuery.keySet().forEach(countQuery::addFacetQuery);
SolrReadAccessFilter.forAuthenticatedUser(rm).ifPresent(filter -> filter.applyTo(countQuery));
QueryResponse response;
try {
response = solrServer.query(countQuery, SolrRequest.METHOD.POST);
} catch (SolrServerException | IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Facet counts could not be fetched from Solr", e);
}
Map<String, Integer> facetQueryCounts = response.getFacetQuery();
if (facetQueryCounts != null) {
facetQueryCounts.forEach((facetQuery, count) -> {
Map.Entry<String, String> fieldAndLabel = fieldAndLabelByQuery.get(facetQuery);
if (fieldAndLabel != null && count != null && count > 0) {
counts.get(fieldAndLabel.getKey()).put(fieldAndLabel.getValue(), count.longValue());
}
});
}
return counts;
}
/**
* Whether {@code field} holds at least one term anywhere in the index, as a bounded {@code rows=0} facet with
* {@code facet.limit=1} over {@code *:*}. Callers read a negative answer as an index built before the field
* existed; a positive answer cannot become false again, so it is cached and the probe runs once per field. Not
* restricted by read access: the answer says whether the index has the field, not what any entry holds.
*
* @throws SolrException when Solr rejects the request, or with {@code SERVICE_UNAVAILABLE} when it cannot be
* reached; a failed probe is not reported as either answer
*/
public boolean hasFacetTerms(String field) {
if (fieldsKnownToHaveTerms.contains(field)) {
return true;
}
SolrQuery probe = facetOnlyQuery("*:*", null);
probe.addFacetField(field);
probe.setFacetLimit(1);
probe.setFacetMinCount(1);
probe.setFacetMissing(false);
FacetField facetField;
try {
facetField = solrServer.query(probe).getFacetField(field);
} catch (SolrServerException | IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Facet field could not be probed in Solr", e);
}
boolean hasTerms = facetField != null && facetField.getValueCount() > 0;
if (hasTerms) {
fieldsKnownToHaveTerms.add(field);
}
return hasTerms;
}
/** A {@code rows=0} faceting request over the given query and filters, the shared shape of the facet side requests. */
private static SolrQuery facetOnlyQuery(String query, String[] filters) {
SolrQuery facetQuery = new SolrQuery(query);
facetQuery.set("expandMacros", "false");
if (filters != null) {
facetQuery.setFilterQueries(filters);
}
facetQuery.setRows(0);
facetQuery.setFacet(true);
return facetQuery;
}
private long sendQueryForEntryURIs(SolrQuery query, Set<URI> result, List<FacetField> facetFields, SolrClient solrServer, int offset) {
if (query == null) {
throw new IllegalArgumentException("Query object must not be null");
}
if (offset > -1) {
query.setStart(offset);
}
// We only need the "uri" field in the response,
// so we skip the rest (default is "*")
query.setFields("uri");
// the REST layer validates client query text literally, so a ${…} macro must not rewrite it in Solr
query.set("expandMacros", "false");
long hits = -1;
QueryResponse r;
try {
// POST: the read-access filter carries the caller's principals, which can exceed a URL's length limit
r = solrServer.query(query, SolrRequest.METHOD.POST);
r.getElapsedTime();
// Facets are identical on every result-fill pass of sendQuery; taking them once keeps each field single.
if (r.getFacetFields() != null && facetFields.isEmpty()) {
facetFields.addAll(r.getFacetFields());
}
SolrDocumentList docs = r.getResults();
hits = docs.getNumFound();
for (SolrDocument solrDocument : docs) {
if (solrDocument.containsKey("uri")) {
String uri = (String) solrDocument.getFieldValue("uri");
if (uri != null) {
result.add(URI.create(uri));
}
}
}
log.debug("Query time: {} ms, elapsed time: {} ms", r.getQTime(), r.getElapsedTime());
} catch (SolrServerException | IOException e) {
if (e instanceof SolrServerException && ((SolrServerException) e).getRootCause() instanceof IllegalArgumentException) {
log.info(e.getMessage());
} else {
log.error(e.getMessage());
}
}
return hits;
}
/**
* Runs {@code clientQuery} restricted by the caller's {@link SolrReadAccessFilter}, so that rows, the hit count
* and facets cover only entries the caller may read. Every returned hit is still checked against the repository,
* which drops a hit an index lagging behind an ACL change admitted. {@code clientQuery} itself is left unchanged.
*/
public QueryResult sendQuery(SolrQuery clientQuery) throws SolrException {
Optional<SolrReadAccessFilter> readFilter = SolrReadAccessFilter.forAuthenticatedUser(rm);
SolrQuery query = clientQuery.getCopy();
readFilter.ifPresent(filter -> filter.applyTo(query));
Set<Entry> result = new LinkedHashSet<>();
long hits = -1;
long inaccessibleHits = 0;
int limit = query.getRows();
int offset = query.getStart();
List<FacetField> facetFields = new ArrayList<>();
query.setIncludeScore(true);
int resultFillIteration = 0;
do {
if (resultFillIteration++ > 0) {
// We have a small limit and we don't get enough results with permissive ACL per iteration,
// so we need to increase the result size windows, but not the result limit itself
// (i.e., we only change the rows towards Solr, but not the query limit of EntryStore.
// We only need to do this once when resultFillIteration equals 1.
if (resultFillIteration == 1 && limit <= 10) {
query.setRows(100);
}
if (resultFillIteration > 10) {
log.warn("Breaking after 10 result fill interations to prevent too many loops");
break;
}
offset += Math.min(limit, 50);
log.warn("Increasing offset to " + offset + " in an attempt to fill the result limit");
}
Set<URI> entryURIs = new LinkedHashSet<>();
hits = sendQueryForEntryURIs(query, entryURIs, facetFields, solrServer, offset);
// Facets depend on the query, not on the page, so every further fill pass would recompute the same
// buckets and throw them away. Switching faceting off makes the extra passes cheaper for Solr.
query.setFacet(false);
Date before = new Date();
for (URI uri : entryURIs) {
URI referencedEntryURI = null;
try {
Entry entry = rm.getContextManager().getEntry(uri);
if (entry != null) {
if (entry.isDeleted()) {
log.warn("Deleted entry {} is still in Solr index, removing it now", uri);
removeEntry(entry);
throw new IllegalStateException("Cannot return deleted entry in search result: " + uri);
}
if ((entry.getEntryType() == EntryType.Reference || entry.getEntryType() == EntryType.LinkReference)
&& entry.getCachedExternalMetadata() instanceof LocalMetadataWrapper wrapper) {
referencedEntryURI = wrapper.getReferencedEntryURI();
}
ContextManagerImpl.checkEntryMetadataReadable(entry);
result.add(entry);
if (result.size() == limit) {
// we have enough results
break;
}
}
} catch (AuthorizationException | IllegalStateException e) {
inaccessibleHits++;
if (e instanceof AuthorizationException && readFilter.isPresent()) {
readFilterMismatches.incrementAndGet();
log.debug("Search hit {} passed the read-access filter but the access check denied it", uri);
}
} catch (RuntimeException | StackOverflowError e) {
// Only corrupt data of the hit, its referenced entry or their context entries skips the hit; other
// failures, e.g. of the store or a corrupt principal in an access check, must fail the search
if (ExceptionUtils.indexOfType(e, StackOverflowError.class) < 0
&& !CorruptData.isCorruptDataOf(e, uri, rm.getRepositoryURL())
&& !CorruptData.isCorruptDataOf(e, referencedEntryURI, rm.getRepositoryURL())) {
throw e;
}
log.warn("Skipping search hit {} because its entry or its referenced entry is corrupt: {}", uri,
CorruptData.describe(e));
inaccessibleHits++;
}
}
log.info("Entry fetching took " + (new Date().getTime() - before.getTime()) + " ms");
} while ((limit > result.size()) && (hits > (offset + limit)));
long adjustedHitCount = hits - inaccessibleHits;
// We prevent possible information leakage (i.e., "Can we get to know whether a resource
// with a certain name exists even though we are not allowed to access it?") by manually
// setting the hit count to zero in certain conditions. Should protect against malicious
// probing requests.
//
// Test if the condition covers too much and add "offset == 0 &&" if necessary
if (result.isEmpty() && hits > 0) {
adjustedHitCount = 0;
}
if (adjustedHitCount < 0) {
log.warn("Adjusted hit count is negative, this should not happen");
// TODO perhaps we should just set it to a high number in order not to break clients, e.g. Integer.MAX_VALUE
}
return new QueryResult(result, adjustedHitCount, facetFields);
}
public SolrDocument fetchDocument(String uri) {
try {
SolrQuery q = new SolrQuery("uri:" + ClientUtils.escapeQueryChars(uri));
q.setStart(0);
q.setRows(1);
QueryResponse r = solrServer.query(q);
SolrDocumentList docs = r.getResults();
if (!docs.isEmpty()) {
return docs.getFirst();
}
} catch (SolrServerException | IOException e) {
log.error(e.getMessage());
}
return null;
}
public static String extractFulltext(File f) {
return null;
// FIXME this method works but is deactivated. the needed apache tika
// framework has many dependencies so it is not activated in the pom.xml
// of scam-core for now. please activate it there before enabling the
// following code again.
/*
* InputStream stream = null; String textContent = null; String mimeType
* = null; try { TikaConfig tc = TikaConfig.getDefaultConfig();
* InputStream mimeIS = null; try { mimeIS = Files.newInputStream(f.toPath());
* mimeType = tc.getMimeRepository().getMimeType(mimeIS).getName(); }
* finally { if (mimeIS != null) { mimeIS.close(); } }
*
* if (mimeType != null) { stream = new BufferedInputStream(
* Files.newInputStream(f.toPath())); Parser parser = tc.getParser(mimeType); if
* (parser != null) { ContentHandler handler = new BodyContentHandler();
* try { log.info("Parsing document with MIME type " + mimeType + ": " +
* f.toString()); parser.parse(stream, handler, new Metadata(), new
* ParseContext()); textContent = handler.toString(); } catch (Exception
* e) { log.error("Unable to parse document: " + e.getMessage()); } }
* else { log.warn("Unable to detect parser for MIME type " + mimeType);
* } } else { log.warn("Unable to detect the MIME type"); } } catch
* (IOException e) { log.error(e.getMessage()); } finally { try { if
* (stream != null) { stream.close(); } } catch (IOException e) {
* log.error(e.getMessage()); } } return textContent;
*/
}
}