private JArray GetJsonDocuments(Guid etag)
{
JArray jsonDocuments = null;
try
{
var instanceId = docDb.TransactionalStorage.Id.ToString();
docDb.TransactionalStorage.Batch(actions =>
{
var docRetr = new DocumentRetriever(actions, Enumerable.Empty<AbstractReadTrigger>());
jsonDocuments = new JArray(actions.Documents.GetDocumentsAfter(etag)
.Where(x => x.Key.StartsWith("Raven/") == false) // don't replicate system docs
.Where(x => x.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) == instanceId) // only replicate documents created on this instance
.Where(x=> x.Metadata[ReplicationConstants.RavenReplicationConflict] == null) // don't replicate conflicted documents, that just propgate the conflict
.Select(x=>
{
docRetr.EnsureIdInMetadata(x);
return x;
})
.Take(100)
.Select(x => x.ToJson()));
});
}
catch (Exception e)
{
log.Warn("Could not get documents to replicate after: " + etag, e);
}
return jsonDocuments;
}
public bool IndexDocuments(IStorageActionsAccessor actions, string index, Guid etagToIndexFrom)
{
log.DebugFormat("Indexing documents for {0}, etag to index from: {1}", index, etagToIndexFrom);
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
return false; // index was deleted, probably
var jsonDocs = actions.Documents.GetDocumentsAfter(etagToIndexFrom)
.Where(x => x != null)
.Take(context.Configuration.MaxNumberOfItemsToIndexInSingleBatch) // ensure that we won't go overboard with reading and blow up with OOM
.ToArray();
if(jsonDocs.Length == 0)
return false;
var dateTime = jsonDocs.Select(x=>x.LastModified).Min();
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever
.EnsureIdInMetadata(doc)
.ProcessReadVetoes(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
return true;
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
return true;
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
return false;
}
finally
{
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
var last = jsonDocs.Last();
actions.Indexing.UpdateLastIndexed(index, last.Etag, last.LastModified);
}
}
public RemoteQueryResults Query(LinearQuery query)
{
var viewGenerator = queryCache.GetOrAdd(query.Query,
s =>
new DynamicViewCompiler("query", new IndexDefinition { Map = query.Query, },
new AbstractDynamicCompilationExtension[0])
{
RequiresSelectNewAnonymousType = false
}.GenerateInstance());
var results = new List<string>();
var errors = new List<string>();
int lastResult = 0;
int finalResult = 0;
remoteStorage.Batch(actions =>
{
var firstAndLastDocumentIds = actions.Documents.FirstAndLastDocumentIds();
finalResult = firstAndLastDocumentIds.Item2;
var start = Math.Max(firstAndLastDocumentIds.Item1, query.Start);
var matchingDocs = actions.Documents.DocumentsById(start, firstAndLastDocumentIds.Item2);
if (string.IsNullOrEmpty(viewGenerator.ForEntityName) == false) //optimization
{
matchingDocs =
matchingDocs.Where(x => x.Item1.Metadata.Value<string>("Raven-Entity-Name") == viewGenerator.ForEntityName);
}
var documentRetriever = new DocumentRetriever(actions, new AbstractReadTrigger[0]);
var docs = matchingDocs
.Select(x=>
{
documentRetriever.EnsureIdInMetadata(x.Item1);
return x;
})
.Select(x =>
{
lastResult = x.Item2;
return new DynamicJsonObject(x.Item1.ToJson());
});
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
errors.Add(String.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results.AddRange(
robustEnumerator.RobustEnumeration(docs, viewGenerator.MapDefinition)
.Take(query.PageSize)
.Select(result => JsonExtensions.ToJObject(result).ToString())
);
});
return new RemoteQueryResults
{
LastScannedResult = lastResult,
TotalResults = finalResult,
Errors = errors.ToArray(),
QueryCacheSize = queryCache.Count,
Results = results.ToArray()
};
}
