public IEnumerable <TimeSeriesItem> GetTimeSeries(List <string> collectionsToExport)
{
Debug.Assert(_context != null);
var initialState = new TimeSeriesIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit)
{
StartEtag = _startDocumentEtag,
StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag)
};
var enumerator = new PulsedTransactionEnumerator <TimeSeriesItem, TimeSeriesIterationState>(_context,
state =>
{
if (state.StartEtagByCollection.Count != 0)
{
return(GetTimeSeriesFromCollections(_context, state));
}
return(GetAllTimeSeriesItems(_context, state.StartEtag));
}, initialState);
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
public IEnumerable <DocumentItem> GetRevisionDocuments(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var revisionsStorage = _database.DocumentsStorage.RevisionsStorage;
var enumerator = new PulsedTransactionEnumerator <Document, DocumentsIterationState>(_context,
state =>
{
if (state.StartEtagByCollection.Count != 0)
{
return(GetRevisionsFromCollections(_context, state));
}
return(revisionsStorage.GetRevisionsFrom(_context, state.StartEtag, int.MaxValue));
},
new DocumentsIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit) // initial state
{
StartEtag = _startDocumentEtag,
StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag)
});
while (enumerator.MoveNext())
{
yield return(new DocumentItem
{
Document = enumerator.Current
});
}
}
public IEnumerable <CounterGroupDetail> GetCounterValues(List <string> collectionsToExport, ICounterActions actions)
{
Debug.Assert(_context != null);
var enumerator = new PulsedTransactionEnumerator <CounterGroupDetail, CountersIterationState>(_context,
state =>
{
if (state.StartEtagByCollection.Count != 0)
{
return(GetCounterValuesFromCollections(_context, state));
}
return(_database.DocumentsStorage.CountersStorage.GetCountersFrom(_context, state.StartEtag, 0, long.MaxValue));
},
new CountersIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit) // initial state
{
StartEtag = _startDocumentEtag,
StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag)
});
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public async IAsyncEnumerable <TimeSeriesItem> GetTimeSeriesAsync(List <string> collectionsToExport)
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
Debug.Assert(_context != null);
var initialState = new TimeSeriesIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit)
{
StartEtag = _startDocumentEtag,
StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag)
};
var enumerator = new PulsedTransactionEnumerator <TimeSeriesItem, TimeSeriesIterationState>(_context,
state =>
{
if (state.StartEtagByCollection.Count != 0)
{
return(GetTimeSeriesFromCollections(_context, state));
}
return(GetAllTimeSeriesItems(_context, state.StartEtag));
}, initialState);
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public async IAsyncEnumerable <CounterGroupDetail> GetCounterValuesAsync(List <string> collectionsToExport, ICounterActions actions)
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
Debug.Assert(_context != null);
var enumerator = new PulsedTransactionEnumerator <CounterGroupDetail, CountersIterationState>(_context,
state =>
{
if (state.StartEtagByCollection.Count != 0)
{
return(GetCounterValuesFromCollections(_context, state));
}
return(_database.DocumentsStorage.CountersStorage.GetCountersFrom(_context, state.StartEtag, 0, long.MaxValue));
},
new CountersIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit) // initial state
{
StartEtag = _startDocumentEtag,
StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag)
});
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public async IAsyncEnumerable <DocumentConflict> GetConflictsAsync(List <string> collectionsToExport, INewDocumentActions actions)
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
Debug.Assert(_context != null);
var enumerator = new PulsedTransactionEnumerator <DocumentConflict, DocumentConflictsIterationState>(_context,
state =>
{
if (collectionsToExport.Count != 0)
{
return(GetConflictsFromCollections(_context, collectionsToExport.ToHashSet(), state));
}
return(_database.DocumentsStorage.ConflictsStorage.GetConflictsFrom(_context, state.StartEtag));
},
new DocumentConflictsIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit)
{
StartEtag = _startDocumentEtag,
});
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
public Task StreamDocsGet()
{
var start = GetStart();
var pageSize = GetPageSize();
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var initialState = new DocsStreamingIterationState(context, Database.Configuration.Databases.PulseReadTransactionLimit)
{
Start = start,
Take = pageSize
};
if (HttpContext.Request.Query.ContainsKey("startsWith"))
{
initialState.StartsWith = HttpContext.Request.Query["startsWith"];
initialState.Excludes = HttpContext.Request.Query["excludes"];
initialState.Matches = HttpContext.Request.Query["matches"];
initialState.StartAfter = HttpContext.Request.Query["startAfter"];
initialState.Skip = new Reference <long>();
}
var documentsEnumerator = new PulsedTransactionEnumerator <Document, DocsStreamingIterationState>(context, state =>
{
if (string.IsNullOrEmpty(state.StartsWith) == false)
{
return(Database.DocumentsStorage.GetDocumentsStartingWith(context, state.StartsWith, state.Matches, state.Excludes, state.StartAfter,
state.LastIteratedEtag == null ? state.Start : 0, // if we iterated already some docs then we pass 0 as Start and rely on state.Skip
state.Take,
state.Skip));
}
if (state.LastIteratedEtag != null)
{
return(Database.DocumentsStorage.GetDocumentsInReverseEtagOrderFrom(context, state.LastIteratedEtag.Value, state.Take, skip: 1)); // we seek to LastIteratedEtag but skip 1 item because we iterated it already
}
return(Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, state.Start, state.Take));
},
initialState);
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteDocuments(context, documentsEnumerator, metadataOnly: false, numberOfResults: out long _);
writer.WriteEndObject();
}
}
return(Task.CompletedTask);
}
public IEnumerable <DocumentConflict> GetConflicts(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var enumerator = new PulsedTransactionEnumerator <DocumentConflict, DocumentConflictsIterationState>(_context,
state =>
{
if (collectionsToExport.Count != 0)
{
return(GetConflictsFromCollections(_context, collectionsToExport.ToHashSet(), state));
}
return(_database.DocumentsStorage.ConflictsStorage.GetConflictsFrom(_context, state.StartEtag));
},
new DocumentConflictsIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit)
{
StartEtag = _startDocumentEtag,
});
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
public IEnumerable <Tombstone> GetTombstones(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var enumerator = new PulsedTransactionEnumerator <Tombstone, TombstonesIterationState>(_context,
state =>
{
if (state.StartEtagByCollection.Count != 0)
{
return(GetTombstonesFromCollections(_context, state));
}
return(_database.DocumentsStorage.GetTombstonesFrom(_context, state.StartEtag, 0, int.MaxValue));
},
new TombstonesIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit)
{
StartEtag = _startDocumentEtag, StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag)
});
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
private void ExecuteCollectionQuery(QueryResultServerSide <Document> resultToFill, IndexQueryServerSide query, string collection, DocumentsOperationContext context, bool pulseReadingTransaction, CancellationToken cancellationToken)
{
using (var queryScope = query.Timings?.For(nameof(QueryTimingsScope.Names.Query)))
{
QueryTimingsScope gatherScope = null;
QueryTimingsScope fillScope = null;
if (queryScope != null && query.Metadata.Includes?.Length > 0)
{
var includesScope = queryScope.For(nameof(QueryTimingsScope.Names.Includes), start: false);
gatherScope = includesScope.For(nameof(QueryTimingsScope.Names.Gather), start: false);
fillScope = includesScope.For(nameof(QueryTimingsScope.Names.Fill), start: false);
}
// we optimize for empty queries without sorting options, appending CollectionIndexPrefix to be able to distinguish index for collection vs. physical index
resultToFill.IsStale = false;
resultToFill.LastQueryTime = DateTime.MinValue;
resultToFill.IndexTimestamp = DateTime.MinValue;
resultToFill.IncludedPaths = query.Metadata.Includes;
var fieldsToFetch = new FieldsToFetch(query, null);
var includeDocumentsCommand = new IncludeDocumentsCommand(Database.DocumentsStorage, context, query.Metadata.Includes, fieldsToFetch.IsProjection);
var totalResults = new Reference <int>();
IEnumerator <Document> enumerator;
if (pulseReadingTransaction == false)
{
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context, includeDocumentsCommand, totalResults);
enumerator = documents.GetEnumerator();
}
else
{
enumerator = new PulsedTransactionEnumerator <Document, CollectionQueryResultsIterationState>(context,
state =>
{
query.Start = state.Start;
query.PageSize = state.Take;
var documents = new CollectionQueryEnumerable(Database, Database.DocumentsStorage, fieldsToFetch, collection, query, queryScope, context,
includeDocumentsCommand, totalResults);
return(documents);
},
new CollectionQueryResultsIterationState(context, Database.Configuration.Databases.PulseReadTransactionLimit)
{
Start = query.Start,
Take = query.PageSize
});
}
IncludeCountersCommand includeCountersCommand = null;
if (query.Metadata.CounterIncludes != null)
{
includeCountersCommand = new IncludeCountersCommand(
Database,
context,
query.Metadata.CounterIncludes.Counters);
}
try
{
using (enumerator)
{
while (enumerator.MoveNext())
{
var document = enumerator.Current;
cancellationToken.ThrowIfCancellationRequested();
resultToFill.AddResult(document);
using (gatherScope?.Start())
includeDocumentsCommand.Gather(document);
includeCountersCommand?.Fill(document);
}
}
}
catch (Exception e)
{
if (resultToFill.SupportsExceptionHandling == false)
{
throw;
}
resultToFill.HandleException(e);
}
using (fillScope?.Start())
includeDocumentsCommand.Fill(resultToFill.Includes);
if (includeCountersCommand != null)
{
resultToFill.AddCounterIncludes(includeCountersCommand);
}
resultToFill.TotalResults = (totalResults.Value == 0 && resultToFill.Results.Count != 0) ? -1 : totalResults.Value;
if (query.Offset != null || query.Limit != null)
{
if (resultToFill.TotalResults == -1)
{
resultToFill.CappedMaxResults = query.Limit ?? -1;
}
else
{
resultToFill.CappedMaxResults = Math.Min(
query.Limit ?? int.MaxValue,
resultToFill.TotalResults - (query.Offset ?? 0)
);
}
}
}
}