public Etag CalculateSynchronizationEtag(Etag etag, Etag lastProcessedEtag) { if (etag == null) { if (lastProcessedEtag != null) { lock (locker) { if (currentEtag == null && lastProcessedEtag.CompareTo(synchronizationEtag) != 0) { synchronizationEtag = lastProcessedEtag; PersistSynchronizationState(); } } return lastProcessedEtag; } return Etag.Empty; } if (lastProcessedEtag == null) return Etag.Empty; if (etag.CompareTo(lastProcessedEtag) < 0) return EtagUtil.Increment(etag, -1); return lastProcessedEtag; }
protected override async Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag) { var totalCount = 0; while (true) { var array = GetAttachments(totalCount, lastEtag); if (array.Length == 0) { var databaseStatistics = await GetStats(); if (lastEtag == null) lastEtag = Etag.Empty; if (lastEtag.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } totalCount += array.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount); foreach (var item in array) { item.WriteTo(jsonWriter); } lastEtag = Etag.Parse(array.Last().Value<string>("Etag")); } }
private bool UpdateSynchronizationStateInternal(Etag lowestEtag) { if (currentEtag == null || lowestEtag.CompareTo(currentEtag) < 0) { currentEtag = lowestEtag; } return lowestEtag.CompareTo(synchronizationEtag) < 0; }
public IEnumerable<ListItem> Read(string name, Etag start, Etag end, int take) { return storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject { { "name", name }, { "etag", start.ToByteArray() } }) .TakeWhile(x=> StringComparer.OrdinalIgnoreCase.Equals(x.Value<string>("name"), name) && (end == null || end.CompareTo(Etag.Parse(x.Value<byte[]>("etag"))) > 0) ) .Take(take) .Select(result => { var readResult = storage.Lists.Read(result); return new ListItem { Data = readResult.Data().ToJObject(), Etag = Etag.Parse(readResult.Key.Value<byte[]>("etag")), Key = readResult.Key.Value<string>("key") }; }); }
public static bool IsGreaterThanOrEqual(Etag x, Etag y) { return(x.CompareTo(y) >= 0); }
public static bool IsGreaterThan(Etag x, Etag y) { return(x.CompareTo(y) > 0); }
protected virtual async Task<Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { var totalCount = 0; var maxEtagReached = false; while (true) { try { if (SmugglerOptions.Limit - totalCount <= 0 || maxEtagReached) { Operations.ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize); var attachments = await Operations.GetAttachments(totalCount, lastEtag, maxRecords); if (attachments.Count == 0) { var databaseStatistics = await Operations.GetStats(); if (lastEtag == null) lastEtag = Etag.Empty; if (lastEtag.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); Operations.ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } Operations.ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } totalCount += attachments.Count; Operations.ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Count, totalCount); foreach (var attachment in attachments) { if (maxEtag != null && attachment.Etag.CompareTo(maxEtag) > 0) { maxEtagReached = true; break; } var attachmentData = await Operations.GetAttachmentData(attachment); if (attachmentData == null) continue; new RavenJObject { { "Data", attachmentData }, { "Metadata", attachment.Metadata }, { "Key", attachment.Key }, { "Etag", new RavenJValue(attachment.Etag.ToString()) } }.WriteTo(jsonWriter); lastEtag = attachment.Etag; } } catch (Exception e) { Operations.ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); Operations.ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } }
private bool TryGetDocumentsFromQueue(Etag nextDocEtag, List<JsonDocument> items, int? take) { JsonDocument result; bool hasDocs = false; while (items.Count < GetNumberOfItemsToProcessInSingleBatch() && prefetchingQueue.TryPeek(out result) && // we compare to current or _smaller_ so we will remove from the queue old versions // of documents that we have already loaded nextDocEtag.CompareTo(result.Etag) >= 0) { // safe to do peek then dequeue because we are the only one doing the dequeues // and here we are single threaded, but still, better to check if (prefetchingQueue.TryDequeue(out result) == false) continue; // this shouldn't happen, but... if (result == null) continue; if (result.Etag != nextDocEtag) continue; items.Add(result); hasDocs = true; if (take.HasValue && items.Count >= take.Value) break; nextDocEtag = Abstractions.Util.EtagUtil.Increment(nextDocEtag, 1); } return hasDocs; }
private Etag ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { int totalCount = 0; while (true) { var watch = Stopwatch.StartNew(); var documents = GetDocuments(lastEtag); watch.Stop(); if (documents.Length == 0) { var databaseStatistics = GetStats(); if (lastEtag == null) lastEtag = Etag.Empty; if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, smugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}",lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return lastEtag; } var currentProcessingTime = watch.Elapsed; ModifyBatchSize(options, currentProcessingTime); var final = documents.Where(options.MatchFilters).ToList(); if (options.ShouldExcludeExpired) final = documents.Where(options.ExcludeExpired).ToList(); final.ForEach(item => item.WriteTo(jsonWriter)); totalCount += final.Count; ShowProgress("Reading batch of {0,3} documents, read so far: {1,10:#,#;;0}", documents.Length, totalCount); lastEtag = Etag.Parse(documents.Last().Value<RavenJObject>("@metadata").Value<string>("@etag")); } }
public IEnumerable<ListItem> Read(string name, Etag start, Etag end, int take) { Api.JetSetCurrentIndex(session, Lists, "by_name_and_etag"); Api.MakeKey(session, Lists, name, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, Lists, start.TransformToValueForEsentSorting(), MakeKeyGrbit.None); if (Api.TrySeek(session, Lists, SeekGrbit.SeekGT) == false) yield break; int count = 0; do { var nameFromDb = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["name"], Encoding.Unicode); if (string.Equals(name, nameFromDb, StringComparison.InvariantCultureIgnoreCase) == false) yield break; var etag = Etag.Parse(Api.RetrieveColumn(session, Lists, tableColumnsCache.ListsColumns["etag"])); if (end != null && end.CompareTo(etag) <= 0) yield break; count++; using (Stream stream = new BufferedStream(new ColumnStream(session, Lists, tableColumnsCache.ListsColumns["data"]))) { yield return new ListItem { Etag = etag, Data = stream.ToJObject(), Key = Api.RetrieveColumnAsString(session, Lists, tableColumnsCache.ListsColumns["key"], Encoding.Unicode), CreatedAt = Api.RetrieveColumnAsDateTime(session, Lists, tableColumnsCache.ListsColumns["created_at"]).Value }; } } while (Api.TryMoveNext(session, Lists) && count < take); }
public void CleanupDocuments(Etag lastIndexedEtag) { if (lastIndexedEtag == null) return; foreach (var docToRemove in documentsToRemove) { if (docToRemove.Value == null) continue; if (docToRemove.Value.All(etag => lastIndexedEtag.CompareTo(etag) > 0) == false) continue; HashSet<Etag> _; documentsToRemove.TryRemove(docToRemove.Key, out _); } HandleCleanupOfUnusedDocumentsInQueue(); }
public static bool IsGreaterThanOrEqual(Etag x, Etag y) { return x.CompareTo(y) >= 0; }
public static bool IsGreaterThan(Etag x, Etag y) { return x.CompareTo(y) > 0; }
public void CleanupDocuments(Etag lastIndexedEtag) { foreach (var docToRemove in documentsToRemove) { if (docToRemove.Value.All(etag => lastIndexedEtag.CompareTo(etag) > 0) == false) continue; HashSet<Etag> _; documentsToRemove.TryRemove(docToRemove.Key, out _); } updatedDocumentsLock.EnterWriteLock(); try { updatedDocuments.RemoveSmallerOrEqual(lastIndexedEtag); } finally { updatedDocumentsLock.ExitWriteLock(); } JsonDocument result; while (prefetchingQueue.TryPeek(out result) && lastIndexedEtag.CompareTo(result.Etag) >= 0) { prefetchingQueue.TryDequeue(out result); } }
private void HandleCollectingDocumentsAfterCommit(Etag requestedEtag) { if (ShouldHandleUnusedDocumentsAddedAfterCommit == false) return; var current = lowestInMemoryDocumentAddedAfterCommit; if (DisableCollectingDocumentsAfterCommit) { if (current != null && requestedEtag.CompareTo(current.Etag) > 0) { lowestInMemoryDocumentAddedAfterCommit = null; DisableCollectingDocumentsAfterCommit = false; numberOfTimesIngestRateWasTooHigh = 0; } } else if (current != null) { var oldestTimeInQueue = SystemTime.UtcNow - current.AddedAt; if ( // this can happen if we have a very long indexing time, which takes // us a long while to process, so we might as might as well stop keeping // stuff in memory, because we lag oldestTimeInQueue > TimeSpan.FromMinutes(10) ) { DisableCollectingDocumentsAfterCommit = true; // If we disable in memory collection of data, we need to also remove all the // items in the prefetching queue that are after the last in memory docs // immediately, not wait for them to be indexed. They have already been in // memory for ten minutes // But not if our time has finally been reached if (requestedEtag.CompareTo(current.Etag) >= 0) prefetchingQueue.RemoveAfter(current.Etag); } } }
protected override async Task<Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { var totalCount = 0; var maxEtagReached = false; while (true) { try { if (SmugglerOptions.Limit - totalCount <= 0 || maxEtagReached) { ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize); var array = GetAttachments(totalCount, lastEtag, maxRecords); if (array.Length == 0) { var databaseStatistics = await GetStats(); if (lastEtag == null) lastEtag = Etag.Empty; if (lastEtag.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } totalCount += array.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount); foreach (var item in array) { var tempLastEtag = item.Value<string>("Etag"); if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0) { maxEtagReached = true; break; } item.WriteTo(jsonWriter); lastEtag = tempLastEtag; } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } }
private void RemoveOutdatedFutureIndexBatches(Etag etag) { foreach (FutureIndexBatch source in futureIndexBatches.Values.Where(x => etag.CompareTo(x.StartingEtag) > 0)) { ObserveDiscardedTask(source); DisposeCancellationToken(source.CancellationTokenSource); FutureIndexBatch batch; futureIndexBatches.TryRemove(source.StartingEtag, out batch); } }
private bool TryGetDocumentsFromQueue(Etag nextDocEtag, ref List<JsonDocument> items) { JsonDocument result; nextDocEtag = HandleEtagGapsIfNeeded(nextDocEtag); bool hasDocs = false; while (items.Count < autoTuner.NumberOfItemsToIndexInSingleBatch && prefetchingQueue.TryPeek(out result) && nextDocEtag.CompareTo(result.Etag) >= 0) { // safe to do peek then dequeue because we are the only one doing the dequeues // and here we are single threaded prefetchingQueue.TryDequeue(out result); if (result.Etag != nextDocEtag) continue; items.Add(result); hasDocs = true; nextDocEtag = EtagUtil.Increment(nextDocEtag, 1); nextDocEtag = HandleEtagGapsIfNeeded(nextDocEtag); } return hasDocs; }
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { var now = SystemTime.UtcNow; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorcount = 0; ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = options.Limit - totalCount; var amountToTakeFromServer = Math.Min(maxRecords, options.BatchSize); if (maxRecords > 0) { using (var documents = await GetDocuments(lastEtag, amountToTakeFromServer)) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { hasDocs = true; var document = documents.Current; lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag")); if (!options.MatchFilters(document)) continue; if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) continue; document.WriteTo(jsonWriter); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } if (watch.ElapsedMilliseconds > 100) errorcount++; watch.Start(); } } if (hasDocs) continue; // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, amountToTakeFromServer); if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0) { lastEtag = databaseStatistics.LastDocEtag; } ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return lastEtag; } }