public Etag GetBestNextDocumentEtag(Etag etag) { if (etag == null) throw new ArgumentNullException("etag"); using (var iter = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag) .Iterate(Snapshot, writeBatch.Value)) { if (!iter.Seek((Slice)etag.ToString()) && !iter.Seek(Slice.BeforeAllKeys)) //if parameter etag not found, scan from beginning. if empty --> return original etag return etag; do { var docEtag = Etag.Parse(iter.CurrentKey.ToString()); if (EtagUtil.IsGreaterThan(docEtag, etag)) return docEtag; } while (iter.MoveNext()); } return etag; //if not found, return the original etag }
public IEnumerable<JsonDocument> GetDocumentsAfterWithIdStartingWith(Etag etag, string idPrefix, int take, CancellationToken cancellationToken, long? maxSize = null, Etag untilEtag = null, TimeSpan? timeout = null, Action<Etag> lastProcessedDocument = null, Reference<bool> earlyExit = null) { if (earlyExit != null) earlyExit.Value = false; if (take < 0) throw new ArgumentException("must have zero or positive value", "take"); if (take == 0) yield break; if (string.IsNullOrEmpty(etag)) throw new ArgumentNullException("etag"); Stopwatch duration = null; if (timeout != null) duration = Stopwatch.StartNew(); Etag lastDocEtag = null; using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag) .Iterate(Snapshot, writeBatch.Value)) { var slice = (Slice) etag.ToString(); if (iterator.Seek(slice) == false) yield break; if (iterator.CurrentKey.Equals(slice)) // need gt, not ge { if (iterator.MoveNext() == false) yield break; } long fetchedDocumentTotalSize = 0; int fetchedDocumentCount = 0; Etag docEtag = etag; do { cancellationToken.ThrowIfCancellationRequested(); docEtag = Etag.Parse(iterator.CurrentKey.ToString()); // We can skip many documents so the timeout should be at the start of the process to be executed. if (timeout != null) { if (duration.Elapsed > timeout.Value) { if (earlyExit != null) earlyExit.Value = true; break; } } if (untilEtag != null) { // This is not a failure, we are just ahead of when we expected to. if (EtagUtil.IsGreaterThan(docEtag, untilEtag)) break; } var key = GetKeyFromCurrent(iterator); if (!string.IsNullOrEmpty(idPrefix)) { if (!key.StartsWith(idPrefix, StringComparison.OrdinalIgnoreCase)) { // We assume that we have processed it because it is not of our interest. lastDocEtag = docEtag; continue; } } var document = DocumentByKey(key); if (document == null) //precaution - should never be true { if (SkipConsistencyCheck) continue; throw new InvalidDataException(string.Format("Data corruption - the key = '{0}' was found in the documents index, but matching document was not found", key)); } if (!document.Etag.Equals(docEtag) && !SkipConsistencyCheck) { throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between document and its index", key)); } fetchedDocumentTotalSize += document.SerializedSizeOnDisk; fetchedDocumentCount++; yield return document; lastDocEtag = docEtag; if (maxSize.HasValue && fetchedDocumentTotalSize >= maxSize) { if (untilEtag != null && earlyExit != null) earlyExit.Value = true; break; } if (fetchedDocumentCount >= take) { if (untilEtag != null && earlyExit != null) earlyExit.Value = true; break; } } while (iterator.MoveNext()); } // We notify the last that we considered. if (lastProcessedDocument != null) lastProcessedDocument(lastDocEtag); }
public Etag GetEtagAfterSkip(Etag etag, int take, CancellationToken cancellationToken) { if (take < 0) throw new ArgumentException("must have zero or positive value", "take"); if (take == 0) return etag; if (string.IsNullOrEmpty(etag)) throw new ArgumentNullException("etag"); using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag) .Iterate(Snapshot, writeBatch.Value)) { var slice = (Slice)etag.ToString(); if (iterator.Seek(slice) == false) return etag; if (iterator.CurrentKey.Equals(slice)) // need gt, not ge { if (iterator.MoveNext() == false) return etag; } ValueReader etagReader; long totalSize = 0; do { cancellationToken.ThrowIfCancellationRequested(); etagReader = iterator.CurrentKey.CreateReader(); /*var docKey = GetKeyFromCurrent(iterator); var readResult = tableStorage.Documents.Read(Snapshot, docKey, null); totalSize += readResult.Reader.Length; if (maxSize.HasValue && totalSize >= maxSize) break;*/ } while (iterator.MoveNext() && --take > 0); return Etag.Parse(etagReader.ReadBytes(16, out take)); } }
private void Index(IndexWriter writer, string key, RavenJObject metadata, Etag etag, bool recreateSearcher) { if (filesystem.ReadTriggers.CanReadFile(key, metadata, ReadOperation.Index) == false) return; lock (writerLock) { var lowerKey = key.ToLowerInvariant(); var doc = CreateDocument(lowerKey, metadata); // REVIEW: Check if there is more straight-forward/efficient pattern out there to work with RavenJObjects. var lookup = metadata.ToLookup(x => x.Key); foreach (var metadataKey in lookup) { foreach (var metadataHolder in metadataKey) { var array = metadataHolder.Value as RavenJArray; if (array != null) { // Object is an array. Therefore, we index each token. foreach (var item in array) AddField(doc, metadataHolder.Key, item.ToString()); } else { AddField(doc, metadataHolder.Key, metadataHolder.Value.ToString()); } } } if (doc.GetField(Constants.MetadataEtagField) == null) doc.Add(new Field(Constants.MetadataEtagField, etag.ToString(), Field.Store.NO, Field.Index.ANALYZED_NO_NORMS)); writer.DeleteDocuments(new Term("__key", lowerKey)); writer.AddDocument(doc); var customCommitData = new Dictionary<string, string> { { "LastETag", etag.ToString() } }; writer.Commit(customCommitData); if (recreateSearcher) ReplaceSearcher(writer); } }
public IEnumerable<JsonDocument> GetDocumentsAfter(Etag etag, int take, CancellationToken cancellationToken, long? maxSize = null, Etag untilEtag = null, TimeSpan? timeout = null) { if (take < 0) throw new ArgumentException("must have zero or positive value", "take"); if (take == 0) yield break; if (string.IsNullOrEmpty(etag)) throw new ArgumentNullException("etag"); Stopwatch duration = null; if (timeout != null) duration = Stopwatch.StartNew(); using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag) .Iterate(Snapshot, writeBatch.Value)) { Slice slice = etag.ToString(); if (iterator.Seek(slice) == false) yield break; if (iterator.CurrentKey.Equals(slice)) // need gt, not ge { if(iterator.MoveNext() == false) yield break; } long fetchedDocumentTotalSize = 0; int fetchedDocumentCount = 0; do { cancellationToken.ThrowIfCancellationRequested(); var docEtag = Etag.Parse(iterator.CurrentKey.ToString()); if (untilEtag != null) { if (EtagUtil.IsGreaterThan(docEtag, untilEtag)) yield break; } var key = GetKeyFromCurrent(iterator); var document = DocumentByKey(key); if (document == null) //precaution - should never be true { throw new InvalidDataException(string.Format("Data corruption - the key = '{0}' was found in the documents indice, but matching document was not found", key)); } if (!document.Etag.Equals(docEtag)) { throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between document and its indice",key)); } fetchedDocumentTotalSize += document.SerializedSizeOnDisk; fetchedDocumentCount++; if (maxSize.HasValue && fetchedDocumentTotalSize >= maxSize) { yield return document; yield break; } yield return document; if (timeout != null) { if (duration.Elapsed > timeout.Value) yield break; } } while (iterator.MoveNext() && fetchedDocumentCount < take); } }
public Etag GetEtagAfterSkip(Etag etag, int skip, CancellationToken cancellationToken, out int skipped) { if (skip < 0) throw new ArgumentException("must have zero or positive value", "skip"); if (skip == 0) { skipped = 0; return etag; } if (string.IsNullOrEmpty(etag)) throw new ArgumentNullException("etag"); using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag) .Iterate(Snapshot, writeBatch.Value)) { var slice = (Slice)etag.ToString(); if (iterator.Seek(slice) == false) { skipped = 0; return etag; } var count = 0; if (iterator.CurrentKey.Equals(slice) || etag == Etag.Empty) // need gt, not ge { if (iterator.MoveNext() == false) { skipped = 0; return etag; } count++; } Slice etagSlice; do { cancellationToken.ThrowIfCancellationRequested(); etagSlice = iterator.CurrentKey; if (count >= skip) break; count++; } while (iterator.MoveNext()); skipped = count; return Etag.Parse(etagSlice.ToString()); } }