public Etag GetBestNextDocumentEtag(Etag etag)
{
if (etag == null) throw new ArgumentNullException("etag");
using (var iter = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
.Iterate(Snapshot, writeBatch.Value))
{
if (!iter.Seek((Slice)etag.ToString()) &&
!iter.Seek(Slice.BeforeAllKeys)) //if parameter etag not found, scan from beginning. if empty --> return original etag
return etag;
do
{
var docEtag = Etag.Parse(iter.CurrentKey.ToString());
if (EtagUtil.IsGreaterThan(docEtag, etag))
return docEtag;
} while (iter.MoveNext());
}
return etag; //if not found, return the original etag
}
public IEnumerable<JsonDocument> GetDocumentsAfterWithIdStartingWith(Etag etag, string idPrefix, int take, CancellationToken cancellationToken, long? maxSize = null, Etag untilEtag = null, TimeSpan? timeout = null, Action<Etag> lastProcessedDocument = null,
Reference<bool> earlyExit = null)
{
if (earlyExit != null)
earlyExit.Value = false;
if (take < 0)
throw new ArgumentException("must have zero or positive value", "take");
if (take == 0)
yield break;
if (string.IsNullOrEmpty(etag))
throw new ArgumentNullException("etag");
Stopwatch duration = null;
if (timeout != null)
duration = Stopwatch.StartNew();
Etag lastDocEtag = null;
using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
.Iterate(Snapshot, writeBatch.Value))
{
var slice = (Slice) etag.ToString();
if (iterator.Seek(slice) == false)
yield break;
if (iterator.CurrentKey.Equals(slice)) // need gt, not ge
{
if (iterator.MoveNext() == false)
yield break;
}
long fetchedDocumentTotalSize = 0;
int fetchedDocumentCount = 0;
Etag docEtag = etag;
do
{
cancellationToken.ThrowIfCancellationRequested();
docEtag = Etag.Parse(iterator.CurrentKey.ToString());
// We can skip many documents so the timeout should be at the start of the process to be executed.
if (timeout != null)
{
if (duration.Elapsed > timeout.Value)
{
if (earlyExit != null)
earlyExit.Value = true;
break;
}
}
if (untilEtag != null)
{
// This is not a failure, we are just ahead of when we expected to.
if (EtagUtil.IsGreaterThan(docEtag, untilEtag))
break;
}
var key = GetKeyFromCurrent(iterator);
if (!string.IsNullOrEmpty(idPrefix))
{
if (!key.StartsWith(idPrefix, StringComparison.OrdinalIgnoreCase))
{
// We assume that we have processed it because it is not of our interest.
lastDocEtag = docEtag;
continue;
}
}
var document = DocumentByKey(key);
if (document == null) //precaution - should never be true
{
if (SkipConsistencyCheck) continue;
throw new InvalidDataException(string.Format("Data corruption - the key = '{0}' was found in the documents index, but matching document was not found", key));
}
if (!document.Etag.Equals(docEtag) && !SkipConsistencyCheck)
{
throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between document and its index", key));
}
fetchedDocumentTotalSize += document.SerializedSizeOnDisk;
fetchedDocumentCount++;
yield return document;
lastDocEtag = docEtag;
if (maxSize.HasValue && fetchedDocumentTotalSize >= maxSize)
{
if (untilEtag != null && earlyExit != null)
earlyExit.Value = true;
break;
}
if (fetchedDocumentCount >= take)
{
if (untilEtag != null && earlyExit != null)
earlyExit.Value = true;
break;
}
} while (iterator.MoveNext());
}
// We notify the last that we considered.
if (lastProcessedDocument != null)
lastProcessedDocument(lastDocEtag);
}
public Etag GetEtagAfterSkip(Etag etag, int take, CancellationToken cancellationToken)
{
if (take < 0)
throw new ArgumentException("must have zero or positive value", "take");
if (take == 0)
return etag;
if (string.IsNullOrEmpty(etag))
throw new ArgumentNullException("etag");
using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
.Iterate(Snapshot, writeBatch.Value))
{
var slice = (Slice)etag.ToString();
if (iterator.Seek(slice) == false)
return etag;
if (iterator.CurrentKey.Equals(slice)) // need gt, not ge
{
if (iterator.MoveNext() == false)
return etag;
}
ValueReader etagReader;
long totalSize = 0;
do
{
cancellationToken.ThrowIfCancellationRequested();
etagReader = iterator.CurrentKey.CreateReader();
/*var docKey = GetKeyFromCurrent(iterator);
var readResult = tableStorage.Documents.Read(Snapshot, docKey, null);
totalSize += readResult.Reader.Length;
if (maxSize.HasValue && totalSize >= maxSize)
break;*/
}
while (iterator.MoveNext() && --take > 0);
return Etag.Parse(etagReader.ReadBytes(16, out take));
}
}
private void Index(IndexWriter writer, string key, RavenJObject metadata, Etag etag, bool recreateSearcher)
{
if (filesystem.ReadTriggers.CanReadFile(key, metadata, ReadOperation.Index) == false)
return;
lock (writerLock)
{
var lowerKey = key.ToLowerInvariant();
var doc = CreateDocument(lowerKey, metadata);
// REVIEW: Check if there is more straight-forward/efficient pattern out there to work with RavenJObjects.
var lookup = metadata.ToLookup(x => x.Key);
foreach (var metadataKey in lookup)
{
foreach (var metadataHolder in metadataKey)
{
var array = metadataHolder.Value as RavenJArray;
if (array != null)
{
// Object is an array. Therefore, we index each token.
foreach (var item in array)
AddField(doc, metadataHolder.Key, item.ToString());
}
else
{
AddField(doc, metadataHolder.Key, metadataHolder.Value.ToString());
}
}
}
if (doc.GetField(Constants.MetadataEtagField) == null)
doc.Add(new Field(Constants.MetadataEtagField, etag.ToString(), Field.Store.NO, Field.Index.ANALYZED_NO_NORMS));
writer.DeleteDocuments(new Term("__key", lowerKey));
writer.AddDocument(doc);
var customCommitData = new Dictionary<string, string> { { "LastETag", etag.ToString() } };
writer.Commit(customCommitData);
if (recreateSearcher)
ReplaceSearcher(writer);
}
}
public IEnumerable<JsonDocument> GetDocumentsAfter(Etag etag, int take, CancellationToken cancellationToken, long? maxSize = null, Etag untilEtag = null, TimeSpan? timeout = null)
{
if (take < 0)
throw new ArgumentException("must have zero or positive value", "take");
if (take == 0) yield break;
if (string.IsNullOrEmpty(etag))
throw new ArgumentNullException("etag");
Stopwatch duration = null;
if (timeout != null)
duration = Stopwatch.StartNew();
using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
.Iterate(Snapshot, writeBatch.Value))
{
Slice slice = etag.ToString();
if (iterator.Seek(slice) == false)
yield break;
if (iterator.CurrentKey.Equals(slice)) // need gt, not ge
{
if(iterator.MoveNext() == false)
yield break;
}
long fetchedDocumentTotalSize = 0;
int fetchedDocumentCount = 0;
do
{
cancellationToken.ThrowIfCancellationRequested();
var docEtag = Etag.Parse(iterator.CurrentKey.ToString());
if (untilEtag != null)
{
if (EtagUtil.IsGreaterThan(docEtag, untilEtag))
yield break;
}
var key = GetKeyFromCurrent(iterator);
var document = DocumentByKey(key);
if (document == null) //precaution - should never be true
{
throw new InvalidDataException(string.Format("Data corruption - the key = '{0}' was found in the documents indice, but matching document was not found", key));
}
if (!document.Etag.Equals(docEtag))
{
throw new InvalidDataException(string.Format("Data corruption - the etag for key ='{0}' is different between document and its indice",key));
}
fetchedDocumentTotalSize += document.SerializedSizeOnDisk;
fetchedDocumentCount++;
if (maxSize.HasValue && fetchedDocumentTotalSize >= maxSize)
{
yield return document;
yield break;
}
yield return document;
if (timeout != null)
{
if (duration.Elapsed > timeout.Value)
yield break;
}
} while (iterator.MoveNext() && fetchedDocumentCount < take);
}
}
public Etag GetEtagAfterSkip(Etag etag, int skip, CancellationToken cancellationToken, out int skipped)
{
if (skip < 0)
throw new ArgumentException("must have zero or positive value", "skip");
if (skip == 0)
{
skipped = 0;
return etag;
}
if (string.IsNullOrEmpty(etag))
throw new ArgumentNullException("etag");
using (var iterator = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag)
.Iterate(Snapshot, writeBatch.Value))
{
var slice = (Slice)etag.ToString();
if (iterator.Seek(slice) == false)
{
skipped = 0;
return etag;
}
var count = 0;
if (iterator.CurrentKey.Equals(slice) || etag == Etag.Empty) // need gt, not ge
{
if (iterator.MoveNext() == false)
{
skipped = 0;
return etag;
}
count++;
}
Slice etagSlice;
do
{
cancellationToken.ThrowIfCancellationRequested();
etagSlice = iterator.CurrentKey;
if (count >= skip)
break;
count++;
}
while (iterator.MoveNext());
skipped = count;
return Etag.Parse(etagSlice.ToString());
}
}