public bool IsIndexStale(int view, DateTime?cutOff, Etag cutoffEtag)
{
Api.JetSetCurrentIndex(session, IndexesStats, "by_key");
Api.MakeKey(session, IndexesStats, view, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, IndexesStats, SeekGrbit.SeekEQ) == false)
{
return(false);
}
Api.JetSetCurrentIndex(session, IndexesStatsReduce, "by_key");
Api.MakeKey(session, IndexesStatsReduce, view, MakeKeyGrbit.NewKey);
var hasReduce = Api.TrySeek(session, IndexesStatsReduce, SeekGrbit.SeekEQ);
if (IsMapStale(view) || hasReduce && IsReduceStale(view))
{
if (cutOff != null)
{
var indexedTimestamp = Api.RetrieveColumnAsInt64(session, IndexesStats, tableColumnsCache.IndexesStatsColumns["last_indexed_timestamp"]).Value;
var lastIndexedTimestamp = DateTime.FromBinary(indexedTimestamp);
if (cutOff.Value >= lastIndexedTimestamp)
{
return(true);
}
if (hasReduce)
{
var lastReduceIndex = Api.RetrieveColumnAsInt64(session, IndexesStatsReduce, tableColumnsCache.IndexesStatsReduceColumns["last_reduced_timestamp"]);
lastIndexedTimestamp = lastReduceIndex == null ? DateTime.MinValue : DateTime.FromBinary(lastReduceIndex.Value);
if (cutOff.Value >= lastIndexedTimestamp)
{
return(true);
}
}
}
else if (cutoffEtag != null)
{
var lastIndexedEtag = Api.RetrieveColumn(session, IndexesStats,
tableColumnsCache.IndexesStatsColumns["last_indexed_etag"]);
if (Buffers.Compare(lastIndexedEtag, cutoffEtag.ToByteArray()) < 0)
{
return(true);
}
}
else
{
return(true);
}
}
return(IsIndexStaleByTask(view, cutOff));
}
public bool IsIndexStale(string name, DateTime?cutOff, Etag cutoffEtag)
{
var indexingStatsReadResult = storage.IndexingStats.Read(name);
var lastIndexedEtagsReadResult = storage.LastIndexedEtags.Read(name);
if (indexingStatsReadResult == null)
{
return(false); // index does not exists
}
if (IsMapStale(name) || IsReduceStale(name))
{
if (cutOff != null)
{
var lastIndexedTime = lastIndexedEtagsReadResult.Key.Value <DateTime>("lastTimestamp");
if (cutOff.Value >= lastIndexedTime)
{
return(true);
}
var lastReducedTime = indexingStatsReadResult.Key.Value <DateTime?>("lastReducedTimestamp");
if (lastReducedTime != null && cutOff.Value >= lastReducedTime.Value)
{
return(true);
}
}
else if (cutoffEtag != null)
{
var lastIndexedEtag = lastIndexedEtagsReadResult.Key.Value <byte[]>("lastEtag");
if (Buffers.Compare(lastIndexedEtag, cutoffEtag.ToByteArray()) < 0)
{
return(true);
}
}
else
{
return(true);
}
}
var tasksAfterCutoffPoint = storage.Tasks["ByIndexAndTime"].SkipTo(new RavenJObject {
{ "index", name }
});
if (cutOff != null)
{
tasksAfterCutoffPoint = tasksAfterCutoffPoint
.Where(x => x.Value <DateTime>("time") <= cutOff.Value);
}
return(tasksAfterCutoffPoint.Any());
}
public bool IsMapStale(string name)
{
var readResult = storage.IndexingStats.Read(name);
if (readResult == null)
{
return(false); // index does not exists
}
var lastIndexedEtag = readResult.Key.Value <byte[]>("lastEtag");
return(storage.Documents["ByEtag"].SkipFromEnd(0)
.Select(doc => doc.Value <byte[]>("etag"))
.Select(docEtag => Buffers.Compare(docEtag, lastIndexedEtag) > 0)
.FirstOrDefault());
}
public void UpdateLastReduced(int id, Etag etag, DateTime timestamp)
{
var key = CreateKey(id);
ushort version;
var reduceStats = Load(tableStorage.ReduceStats, key, out version);
if (Buffers.Compare(reduceStats.Value <byte[]>("lastReducedEtag"), etag.ToByteArray()) >= 0)
{
return;
}
reduceStats["lastReducedEtag"] = etag.ToByteArray();
reduceStats["lastReducedTimestamp"] = timestamp;
tableStorage.ReduceStats.Add(writeBatch.Value, key, reduceStats, version);
}
internal void UpdateLastWrittenEtag(Guid?etag)
{
if (etag == null)
{
return;
}
var newEtag = etag.Value.ToByteArray();
if (lastEtag == null)
{
lock (lastEtagLocker)
{
if (lastEtag == null)
{
lastEtag = new EtagHolder
{
Bytes = newEtag,
Etag = etag.Value
};
return;
}
}
}
// not the most recent etag
if (Buffers.Compare(lastEtag.Bytes, newEtag) >= 0)
{
return;
}
lock (lastEtagLocker)
{
// not the most recent etag
if (Buffers.Compare(lastEtag.Bytes, newEtag) >= 0)
{
return;
}
lastEtag = new EtagHolder
{
Etag = etag.Value,
Bytes = newEtag
};
}
}
public void UpdateLastIndexed(string index, Guid etag, DateTime timestamp)
{
locker.EnterWriteLock();
try
{
bool updateOperationStatus = false;
var sp = Stopwatch.StartNew();
while (!updateOperationStatus)
{
var readResult = storage.IndexingStats.Read(index);
if (readResult == null)
{
throw new ArgumentException("There is no index with the name: " + index);
}
var ravenJObject = (RavenJObject)readResult.Key.CloneToken();
if (Buffers.Compare(ravenJObject.Value <byte[]>("lastEtag"), etag.ToByteArray()) >= 0)
{
break;
}
ravenJObject["lastEtag"] = etag.ToByteArray();
ravenJObject["lastTimestamp"] = timestamp;
updateOperationStatus = storage.IndexingStats.UpdateKey(ravenJObject);
if (!updateOperationStatus)
{
Thread.Sleep(100);
}
if (sp.Elapsed > etagUpdateTimeout)
{
break;
}
}
}
finally
{
locker.ExitWriteLock();
}
}
public async void Should_modify_etag_after_upload()
{
var content = new RandomStream(10);
var client = NewClient();
// note that file upload modifies ETag twice
await client.UploadAsync("test.bin", new RavenJObject(), content);
var resultFileMetadata = await client.GetMetadataForAsync("test.bin");
var etag0 = resultFileMetadata.Value <Guid>("ETag");
await client.UploadAsync("test.bin", new RavenJObject(), content);
resultFileMetadata = await client.GetMetadataForAsync("test.bin");
var etag1 = resultFileMetadata.Value <Guid>("ETag");
Assert.Equal(Buffers.Compare(new Guid("00000000-0000-0100-0000-000000000002").ToByteArray(), etag0.ToByteArray()), 0);
Assert.Equal(Buffers.Compare(new Guid("00000000-0000-0100-0000-000000000004").ToByteArray(), etag1.ToByteArray()), 0);
Assert.True(Buffers.Compare(etag1.ToByteArray(), etag0.ToByteArray()) > 0, "ETag after second update should be greater");
}
public bool IsMapStale(string name)
{
Api.JetSetCurrentIndex(session, IndexesStats, "by_key");
Api.MakeKey(session, IndexesStats, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, IndexesStats, SeekGrbit.SeekEQ) == false)
{
return(false);
}
var lastIndexedEtag = Api.RetrieveColumn(session, IndexesStats,
tableColumnsCache.IndexesStatsColumns["last_indexed_etag"]);
Api.JetSetCurrentIndex(session, Documents, "by_etag");
if (!Api.TryMoveLast(session, Documents))
{
return(false);
}
var lastEtag = Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"]);
return(Buffers.Compare(lastEtag, lastIndexedEtag) > 0);
}
private bool UpdateDocument(JsonDocument referencingDocument)
{
var referencingEntityName = referencingDocument.Metadata.Value <string>(Constants.RavenEntityName);
ReferencingCollectionSetting referencingCollectionSetting = null;
if (!this.setting.ReferencingCollections.TryGetValue(referencingEntityName, out referencingCollectionSetting))
{
log.Debug("{0} document doesn't need to be cascade updated because it doesn't belong to any document collection that hold denormalized references to {1}. Operation {2} ", referencingDocument.Key, setting.ReferencedEntityName, operation.Id);
return(false);
}
var denormalizedReferences = referencingDocument.DataAsJson.GetObjectsAtPath(referencingCollectionSetting.ReferencingPropertyPath);
bool shouldUpdate = false;
foreach (var reference in denormalizedReferences)
{
Guid?referencedEtag = reference.Value <Guid?>("Etag");
var referencedDocId = reference.Value <string>("Id");
if (referencedDocId == referencedDoc.Key && (referencedEtag == null ||
Buffers.Compare(referencedEtag.Value.ToByteArray(), referencedDoc.Etag.Value.ToByteArray()) < 0))
{
shouldUpdate = true;
foreach (var property in setting.DenormalizedReferencePropertyNames)
{
reference[property] = referencedDoc.DataAsJson[property].CloneToken();
}
reference["Etag"] = RavenJToken.FromObject(referencedDoc.Etag.Value);
}
}
if (shouldUpdate)
{
log.Debug("{0} document has been cascade updated in memory beacause it references {1} document and its referencing Etag is prior to the referenced document one {2}", referencingDocument.Key, referencedDoc.Key, referencedDoc.Etag);
}
else
{
log.Debug("{0} document has not been cascade updated in memory beacause it does not references {1} document or its referencing Etag is subsequent to the referenced document one {2}", referencingDocument.Key, referencedDoc.Key, referencedDoc.Etag);
}
return(shouldUpdate);
}
private void SaveSynchronizationSourceInformation(ServerInfo sourceServer, Guid lastSourceEtag, IStorageActionsAccessor accessor)
{
var lastSynchronizationInformation = GetLastSynchronization(sourceServer.Id, accessor);
if (Buffers.Compare(lastSynchronizationInformation.LastSourceFileEtag.ToByteArray(), lastSourceEtag.ToByteArray()) > 0)
{
return;
}
var synchronizationSourceInfo = new SourceSynchronizationInformation
{
LastSourceFileEtag = lastSourceEtag,
SourceServerUrl = sourceServer.FileSystemUrl,
DestinationServerId = Storage.Id
};
var key = SynchronizationConstants.RavenSynchronizationSourcesBasePath + "/" + sourceServer.Id;
accessor.SetConfig(key, JsonExtensions.ToJObject(synchronizationSourceInfo));
Log.Debug("Saved last synchronized file ETag {0} from {1} ({2})", lastSourceEtag, sourceServer.FileSystemUrl, sourceServer.Id);
}
public bool IsReduceStale(string name)
{
var readResult = storage.IndexingStats.Read(name);
if (readResult == null)
{
return(false); // index does not exists
}
var lastReducedEtag = readResult.Key.Value <byte[]>("lastReducedEtag");
var mostRecentReducedEtag = GetMostRecentReducedEtag(name);
if (mostRecentReducedEtag == null)
{
return(false); // there are no mapped results, maybe there are documents to be indexed, not stale
}
if (lastReducedEtag == null)
{
return(true); // first reduce did not happen
}
return(Buffers.Compare(mostRecentReducedEtag.Value.ToByteArray(), lastReducedEtag) > 0);
}
public bool IsReduceStale(string name)
{
Api.JetSetCurrentIndex(session, IndexesStatsReduce, "by_key");
Api.MakeKey(session, IndexesStatsReduce, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, IndexesStatsReduce, SeekGrbit.SeekEQ) == false)
{
return(false); // not a map/reduce index
}
var lastReducedEtag = Api.RetrieveColumn(session, IndexesStatsReduce, tableColumnsCache.IndexesStatsReduceColumns["last_reduced_etag"]);
var mostRecentReducedEtag = GetMostRecentReducedEtag(name);
if (mostRecentReducedEtag == null)
{
return(false); // there are no mapped results, maybe there are documents to be indexed, not stale
}
if (lastReducedEtag == null)
{
return(true); // first reduce did not happen
}
return(Buffers.Compare(mostRecentReducedEtag.Value.ToByteArray(), lastReducedEtag) > 0);
}
internal void UpdateLastWrittenEtag(Guid?etag)
{
if (!etag.HasValue)
{
return;
}
byte[] y = etag.Value.ToByteArray();
if (this.lastEtag == null)
{
lock (this.lastEtagLocker)
{
if (this.lastEtag == null)
{
this.lastEtag = new DocumentStore.EtagHolder()
{
Bytes = y,
Etag = etag.Value
};
return;
}
}
}
if (Buffers.Compare(this.lastEtag.Bytes, y) >= 0)
{
return;
}
lock (this.lastEtagLocker)
{
if (Buffers.Compare(this.lastEtag.Bytes, y) >= 0)
{
return;
}
this.lastEtag = new DocumentStore.EtagHolder()
{
Etag = etag.Value,
Bytes = y
};
}
}
public bool EnqueueSynchronization(string destinationFileSystemUrl, SynchronizationWorkItem workItem)
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim())
.EnterUpgradeableReadLock();
try
{
var pendingForDestination = pendingSynchronizations.GetOrAdd(destinationFileSystemUrl,
new ConcurrentQueue <SynchronizationWorkItem>());
// if delete work is enqueued and there are other synchronization works for a given file then remove them from a queue
if (workItem.SynchronizationType == SynchronizationType.Delete &&
pendingForDestination.Any(x => x.FileName == workItem.FileName && x.SynchronizationType != SynchronizationType.Delete))
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).EnterWriteLock();
try
{
var modifiedQueue = new ConcurrentQueue <SynchronizationWorkItem>();
foreach (var pendingWork in pendingForDestination)
{
if (pendingWork.FileName != workItem.FileName)
{
modifiedQueue.Enqueue(pendingWork);
}
}
modifiedQueue.Enqueue(workItem);
pendingForDestination = pendingSynchronizations.AddOrUpdate(destinationFileSystemUrl, modifiedQueue,
(key, value) => modifiedQueue);
}
finally
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitWriteLock();
}
}
foreach (var pendingWork in pendingForDestination)
{
// if there is a file in pending synchronizations do not add it again
if (pendingWork.Equals(workItem))
{
if (Log.IsDebugEnabled)
{
Log.Debug("{0} for a file {1} and a destination {2} was already existed in a pending queue",
workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl);
}
return(false);
}
// if there is a work for a file of the same type but with lower file ETag just refresh existing work metadata and do not enqueue again
if (pendingWork.FileName == workItem.FileName &&
pendingWork.SynchronizationType == workItem.SynchronizationType &&
Buffers.Compare(workItem.FileETag.ToByteArray(), pendingWork.FileETag.ToByteArray()) > 0)
{
pendingWork.RefreshMetadata();
if (Log.IsDebugEnabled)
{
Log.Debug(
"{0} for a file {1} and a destination {2} was already existed in a pending queue but with older ETag, it's metadata has been refreshed",
workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl);
}
return(false);
}
}
var activeForDestination = activeSynchronizations.GetOrAdd(destinationFileSystemUrl,
new ConcurrentDictionary <string, SynchronizationWorkItem>
());
// if there is a work in an active synchronizations do not add it again
if (activeForDestination.ContainsKey(workItem.FileName) && activeForDestination[workItem.FileName].Equals(workItem))
{
if (Log.IsDebugEnabled)
{
Log.Debug("{0} for a file {1} and a destination {2} was already existed in an active queue",
workItem.GetType().Name, workItem.FileName, destinationFileSystemUrl);
}
return(false);
}
pendingForDestination.Enqueue(workItem);
if (Log.IsDebugEnabled)
{
Log.Debug("{0} for a file {1} and a destination {2} was enqueued", workItem.GetType().Name, workItem.FileName,
destinationFileSystemUrl);
}
}
finally
{
pendingRemoveLocks.GetOrAdd(destinationFileSystemUrl, new ReaderWriterLockSlim()).ExitUpgradeableReadLock();
}
return(true);
}
public bool IsIndexStale(string name, DateTime?cutOff, Guid?cutoffEtag)
{
Api.JetSetCurrentIndex(session, IndexesStats, "by_key");
Api.MakeKey(session, IndexesStats, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, IndexesStats, SeekGrbit.SeekEQ) == false)
{
return(false);
}
Api.JetSetCurrentIndex(session, IndexesStatsReduce, "by_key");
Api.MakeKey(session, IndexesStatsReduce, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
var hasReduce = Api.TrySeek(session, IndexesStatsReduce, SeekGrbit.SeekEQ);
if (IsMapStale(name) || hasReduce && IsReduceStale(name))
{
if (cutOff != null)
{
var indexedTimestamp = Api.RetrieveColumnAsInt64(session, IndexesStats, tableColumnsCache.IndexesStatsColumns["last_indexed_timestamp"]).Value;
var lastIndexedTimestamp = DateTime.FromBinary(indexedTimestamp);
if (cutOff.Value >= lastIndexedTimestamp)
{
return(true);
}
if (hasReduce)
{
var lastReduceIndex = Api.RetrieveColumnAsInt64(session, IndexesStatsReduce, tableColumnsCache.IndexesStatsReduceColumns["last_reduced_timestamp"]);
lastIndexedTimestamp = lastReduceIndex == null ? DateTime.MinValue : DateTime.FromBinary(lastReduceIndex.Value);
if (cutOff.Value >= lastIndexedTimestamp)
{
return(true);
}
}
}
else if (cutoffEtag != null)
{
var lastIndexedEtag = Api.RetrieveColumn(session, IndexesStats,
tableColumnsCache.IndexesStatsColumns["last_indexed_etag"]);
if (Buffers.Compare(lastIndexedEtag, cutoffEtag.Value.ToByteArray()) < 0)
{
return(true);
}
}
else
{
return(true);
}
}
Api.JetSetCurrentIndex(session, Tasks, "by_index");
Api.MakeKey(session, Tasks, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Tasks, SeekGrbit.SeekEQ) == false)
{
return(false);
}
if (cutOff == null)
{
return(true);
}
// we are at the first row for this index
var addedAt = Api.RetrieveColumnAsInt64(session, Tasks, tableColumnsCache.TasksColumns["added_at"]).Value;
return(cutOff.Value >= DateTime.FromBinary(addedAt));
}
public bool TryStartOperation(UpdateCascadeOperation operation, JsonDocument referencedDoc)
{
var services = Services.GetServices(this.db);
if (services.IsShutDownInProgress)
{
log.Warn("Tried to start operation {0} while shuting down", operation.Id);
return(false);
}
RunningOperation ro = null;
UpdateCascadeSetting setting;
if (!services.SettingsCache.TryGetValue(operation.UpdateCascadeSettingId, out setting))
{
log.Error("Tried to add and run the operation {0}. But there is no corresponding setting {1}", operation.Id, operation.UpdateCascadeSettingId);
return(false);
}
lock (runningOperations)
{
if (runningOperations.TryGetValue(operation.ReferencedDocId, out ro))
{
// the operation might be already here. This shouldn't occur
if (operation.Id == ro.Operation.Id)
{
log.Warn("Tried to start an operation that is already started. Operation Id = {0}", operation.Id);
return(false);
}
// the operation might refer to an older entity. This is unprobable, I think
if (Buffers.Compare(operation.ReferencedDocEtag.ToByteArray(), ro.Operation.ReferencedDocEtag.ToByteArray()) < 0)
{
log.Warn("Tried to start an operation that refers to an entity which is older than the referenced by a running operation. Older operation id: {0}, existing operation id: {1}", operation.Id, ro.Operation.Id);
return(false);
}
log.Warn("The same referenced entity {0} has been updated while a previous update cascade operation of that entity is in progress, that might indicate that the document is updated so often that referencing entities cannot be updated at time. Update cascade bundle is not recomended in this scenario", operation.ReferencedDocId);
// the same referenced entity has been updated while a previous update cascade operation of that entity is in progress
// we need to cancel that operation and span a new one.
var tokenSource = ro.TokenSource;
if (tokenSource != null)
{
tokenSource.Cancel();
}
try
{
var task = ro.ExecutorTask;
if (task != null)
{
task.Wait();
}
}
catch (AggregateException ex)
{
ex.Handle(x => x is TaskCanceledException);
}
}
var runningOperation = new RunningOperation
{
Operation = operation,
TokenSource = new CancellationTokenSource(),
Executor = new UpdateCascadeOperationExecutor(db, setting, operation, referencedDoc),
};
runningOperations[operation.ReferencedDocId] = runningOperation;
log.Trace("Starting operation: {0}", operation.Id);
runningOperation.ExecutorTask = runningOperation.Executor.ExecuteAsync(runningOperation.TokenSource.Token);
runningOperation.ExecutorTask.ContinueWith(t =>
{
if (!services.IsShutDownInProgress)
{
lock (runningOperations)
{
RunningOperation rop;
if (runningOperations.TryGetValue(operation.ReferencedDocId, out rop))
{
if (rop.Operation.Id == operation.Id)
{
runningOperations.Remove(operation.ReferencedDocId);
}
}
t.Dispose();
}
}
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
return(true);
}
}