public Etag AddAttachment(string key, Etag etag, Stream data, RavenJObject headers)
{
AssertValidEtag(key, etag, "PUT");
if(data == null)
{
var attachment = GetAttachment(key);
if(attachment == null)
throw new InvalidOperationException("When adding new attachment, the attachment data must be specified");
attachment.Metadata = headers;
return AddAttachment(key, etag, attachment.Data(), headers);
}
var ms = new MemoryStream();
headers.WriteTo(ms);
data.CopyTo(ms);
var newEtag = generator.CreateSequentialUuid(UuidType.Attachments);
var result = storage.Attachments.Put(new RavenJObject
{
{"key", key},
{"etag", newEtag.ToByteArray()}
}, ms.ToArray());
if (result == false)
throw new ConcurrencyException("PUT attempted on attachment '" + key + "' while it was locked by another transaction");
logger.Debug("Adding attachment {0}", key);
return newEtag;
}
public IndexingBatch(Etag highestEtagBeforeFiltering)
{
HighestEtagBeforeFiltering = highestEtagBeforeFiltering;
Ids = new List<string>();
Docs = new List<dynamic>();
SkipDeleteFromIndex = new List<bool>();
}
internal static void AddEtagHeader(HttpJsonRequest request, Etag etag)
{
if (etag != null)
{
request.AddHeader("If-None-Match", "\"" + etag + "\"");
}
}
public IndexingBatch(Etag highestEtagInBatch)
{
HighestEtagInBatch = highestEtagInBatch;
Ids = new List<string>();
Docs = new List<dynamic>();
SkipDeleteFromIndex = new List<bool>();
}
public override void AfterPut(string key, RavenJObject document, RavenJObject metadata, Etag etag, TransactionInformation transactionInformation)
{
if (key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase))
return;
var status = document.Value<string>("Status");
if (string.IsNullOrWhiteSpace(status) || !status.Equals("Published", StringComparison.InvariantCultureIgnoreCase))
return;
if (metadata.Value<string>(PublishedVersioningConstants.AtisDocumentRevisionStatus) == "Historical")
return;
using (Database.DisableAllTriggersForCurrentThread())
{
var copyMetadata = new RavenJObject(metadata);
copyMetadata[PublishedVersioningConstants.AtisDocumentRevisionStatus] = RavenJToken.FromObject("Historical");
copyMetadata[Constants.RavenReadOnly] = true;
copyMetadata.Remove(PublishedVersioningConstants.AtisDocumentRevision);
object value;
metadata.__ExternalState.TryGetValue("Next-Revision", out value);
var revisionKey = string.Format("{0}/PublishedRevisions/{1}", key, value);
Database.Put(revisionKey, null, (RavenJObject)document.CreateSnapshot(), copyMetadata, transactionInformation);
log.Debug(() => string.Format("Created published revision with key [{0}]", revisionKey));
}
}
private void StreamToClient(Stream stream, int pageSize, Etag etag, OrderedPartCollection<AbstractFileReadTrigger> readTriggers)
{
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(FileSystemsLandlord.SystemConfiguration.DatabaseOperationTimeout))
using (var writer = new JsonTextWriter(new StreamWriter(stream)))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
Storage.Batch(accessor =>
{
var files = accessor.GetFilesAfter(etag, pageSize);
foreach (var file in files)
{
if (readTriggers.CanReadFile(file.FullPath, file.Metadata, ReadOperation.Load) == false)
continue;
timeout.Delay();
var doc = RavenJObject.FromObject(file);
doc.WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
}
});
writer.WriteEndArray();
writer.WriteEndObject();
writer.Flush();
}
}
protected override Task<Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag)
{
if (maxEtag != null)
throw new ArgumentException("We don't support maxEtag in SmugglerDatabaseApi", maxEtag);
return base.ExportAttachments(src, jsonWriter, lastEtag, null);
}
private Etag EnsureDocumentEtagMatch(string key, Etag etag, string method)
{
var existingEtag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"]));
if (etag != null)
{
Etag next;
while (etagTouches.TryGetValue(etag, out next))
{
etag = next;
}
if (existingEtag != etag)
{
if (etag == Etag.Empty)
{
var metadata = Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["metadata"]).ToJObject();
if (metadata.ContainsKey(Constants.RavenDeleteMarker) &&
metadata.Value<bool>(Constants.RavenDeleteMarker))
{
return existingEtag;
}
}
throw new ConcurrencyException(method + " attempted on document '" + key +
"' using a non current etag")
{
ActualETag = existingEtag,
ExpectedETag = etag
};
}
}
return existingEtag;
}
public HttpResponseMessage Get(string name)
{
name = FileHeader.Canonize(name);
FileAndPagesInformation fileAndPages = null;
try
{
Storage.Batch(accessor => fileAndPages = accessor.GetFile(name, 0, 0));
}
catch (FileNotFoundException)
{
log.Debug("File '{0}' was not found", name);
throw new HttpResponseException(HttpStatusCode.NotFound);
}
if (fileAndPages.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
{
log.Debug("File '{0}' is not accessible to get (Raven-Delete-Marker set)", name);
throw new HttpResponseException(HttpStatusCode.NotFound);
}
var readingStream = StorageStream.Reading(Storage, name);
var filename = Path.GetFileName(name);
var result = StreamResult(filename, readingStream);
var etag = new Etag(fileAndPages.Metadata.Value<string>(Constants.MetadataEtagField));
fileAndPages.Metadata.Remove(Constants.MetadataEtagField);
WriteHeaders(fileAndPages.Metadata, etag, result);
log.Debug("File '{0}' with etag {1} is being retrieved.", name, etag);
return result.WithNoCache();
}
private async void WaitForReplicationFromServerAsync(string url, AsyncCountdownEvent countDown, Etag etag, BlockingCollection<Exception> errors)
{
try
{
while (countDown.Active)
{
var etags = await GetReplicatedEtagsFor(url);
var replicated = etag.CompareTo(etags.DocumentEtag) <= 0 || etag.CompareTo(etags.AttachmentEtag) <= 0;
if (!replicated)
{
if (countDown.Active)
{
#if NET45
await Task.Delay(100);
#else
await TaskEx.Delay(100);
#endif
}
continue;
}
countDown.Signal();
return;
}
}
catch (Exception ex)
{
errors.Add(ex);
countDown.Signal();
}
}
public PeriodicExportStatus()
{
LastDocsEtag = Etag.Empty;
LastAttachmentsEtag = Etag.Empty;
LastDocsDeletionEtag = Etag.Empty;
LastAttachmentDeletionEtag = Etag.Empty;
}
public IEnumerable<ListItem> Read(string name, Etag start, Etag end, int take)
{
var listsByName = tableStorage.Lists.GetIndex(Tables.Lists.Indices.ByName);
using (var iterator = listsByName.MultiRead(Snapshot, CreateKey(name)))
{
if (!iterator.Seek(start.ToString()))
yield break;
int count = 0;
do
{
var etag = Etag.Parse(iterator.CurrentKey.ToString());
if (start.CompareTo(etag) >= 0)
continue;
if (end != null && end.CompareTo(etag) <= 0)
yield break;
count++;
yield return ReadInternal(etag);
}
while (iterator.MoveNext() && count < take);
}
}
public Etag GetIndexEtag(string indexName, Etag previousEtag, string resultTransformer = null)
{
Etag lastDocEtag = Etag.Empty;
Etag lastReducedEtag = null;
bool isStale = false;
int touchCount = 0;
TransactionalStorage.Batch(accessor =>
{
var indexInstance = Database.IndexStorage.GetIndexInstance(indexName);
if (indexInstance == null)
return;
isStale = (indexInstance.IsMapIndexingInProgress) ||
accessor.Staleness.IsIndexStale(indexInstance.indexId, null, null);
lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
var indexStats = accessor.Indexing.GetIndexStats(indexInstance.indexId);
if (indexStats != null)
{
lastReducedEtag = indexStats.LastReducedEtag;
}
touchCount = accessor.Staleness.GetIndexTouchCount(indexInstance.indexId);
});
var indexDefinition = GetIndexDefinition(indexName);
if (indexDefinition == null)
return Etag.Empty; // this ensures that we will get the normal reaction of IndexNotFound later on.
using (var md5 = MD5.Create())
{
var list = new List<byte>();
list.AddRange(indexDefinition.GetIndexHash());
list.AddRange(Encoding.Unicode.GetBytes(indexName));
if (string.IsNullOrWhiteSpace(resultTransformer) == false)
{
var abstractTransformer = IndexDefinitionStorage.GetTransformer(resultTransformer);
if (abstractTransformer == null)
throw new InvalidOperationException("The result transformer: " + resultTransformer + " was not found");
list.AddRange(abstractTransformer.GetHashCodeBytes());
}
list.AddRange(lastDocEtag.ToByteArray());
list.AddRange(BitConverter.GetBytes(touchCount));
list.AddRange(BitConverter.GetBytes(isStale));
if (lastReducedEtag != null)
{
list.AddRange(lastReducedEtag.ToByteArray());
}
var indexEtag = Etag.Parse(md5.ComputeHash(list.ToArray()));
if (previousEtag != null && previousEtag != indexEtag)
{
// the index changed between the time when we got it and the time
// we actually call this, we need to return something random so that
// the next time we won't get 304
return Etag.InvalidEtag;
}
return indexEtag;
}
}
public virtual void AfterCommit(
string key,
RavenJObject document,
RavenJObject metadata,
Etag etag)
{
}
public void AfterDelete(string key, Etag deletedEtag)
{
foreach (var behavior in prefetchingBehaviors)
{
behavior.AfterDelete(key, deletedEtag);
}
}
public static HttpStatusCode PerformPost(string url, string payload, Etag requestEtag, out Etag responseEtag)
{
var request = WebRequest.Create(url);
if (requestEtag != null)
request.Headers.Add("If-None-Match", requestEtag.ToString());
request.Method = "POST";
byte[] buffer = Encoding.UTF8.GetBytes(payload);
request.GetRequestStream().Write(buffer, 0, buffer.Length);
using (var response = GetHttpResponseHandle304(request))
{
try
{
responseEtag = response.GetEtagHeader();
}
catch (Exception)
{
responseEtag = null;
}
return response.StatusCode;
}
}
public void AfterUpdate(string key, Etag etagBeforeUpdate)
{
foreach (var behavior in prefetchingBehaviors)
{
behavior.AfterUpdate(key, etagBeforeUpdate);
}
}
public RenameFileOperation(string name, string rename, Etag currentEtag, RavenJObject metadataAfterOperation)
{
Name = name;
Rename = rename;
Etag = currentEtag;
MetadataAfterOperation = metadataAfterOperation;
}
private void StreamToClient(Stream stream, int pageSize, Etag etag)
{
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(FileSystemsLandlord.SystemConfiguration.DatabaseOperationTimeout))
using (var writer = new JsonTextWriter(new StreamWriter(stream)))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
Storage.Batch(accessor =>
{
var files = accessor.GetFilesAfter(etag, pageSize);
foreach (var file in files)
{
timeout.Delay();
var doc = RavenJObject.FromObject(file);
doc.WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
}
});
writer.WriteEndArray();
writer.WriteEndObject();
writer.Flush();
}
}
public void SetCachedDocument(string key, Etag etag, RavenJObject doc, RavenJObject metadata, int size)
{
if (skipSettingDocumentInCache)
return;
var documentClone = ((RavenJObject)doc.CloneToken());
documentClone.EnsureCannotBeChangeAndEnableSnapshotting();
var metadataClone = ((RavenJObject)metadata.CloneToken());
metadataClone.EnsureCannotBeChangeAndEnableSnapshotting();
try
{
cachedSerializedDocuments.Set("Doc/" + key + "/" + etag, new CachedDocument
{
Document = documentClone,
Metadata = metadataClone,
Size = size
}, new CacheItemPolicy
{
SlidingExpiration = configuration.MemoryCacheExpiration,
});
}
catch (OverflowException)
{
// this is a bug in the framework
// http://connect.microsoft.com/VisualStudio/feedback/details/735033/memorycache-set-fails-with-overflowexception-exception-when-key-is-u7337-u7f01-u2117-exception-message-negating-the-minimum-value-of-a-twos-complement-number-is-invalid
// in this case, we just threat it as uncachable
}
}
public virtual void AfterPut(
string key,
RavenJObject document,
RavenJObject metadata,
Etag etag,
TransactionInformation transactionInformation)
{
}
public void DeleteAttachment(string key, Etag etag)
{
AssertValidEtag(key, etag, "DELETE");
if (!storage.Attachments.Remove(new RavenJObject { { "key", key } }))
throw new ConcurrencyException("DELETE attempted on attachment '" + key +
"' while it was locked by another transaction");
logger.Debug("Attachment with key '{0}' was deleted", key);
}
public override void AfterCommit(string key, RavenJObject document, RavenJObject metadata, Etag etag)
{
var entityName = metadata.Value<string>(Constants.RavenEntityName);
if (entityName == null)
return;
Database.LastCollectionEtags.Update(entityName, etag);
}
public PatchResultData ApplyPatch(string docId, Etag etag, PatchRequest[] patchDoc,
TransactionInformation transactionInformation, bool debugMode = false)
{
if (docId == null)
throw new ArgumentNullException("docId");
return ApplyPatchInternal(docId, etag, transactionInformation,
jsonDoc => new JsonPatcher(jsonDoc.ToJson()).Apply(patchDoc),
() => null, () => null, debugMode);
}
public SynchronizationBehavior(string fileName, Etag sourceFileEtag, RavenJObject sourceMetadata, FileSystemInfo sourceFs, SynchronizationType type, RavenFileSystem fs)
{
this.fileName = fileName;
this.sourceFileEtag = sourceFileEtag;
this.sourceMetadata = sourceMetadata;
this.sourceFs = sourceFs;
this.type = type;
this.fs = fs;
}
public override void DeleteDocumentInTransaction(
TransactionInformation transactionInformation,
string key,
Etag etag,
Etag committedEtag,
IUuidGenerator uuidGenerator)
{
throw new InvalidOperationException("DTC is not supported by " + storageName + " storage.");
}
public DeleteFileOperation(InMemoryFilesSessionOperations sessionOperations, string path, Etag etag)
{
if (string.IsNullOrWhiteSpace(path))
throw new ArgumentNullException("path", "The path cannot be null, empty or whitespace.");
this.sessionOperations = sessionOperations;
this.Filename = path;
this.Etag = etag;
}
public bool IsIndexStale(string name, DateTime? cutOff, Etag cutoffEtag)
{
Api.JetSetCurrentIndex(session, IndexesStats, "by_key");
Api.MakeKey(session, IndexesStats, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, IndexesStats, SeekGrbit.SeekEQ) == false)
{
return false;
}
Api.JetSetCurrentIndex(session, IndexesStatsReduce, "by_key");
Api.MakeKey(session, IndexesStatsReduce, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
var hasReduce = Api.TrySeek(session, IndexesStatsReduce, SeekGrbit.SeekEQ);
if (IsMapStale(name) || hasReduce && IsReduceStale(name))
{
if (cutOff != null)
{
var indexedTimestamp = Api.RetrieveColumnAsInt64(session, IndexesStats, tableColumnsCache.IndexesStatsColumns["last_indexed_timestamp"]).Value;
var lastIndexedTimestamp = DateTime.FromBinary(indexedTimestamp);
if (cutOff.Value >= lastIndexedTimestamp)
return true;
if (hasReduce)
{
var lastReduceIndex = Api.RetrieveColumnAsInt64(session, IndexesStatsReduce, tableColumnsCache.IndexesStatsReduceColumns["last_reduced_timestamp"]);
lastIndexedTimestamp = lastReduceIndex == null ? DateTime.MinValue : DateTime.FromBinary(lastReduceIndex.Value);
if (cutOff.Value >= lastIndexedTimestamp)
return true;
}
}
else if (cutoffEtag != null)
{
var lastIndexedEtag = Api.RetrieveColumn(session, IndexesStats,
tableColumnsCache.IndexesStatsColumns["last_indexed_etag"]);
if (Buffers.Compare(lastIndexedEtag, cutoffEtag.ToByteArray()) < 0)
return true;
}
else
{
return true;
}
}
Api.JetSetCurrentIndex(session, Tasks, "by_index");
Api.MakeKey(session, Tasks, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Tasks, SeekGrbit.SeekEQ) == false)
{
return false;
}
if (cutOff == null)
return true;
// we are at the first row for this index
var addedAt = Api.RetrieveColumnAsInt64(session, Tasks, tableColumnsCache.TasksColumns["added_at"]).Value;
return cutOff.Value >= DateTime.FromBinary(addedAt);
}
/// <summary>
/// Represents an replication operation to all destination servers of an item specified by ETag
/// </summary>
/// <param name="etag">ETag of an replicated item</param>
/// <param name="timeout">Optional timeout</param>
/// <param name="database">The database from which to check, if null, the default database for the document store connection string</param>
/// <param name="replicas">The min number of replicas that must have the value before we can return (or the number of destinations, if higher)</param>
/// <returns>Task which will have the number of nodes that the caught up to the specified etag</returns>
public async Task<int> WaitAsync(Etag etag = null, TimeSpan? timeout = null, string database = null, int replicas = 2)
{
etag = etag ?? documentStore.LastEtagHolder.GetLastWrittenEtag();
if (etag == Etag.Empty)
return replicas; // if the etag is empty, nothing to do
var asyncDatabaseCommands = documentStore.AsyncDatabaseCommands;
if (database != null)
asyncDatabaseCommands = asyncDatabaseCommands.ForDatabase(database);
asyncDatabaseCommands.ForceReadFromMaster();
var doc = await asyncDatabaseCommands.GetAsync("Raven/Replication/Destinations");
if (doc == null)
return -1;
var replicationDocument = doc.DataAsJson.JsonDeserialization<ReplicationDocument>();
if (replicationDocument == null)
return -1;
var destinationsToCheck = replicationDocument.Destinations
.Where(
x => x.Disabled == false && x.IgnoredClient == false)
.Select(x => x.ClientVisibleUrl ?? x.Url)
.ToList();
if (destinationsToCheck.Count == 0)
return 0;
int toCheck = Math.Min(replicas, destinationsToCheck.Count);
var countDown = new AsyncCountdownEvent(toCheck);
var errors = new BlockingCollection<Exception>();
foreach (var url in destinationsToCheck)
{
WaitForReplicationFromServerAsync(url, countDown, etag, errors);
}
if (await countDown.WaitAsync().WaitWithTimeout(timeout) == false)
{
throw new TimeoutException(
string.Format("Confirmed that the specified etag {0} was replicated to {1} of {2} servers, during {3}", etag,
(toCheck - countDown.Count),
toCheck,
timeout));
}
if (errors.Count > 0 && countDown.Count > 0)
throw new AggregateException(errors);
return countDown.Count;
}
protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn, Etag synchronizationEtag)
{
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, indexesToWorkOn,
HandleReduceForIndex);
}
private void BackgroundSqlReplication()
{
int workCounter = 0;
while (Database.WorkContext.DoWork)
{
var config = GetConfiguredReplicationDestinations();
if (config.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var localReplicationStatus = GetReplicationStatus();
var relevantConfigs = config.Where(x =>
{
if (x.Disabled)
{
return(false);
}
var sqlReplicationStatistics = statistics.GetOrDefault(x.Name);
if (sqlReplicationStatistics == null)
{
return(true);
}
return(SystemTime.UtcNow >= sqlReplicationStatistics.LastErrorTime);
}) // have error or the timeout expired
.ToList();
if (relevantConfigs.Count == 0)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var leastReplicatedEtag = GetLeastReplicatedEtag(relevantConfigs, localReplicationStatus);
if (leastReplicatedEtag == null)
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
continue;
}
var documents = prefetchingBehavior.GetDocumentsBatchFrom(leastReplicatedEtag);
Etag latestEtag = null, lastBatchEtag = null;
if (documents.Count != 0)
{
lastBatchEtag = documents[documents.Count - 1].Etag;
}
var replicationDuration = Stopwatch.StartNew();
documents.RemoveAll(x => x.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)); // we ignore system documents here
if (documents.Count != 0)
{
latestEtag = documents[documents.Count - 1].Etag;
}
var deletedDocsByConfig = new Dictionary <SqlReplicationConfig, List <ListItem> >();
foreach (var relevantConfig in relevantConfigs)
{
var cfg = relevantConfig;
Database.TransactionalStorage.Batch(accessor =>
{
deletedDocsByConfig[cfg] = accessor.Lists.Read(GetSqlReplicationDeletionName(cfg),
GetLastEtagFor(localReplicationStatus, cfg),
latestEtag,
1024)
.ToList();
});
}
// No documents AND there aren't any deletes to replicate
if (documents.Count == 0 && deletedDocsByConfig.Sum(x => x.Value.Count) == 0)
{
if (latestEtag != null)
{
// so we filtered some documents, let us update the etag about that.
foreach (var lastReplicatedEtag in localReplicationStatus.LastReplicatedEtags)
{
if (lastReplicatedEtag.LastDocEtag.CompareTo(latestEtag) <= 0)
{
lastReplicatedEtag.LastDocEtag = latestEtag;
}
}
latestEtag = Etag.Max(latestEtag, lastBatchEtag);
SaveNewReplicationStatus(localReplicationStatus, latestEtag);
}
else // no point in waiting if we just saved a new doc
{
Database.WorkContext.WaitForWork(TimeSpan.FromMinutes(10), ref workCounter, "Sql Replication");
}
continue;
}
var successes = new ConcurrentQueue <Tuple <SqlReplicationConfig, Etag> >();
try
{
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(Database.WorkContext, relevantConfigs, replicationConfig =>
{
try
{
var lastReplicatedEtag = GetLastEtagFor(localReplicationStatus, replicationConfig);
var deletedDocs = deletedDocsByConfig[replicationConfig];
var docsToReplicate = documents
.Where(x => lastReplicatedEtag.CompareTo(x.Etag) <= 0) // haven't replicate the etag yet
.ToList();
var currentLatestEtag = HandleDeletesAndChangesMerging(deletedDocs, docsToReplicate);
if (ReplicateDeletionsToDestination(replicationConfig, deletedDocs) &&
ReplicateChangesToDesintation(replicationConfig, docsToReplicate))
{
if (deletedDocs.Count > 0)
{
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.RemoveAllBefore(GetSqlReplicationDeletionName(replicationConfig), deletedDocs[deletedDocs.Count - 1].Etag));
}
successes.Enqueue(Tuple.Create(replicationConfig, currentLatestEtag));
}
}
catch (Exception e)
{
log.WarnException("Error while replication to SQL destination: " + replicationConfig.Name, e);
Database.AddAlert(new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Exception = e.ToString(),
Title = "Sql Replication failure to replication",
Message = "Sql Replication could not replicate to " + replicationConfig.Name,
UniqueKey = "Sql Replication could not replicate to " + replicationConfig.Name
});
}
});
if (successes.Count == 0)
{
continue;
}
foreach (var t in successes)
{
var cfg = t.Item1;
var currentLatestEtag = t.Item2;
var destEtag = localReplicationStatus.LastReplicatedEtags.FirstOrDefault(x => string.Equals(x.Name, cfg.Name, StringComparison.InvariantCultureIgnoreCase));
if (destEtag == null)
{
localReplicationStatus.LastReplicatedEtags.Add(new LastReplicatedEtag
{
Name = cfg.Name,
LastDocEtag = currentLatestEtag ?? Etag.Empty
});
}
else
{
destEtag.LastDocEtag = currentLatestEtag = currentLatestEtag ?? destEtag.LastDocEtag;
}
latestEtag = Etag.Max(latestEtag, currentLatestEtag);
}
latestEtag = Etag.Max(latestEtag, lastBatchEtag);
SaveNewReplicationStatus(localReplicationStatus, latestEtag);
}
finally
{
AfterReplicationCompleted(successes.Count);
var min = localReplicationStatus.LastReplicatedEtags.Min(x => new ComparableByteArray(x.LastDocEtag.ToByteArray()));
if (min != null)
{
var lastMinReplicatedEtag = min.ToEtag();
prefetchingBehavior.CleanupDocuments(lastMinReplicatedEtag);
prefetchingBehavior.UpdateAutoThrottler(documents, replicationDuration.Elapsed);
}
}
}
}
public IEnumerable <JsonDocument> GetDocumentsAfter(Etag etag, int take, long?maxSize = null, Etag untilEtag = null)
{
Api.JetSetCurrentIndex(session, Documents, "by_etag");
Api.MakeKey(session, Documents, etag.TransformToValueForEsentSorting(), MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Documents, SeekGrbit.SeekGT) == false)
{
yield break;
}
long totalSize = 0;
int count = 0;
do
{
if (untilEtag != null && count > 0)
{
var docEtag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"]));
if (EtagUtil.IsGreaterThanOrEqual(docEtag, untilEtag))
{
yield break;
}
}
var readCurrentDocument = ReadCurrentDocument();
totalSize += readCurrentDocument.SerializedSizeOnDisk;
if (maxSize != null && totalSize > maxSize.Value)
{
yield return(readCurrentDocument);
yield break;
}
yield return(readCurrentDocument);
count++;
} while (Api.TryMoveNext(session, Documents) && count < take);
}
public Etag GetDocumentsWithIdStartingWith(string idPrefix, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument)
{
Etag lastDocumentReadEtag = null;
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = actions.Documents.GetDocumentsAfterWithIdStartingWith(etag, idPrefix, pageSize, token, timeout: TimeSpan.FromSeconds(2), lastProcessedDocument: x => lastDocumentReadEtag = x);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
etag = doc.Etag;
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
bool canContinue = addDocument(document);
if (!canContinue)
{
break;
}
}
if (returnedDocs)
{
break;
}
// No document was found that matches the requested criteria
if (docCount == 0)
{
// If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
if (lastDocumentReadEtag != null)
{
etag = lastDocumentReadEtag;
}
break;
}
}
});
return(etag);
}
private async Task <HttpResponseMessage> GetQueriesResponse(bool isGet)
{
RavenJArray itemsToLoad;
if (isGet == false)
{
try
{
itemsToLoad = await ReadJsonArrayAsync().ConfigureAwait(false);
}
catch (InvalidOperationException e)
{
if (Log.IsDebugEnabled)
{
Log.DebugException("Failed to deserialize query request.", e);
}
return(GetMessageWithObject(new
{
Message = "Could not understand json, please check its validity."
}, (HttpStatusCode)422)); //http code 422 - Unprocessable entity
}
catch (InvalidDataException e)
{
if (Log.IsDebugEnabled)
{
Log.DebugException("Failed to deserialize query request.", e);
}
return(GetMessageWithObject(new
{
e.Message
}, (HttpStatusCode)422)); //http code 422 - Unprocessable entity
}
AddRequestTraceInfo(sb =>
{
foreach (var item in itemsToLoad)
{
sb.Append("\t").Append(item).AppendLine();
}
});
}
else
{
itemsToLoad = new RavenJArray(GetQueryStringValues("id").Cast <object>());
}
var result = new MultiLoadResult();
var loadedIds = new HashSet <string>();
var includedIds = new HashSet <string>();
var includes = GetQueryStringValues("include") ?? new string[0];
var transformer = GetQueryStringValue("transformer") ?? GetQueryStringValue("resultTransformer");
var transformerParameters = this.ExtractTransformerParameters();
var includedEtags = new List <byte>();
if (string.IsNullOrEmpty(transformer) == false)
{
var transformerDef = Database.IndexDefinitionStorage.GetTransformer(transformer);
if (transformerDef == null)
{
return(GetMessageWithObject(new { Error = "No such transformer: " + transformer }, HttpStatusCode.BadRequest));
}
includedEtags.AddRange(transformerDef.GetHashCodeBytes());
}
Database.TransactionalStorage.Batch(actions =>
{
foreach (RavenJToken item in itemsToLoad)
{
var value = item.Value <string>();
if (loadedIds.Add(value) == false)
{
continue;
}
var documentByKey = string.IsNullOrEmpty(transformer)
? Database.Documents.Get(value)
: Database.Documents.GetWithTransformer(value, transformer, transformerParameters, out includedIds);
if (documentByKey == null)
{
if (ClientIsV3OrHigher(Request))
{
result.Results.Add(null);
}
continue;
}
result.Results.Add(documentByKey.ToJson());
if (documentByKey.Etag != null)
{
includedEtags.AddRange(documentByKey.Etag.ToByteArray());
}
// TODO: Revise this.
includedEtags.Add((false) ? (byte)0 : (byte)1);
}
var addIncludesCommand = new AddIncludesCommand(Database, (etag, includedDoc) =>
{
includedEtags.AddRange(etag.ToByteArray());
result.Includes.Add(includedDoc);
}, includes, loadedIds);
foreach (var item in result.Results.Where(item => item != null))
{
addIncludesCommand.Execute(item);
}
});
foreach (var includedId in includedIds)
{
var doc = Database.Documents.Get(includedId);
if (doc == null)
{
continue;
}
includedEtags.AddRange(doc.Etag.ToByteArray());
result.Includes.Add(doc.ToJson());
}
var computeHash = Hashing.Metro128.Calculate(includedEtags.ToArray());
Etag computedEtag = Etag.FromHash(computeHash);
if (MatchEtag(computedEtag))
{
return(GetEmptyMessage(HttpStatusCode.NotModified));
}
var msg = GetMessageWithObject(result);
WriteETag(computedEtag, msg);
AddRequestTraceInfo(sb => sb.Append("Results count: {0}, includes count: {1}", result.Results.Count, result.Includes.Count).AppendLine());
return(msg);
}
protected override bool IsIndexStale(IndexStats indexesStat, Etag synchronizationEtag, IStorageActionsAccessor actions, bool isIdle, Reference <bool> onlyFoundIdleWork)
{
onlyFoundIdleWork.Value = false;
return(actions.Staleness.IsReduceStale(indexesStat.Name));
}
public bool Delete(string key, Etag etag, TransactionInformation transactionInformation, string[] participatingIds = null)
{
RavenJObject metadata;
return(Delete(key, etag, transactionInformation, out metadata, participatingIds));
}
public virtual async Task SideBySideExecuteAsync(IAsyncDatabaseCommands asyncDatabaseCommands, DocumentConvention documentConvention, Etag minimumEtagBeforeReplace = null, DateTime?replaceTimeUtc = null, CancellationToken token = default(CancellationToken))
{
Conventions = documentConvention;
var indexDefinition = CreateIndexDefinition();
var serverDef = await asyncDatabaseCommands.GetIndexAsync(IndexName, token).ConfigureAwait(false);
if (serverDef != null)
{
if (CurrentOrLegacyIndexDefinitionEquals(documentConvention, serverDef, indexDefinition))
{
return;
}
var replaceIndexName = "ReplacementOf/" + IndexName;
await asyncDatabaseCommands.PutIndexAsync(replaceIndexName, indexDefinition, token).ConfigureAwait(false);
await asyncDatabaseCommands
.PutAsync(Constants.IndexReplacePrefix + replaceIndexName,
null,
RavenJObject.FromObject(new IndexReplaceDocument {
IndexToReplace = serverDef.Name, MinimumEtagBeforeReplace = minimumEtagBeforeReplace, ReplaceTimeUtc = replaceTimeUtc
}),
new RavenJObject(),
token).ConfigureAwait(false);
}
else
{
// since index doesn't exist yet - create it in normal mode
await asyncDatabaseCommands.PutIndexAsync(IndexName, indexDefinition, token).ConfigureAwait(false);
}
}
protected bool MatchEtag(Etag etag)
{
return(EtagHeaderToEtag() == etag);
}
/// <summary>
/// Executes the index creation using in side-by-side mode.
/// </summary>
/// <param name="databaseCommands"></param>
/// <param name="documentConvention"></param>
public virtual void SideBySideExecute(IDatabaseCommands databaseCommands, DocumentConvention documentConvention, Etag minimumEtagBeforeReplace = null, DateTime?replaceTimeUtc = null)
{
Conventions = documentConvention;
var indexDefinition = CreateIndexDefinition();
var serverDef = databaseCommands.GetIndex(IndexName);
if (serverDef != null)
{
if (CurrentOrLegacyIndexDefinitionEquals(documentConvention, serverDef, indexDefinition))
{
return;
}
var replaceIndexName = "ReplacementOf/" + IndexName;
databaseCommands.PutIndex(replaceIndexName, indexDefinition);
databaseCommands
.Put(Constants.IndexReplacePrefix + replaceIndexName,
null,
RavenJObject.FromObject(new IndexReplaceDocument {
IndexToReplace = serverDef.Name, MinimumEtagBeforeReplace = minimumEtagBeforeReplace, ReplaceTimeUtc = replaceTimeUtc
}),
new RavenJObject());
}
else
{
// since index doesn't exist yet - create it in normal mode
databaseCommands.PutIndex(IndexName, indexDefinition);
}
}
/// <summary>
/// Executes the index creation against the specified document store in side-by-side mode.
/// </summary>
public Task SideBySideExecuteAsync(IDocumentStore store, Etag minimumEtagBeforeReplace = null, DateTime?replaceTimeUtc = null)
{
return(store.SideBySideExecuteIndexAsync(this, minimumEtagBeforeReplace, replaceTimeUtc));
}
/// <summary>
/// Executes the index creation against the specified document store in side-by-side mode.
/// </summary>
/// <param name="store"></param>
public void SideBySideExecute(IDocumentStore store, Etag minimumEtagBeforeReplace = null, DateTime?replaceTimeUtc = null)
{
store.SideBySideExecuteIndex(this, minimumEtagBeforeReplace, replaceTimeUtc);
}
public IEnumerable <JsonDocument> GetDocumentsAfter(Etag etag, int take, long?maxSize = null, Etag untilEtag = null,
/* this value is ignored in Munin */
TimeSpan?timeout = null)
{
var docs = storage.Documents["ByEtag"].SkipAfter(new RavenJObject {
{ "etag", etag.ToByteArray() }
})
.Select(result => DocumentByKey(result.Value <string>("key"), null))
.Take(take);
long totalSize = 0;
int count = 0;
foreach (var doc in docs)
{
totalSize += doc.SerializedSizeOnDisk;
if (maxSize != null && totalSize > maxSize.Value)
{
yield return(doc);
yield break;
}
if (untilEtag != null && count > 0)
{
if (EtagUtil.IsGreaterThanOrEqual(doc.Etag, untilEtag))
{
yield break;
}
}
count++;
yield return(doc);
}
}
public AddDocumentResult InsertDocument(string key, RavenJObject data, RavenJObject metadata, bool checkForUpdates)
{
var prep = JET_prep.Insert;
bool isUpdate = false;
Etag existingETag = null;
if (checkForUpdates)
{
Api.JetSetCurrentIndex(session, Documents, "by_key");
Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ);
if (isUpdate)
{
existingETag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"]));
prep = JET_prep.Replace;
}
}
using (var update = new Update(session, Documents, prep))
{
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode);
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"]))
{
if (isUpdate)
{
columnStream.SetLength(0);
}
using (Stream stream = new BufferedStream(columnStream))
using (var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current)))
{
data.WriteTo(finalStream);
finalStream.Flush();
}
}
Etag newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents);
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"], newEtag.TransformToValueForEsentSorting());
DateTime savedAt = SystemTime.UtcNow;
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt.ToBinary());
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"]))
{
if (isUpdate)
{
columnStream.SetLength(0);
}
using (Stream stream = new BufferedStream(columnStream))
{
metadata.WriteTo(stream);
stream.Flush();
}
}
update.Save();
return(new AddDocumentResult
{
Etag = newEtag,
PrevEtag = existingETag,
SavedAt = savedAt,
Updated = isUpdate
});
}
}
private async Task <Etag> ExportAttachments(ISmugglerDatabaseOperations exportOperations, ISmugglerDatabaseOperations importOperations, SmugglerDatabaseOptions databaseOptions)
{
Etag lastEtag = databaseOptions.StartAttachmentsEtag;
int totalCount = 0;
while (true)
{
try
{
if (databaseOptions.Limit - totalCount <= 0)
{
await importOperations.PutAttachment(null).ConfigureAwait(false); // force flush
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
var maxRecords = Math.Min(databaseOptions.Limit - totalCount, databaseOptions.BatchSize);
var attachments = await exportOperations.GetAttachments(totalCount, lastEtag, maxRecords).ConfigureAwait(false);
if (attachments.Count == 0)
{
var databaseStatistics = await exportOperations.GetStats().ConfigureAwait(false);
if (lastEtag == null)
{
lastEtag = Etag.Empty;
}
if (lastEtag.CompareTo(databaseStatistics.LastAttachmentEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}",
lastEtag);
continue;
}
ShowProgress("Done with reading attachments, total: {0}", totalCount);
return(lastEtag);
}
totalCount += attachments.Count;
ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Count, totalCount);
foreach (var attachment in attachments)
{
var attachmentData = await exportOperations.GetAttachmentData(attachment).ConfigureAwait(false);
if (attachmentData == null)
{
continue;
}
var attachmentToExport = new AttachmentExportInfo
{
Key = attachment.Key,
Metadata = attachment.Metadata,
Data = new MemoryStream(attachmentData)
};
if (databaseOptions.StripReplicationInformation)
{
attachmentToExport.Metadata = StripReplicationInformationFromMetadata(attachmentToExport.Metadata);
}
await importOperations.PutAttachment(attachmentToExport).ConfigureAwait(false);
lastEtag = attachment.Etag;
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
}
}
public IEnumerable <MappedResultInfo> GetItemsToReduce(GetItemsToReduceParams getItemsToReduceParams)
{
var scheduledReductionsByViewAndLevelAndReduceKey = tableStorage.ScheduledReductions.GetIndex(Tables.ScheduledReductions.Indices.ByViewAndLevelAndReduceKey);
var deleter = new ScheduledReductionDeleter(getItemsToReduceParams.ItemsToDelete, o =>
{
var json = o as RavenJObject;
if (json == null)
{
return(null);
}
var etag = Etag.Parse(json.Value <byte[]>("etag"));
return(etag.ToString());
});
var seenLocally = new HashSet <Tuple <string, int> >();
foreach (var reduceKey in getItemsToReduceParams.ReduceKeys.ToArray())
{
var reduceKeyHash = HashKey(reduceKey);
var viewAndLevelAndReduceKey = CreateKey(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKey, reduceKeyHash);
using (var iterator = scheduledReductionsByViewAndLevelAndReduceKey.MultiRead(Snapshot, viewAndLevelAndReduceKey))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
continue;
}
do
{
if (getItemsToReduceParams.Take <= 0)
{
break;
}
ushort version;
var value = LoadJson(tableStorage.ScheduledReductions, iterator.CurrentKey, writeBatch.Value, out version);
var reduceKeyFromDb = value.Value <string>("reduceKey");
var bucket = value.Value <int>("bucket");
var rowKey = Tuple.Create(reduceKeyFromDb, bucket);
var thisIsNewScheduledReductionRow = deleter.Delete(iterator.CurrentKey, value);
var neverSeenThisKeyAndBucket = getItemsToReduceParams.ItemsAlreadySeen.Add(rowKey);
if (thisIsNewScheduledReductionRow || neverSeenThisKeyAndBucket)
{
if (seenLocally.Add(rowKey))
{
foreach (var mappedResultInfo in GetResultsForBucket(getItemsToReduceParams.Index, getItemsToReduceParams.Level, reduceKeyFromDb, bucket, getItemsToReduceParams.LoadData))
{
getItemsToReduceParams.Take--;
yield return(mappedResultInfo);
}
}
}
if (getItemsToReduceParams.Take <= 0)
{
yield break;
}
}while (iterator.MoveNext());
}
getItemsToReduceParams.ReduceKeys.Remove(reduceKey);
if (getItemsToReduceParams.Take <= 0)
{
yield break;
}
}
}
protected override Etag CalculateSynchronizationEtag(Etag currentEtag, Etag lastProcessedEtag)
{
return(lastProcessedEtag);
}
public AddDocumentResult AddDocument(string key, Etag etag, RavenJObject data, RavenJObject metadata)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
var byteCount = Encoding.Unicode.GetByteCount(key);
if (byteCount >= 2048)
{
throw new ArgumentException(string.Format("The key must be a maximum of 2,048 bytes in Unicode, 1,024 characters, key is: '{0}'", key), "key");
}
try
{
Api.JetSetCurrentIndex(session, Documents, "by_key");
Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
var isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ);
Etag existingEtag = null;
if (isUpdate)
{
existingEtag = EnsureDocumentEtagMatch(key, etag, "PUT");
}
else
{
if (etag != null && etag != Etag.Empty) // expected something to be there.
{
throw new ConcurrencyException("PUT attempted on document '" + key +
"' using a non current etag (document deleted)")
{
ExpectedETag = etag
}
}
;
if (Api.TryMoveFirst(session, Details))
{
Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["document_count"], 1);
}
}
Etag newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents);
DateTime savedAt;
try
{
using (var update = new Update(session, Documents, isUpdate ? JET_prep.Replace : JET_prep.Insert))
{
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode);
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"]))
{
if (isUpdate)
{
columnStream.SetLength(0); // empty the existing value, since we are going to overwrite the entire thing
}
using (Stream stream = new BufferedStream(columnStream))
using (
var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current))
)
{
data.WriteTo(finalStream);
finalStream.Flush();
}
}
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"],
newEtag.TransformToValueForEsentSorting());
savedAt = SystemTime.UtcNow;
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt.ToBinary());
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"]))
{
if (isUpdate)
{
columnStream.SetLength(0);
}
using (Stream stream = new BufferedStream(columnStream))
{
metadata.WriteTo(stream);
stream.Flush();
}
}
update.Save();
}
}
catch (EsentErrorException e)
{
if (e.Error == JET_err.KeyDuplicate || e.Error == JET_err.WriteConflict)
{
throw new ConcurrencyException("PUT attempted on document '" + key + "' concurrently", e);
}
throw;
}
logger.Debug("Inserted a new document with key '{0}', update: {1}, ",
key, isUpdate);
cacher.RemoveCachedDocument(key, newEtag);
return(new AddDocumentResult
{
Etag = newEtag,
PrevEtag = existingEtag,
SavedAt = savedAt,
Updated = isUpdate
});
}
catch (EsentKeyDuplicateException e)
{
throw new ConcurrencyException("Illegal duplicate key " + key, e);
}
}
public bool Delete(string key, Etag etag, TransactionInformation transactionInformation, out RavenJObject metadata, string[] participatingIds = null)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
key = key.Trim();
var deleted = false;
Log.Debug("Delete a document with key: {0} and etag {1}", key, etag);
RavenJObject metadataVar = null;
using (Database.DocumentLock.Lock())
{
TransactionalStorage.Batch(actions =>
{
AssertDeleteOperationNotVetoed(key, transactionInformation);
if (transactionInformation == null)
{
Database.DeleteTriggers.Apply(trigger => trigger.OnDelete(key, null));
string collection = null;
Etag deletedETag;
if (actions.Documents.DeleteDocument(key, etag, out metadataVar, out deletedETag))
{
deleted = true;
actions.Indexing.RemoveAllDocumentReferencesFrom(key);
WorkContext.MarkDeleted(key);
Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, actions, participatingIds);
collection = metadataVar.Value <string>(Constants.RavenEntityName);
DeleteDocumentFromIndexesForCollection(key, collection, actions);
if (deletedETag != null)
{
Database.Prefetcher.AfterDelete(key, deletedETag);
}
Database.DeleteTriggers.Apply(trigger => trigger.AfterDelete(key, null));
}
TransactionalStorage
.ExecuteImmediatelyOrRegisterForSynchronization(() =>
{
Database.DeleteTriggers.Apply(trigger => trigger.AfterCommit(key));
if (string.IsNullOrEmpty(collection) == false)
{
Database.LastCollectionEtags.Update(collection);
}
Database.Notifications.RaiseNotifications(new DocumentChangeNotification
{
Id = key,
Type = DocumentChangeTypes.Delete,
TypeName = (metadataVar != null) ? metadataVar.Value <string>(Constants.RavenClrType) : null,
CollectionName = (metadataVar != null) ? metadataVar.Value <string>(Constants.RavenEntityName) : null
}, metadataVar);
});
}
else
{
var doc = actions.Documents.DocumentMetadataByKey(key);
Database.InFlightTransactionalState.DeleteDocumentInTransaction(transactionInformation, key,
etag,
doc == null ? Etag.Empty : doc.Etag,
UuidGenerator);
deleted = doc != null;
}
WorkContext.ShouldNotifyAboutWork(() => "DEL " + key);
});
metadata = metadataVar;
return(deleted);
}
}
private async Task <Etag> ExportDocuments(ISmugglerDatabaseOperations exportOperations, ISmugglerDatabaseOperations importOperations, SmugglerDatabaseOptions databaseOptions)
{
var now = SystemTime.UtcNow;
string lastEtag = databaseOptions.StartDocsEtag;
var totalCount = 0;
var numberOfSkippedDocs = 0;
var totalOfSkippedDocs = 0;
var lastForcedFlush = SystemTime.UtcNow;
var maxFlushInterval = TimeSpan.FromSeconds(1);
var lastReport = SystemTime.UtcNow;
var reportInterval = TimeSpan.FromSeconds(2);
ShowProgress("Exporting Documents");
while (true)
{
bool hasDocs = false;
try
{
var maxRecords = databaseOptions.Limit - totalCount;
if (maxRecords > 0)
{
var amountToFetchFromServer = Math.Min(databaseOptions.BatchSize, maxRecords);
using (var documents = await exportOperations.GetDocuments(lastEtag, amountToFetchFromServer).ConfigureAwait(false))
{
while (await documents.MoveNextAsync().ConfigureAwait(false))
{
if (numberOfSkippedDocs > 0 && (SystemTime.UtcNow - lastForcedFlush) > maxFlushInterval)
{
totalOfSkippedDocs += numberOfSkippedDocs;
ShowProgress("Skipped {0:#,#} documents", totalOfSkippedDocs);
lastForcedFlush = SystemTime.UtcNow;
numberOfSkippedDocs = 0;
}
hasDocs = true;
var document = documents.Current;
var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));
Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id")));
lastEtag = tempLastEtag;
if (!databaseOptions.MatchFilters(document))
{
numberOfSkippedDocs++;
continue;
}
if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
{
numberOfSkippedDocs++;
continue;
}
if (databaseOptions.StripReplicationInformation)
{
document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
}
if (databaseOptions.ShouldDisableVersioningBundle)
{
document["@metadata"] = DisableVersioning(document["@metadata"] as RavenJObject);
}
document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);
if (string.IsNullOrEmpty(databaseOptions.TransformScript) == false)
{
document = await exportOperations.TransformDocument(document, databaseOptions.TransformScript).ConfigureAwait(false);
}
if (document == null)
{
numberOfSkippedDocs++;
continue;
}
await importOperations.PutDocument(document, (int)DocumentHelpers.GetRoughSize(document)).ConfigureAwait(false);
totalCount++;
if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
{
ShowProgress("Exported {0} documents", totalCount);
lastReport = SystemTime.UtcNow;
}
}
}
if (hasDocs)
{
continue;
}
// The server can filter all the results. In this case, we need to try to go over with the next batch.
// Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
// (This code provides support for legacy RavenDB version: 1.0)
var databaseStatistics = await exportOperations.GetStats().ConfigureAwait(false);
var lastEtagComparable = new ComparableByteArray(lastEtag);
if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
{
lastEtag = EtagUtil.Increment(lastEtag, amountToFetchFromServer);
if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0)
{
lastEtag = databaseStatistics.LastDocEtag;
}
ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
continue;
}
}
}
catch (Exception e)
{
ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
throw new SmugglerExportException(e.Message, e)
{
LastEtag = lastEtag,
};
}
// Load HiLo documents for selected collections
databaseOptions.Filters.ForEach(filter =>
{
if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
{
filter.Values.ForEach(collectionName =>
{
JsonDocument doc = exportOperations.GetDocument("Raven/Hilo/" + collectionName);
if (doc != null)
{
doc.Metadata["@id"] = doc.Key;
var jsonDoc = doc.ToJson();
AsyncHelpers.RunSync(() => importOperations.PutDocument(jsonDoc, (int)DocumentHelpers.GetRoughSize(jsonDoc)));
totalCount++;
}
});
}
});
await importOperations.PutDocument(null, -1).ConfigureAwait(false); // force flush
ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
return(lastEtag);
}
}
public PutResult Put(string key, Etag etag, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation, string[] participatingIds = null)
{
WorkContext.MetricsCounters.DocsPerSecond.Mark();
key = string.IsNullOrWhiteSpace(key) ? Guid.NewGuid().ToString() : key.Trim();
RemoveReservedProperties(document);
RemoveMetadataReservedProperties(metadata);
var newEtag = Etag.Empty;
using (Database.DocumentLock.Lock())
{
TransactionalStorage.Batch(actions =>
{
if (key.EndsWith("/"))
{
key += GetNextIdentityValueWithoutOverwritingOnExistingDocuments(key, actions);
}
AssertPutOperationNotVetoed(key, metadata, document, transactionInformation);
if (transactionInformation == null)
{
if (Database.InFlightTransactionalState.IsModified(key))
{
throw new ConcurrencyException("PUT attempted on : " + key +
" while it is being locked by another transaction");
}
Database.PutTriggers.Apply(trigger => trigger.OnPut(key, document, metadata, null));
var addDocumentResult = actions.Documents.AddDocument(key, etag, document, metadata);
newEtag = addDocumentResult.Etag;
Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, actions, participatingIds);
metadata[Constants.LastModified] = addDocumentResult.SavedAt;
metadata.EnsureSnapshot(
"Metadata was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
document.EnsureSnapshot(
"Document was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?");
actions.AfterStorageCommitBeforeWorkNotifications(new JsonDocument
{
Metadata = metadata,
Key = key,
DataAsJson = document,
Etag = newEtag,
LastModified = addDocumentResult.SavedAt,
SkipDeleteFromIndex = addDocumentResult.Updated == false
}, documents =>
{
if (Database.IndexDefinitionStorage.IndexesCount == 0 || Database.WorkContext.RunIndexing == false)
{
return;
}
Database.Prefetcher.AfterStorageCommitBeforeWorkNotifications(PrefetchingUser.Indexer, documents);
});
Database.PutTriggers.Apply(trigger => trigger.AfterPut(key, document, metadata, newEtag, null));
TransactionalStorage
.ExecuteImmediatelyOrRegisterForSynchronization(() =>
{
Database.PutTriggers.Apply(trigger => trigger.AfterCommit(key, document, metadata, newEtag));
var newDocumentChangeNotification =
new DocumentChangeNotification
{
Id = key,
Type = DocumentChangeTypes.Put,
TypeName = metadata.Value <string>(Constants.RavenClrType),
CollectionName = metadata.Value <string>(Constants.RavenEntityName),
Etag = newEtag
};
Database.Notifications.RaiseNotifications(newDocumentChangeNotification, metadata);
});
WorkContext.ShouldNotifyAboutWork(() => "PUT " + key);
}
else
{
var doc = actions.Documents.DocumentMetadataByKey(key);
newEtag = Database.InFlightTransactionalState.AddDocumentInTransaction(key, etag, document, metadata,
transactionInformation,
doc == null
? Etag.Empty
: doc.Etag,
UuidGenerator);
}
});
Log.Debug("Put document {0} with etag {1}", key, newEtag);
return(new PutResult
{
Key = key,
ETag = newEtag
});
}
}
private void SaveNewReplicationStatus(SqlReplicationStatus localReplicationStatus, Etag latestEtag)
{
int retries = 5;
while (retries > 0)
{
retries--;
try
{
var obj = RavenJObject.FromObject(localReplicationStatus);
Database.Put(RavenSqlreplicationStatus, null, obj, new RavenJObject(), null);
lastLatestEtag = latestEtag;
break;
}
catch (ConcurrencyException)
{
Thread.Sleep(50);
}
}
}
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action <JsonDocument> addDocument)
{
return(GetDocuments(start, pageSize, etag, token, x => { addDocument(x); return true; }));
}
public Task <IAsyncEnumerator <StreamResult <T> > > StreamAsync <T>(Etag fromEtag, int start = 0, int pageSize = Int32.MaxValue, RavenPagingInformation pagingInformation = null)
{
throw new NotSupportedException("Streams are currently not supported by sharded document store");
}
public void Init()
{
highlightings = new Dictionary <string, Dictionary <string, string[]> >();
scoreExplanations = new Dictionary <string, string>();
Func <IndexQueryResult, object> tryRecordHighlightingAndScoreExplanation = queryResult =>
{
if (queryResult.Highligtings != null && (queryResult.Key != null || queryResult.HighlighterKey != null))
{
highlightings.Add(queryResult.Key ?? queryResult.HighlighterKey, queryResult.Highligtings);
}
if ((queryResult.Key != null || queryResult.ReduceVal != null) && queryResult.ScoreExplanation != null)
{
scoreExplanations.Add(queryResult.Key ?? queryResult.ReduceVal, queryResult.ScoreExplanation);
}
return(null);
};
stale = false;
indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
resultEtag = Etag.Empty;
nonAuthoritativeInformation = false;
if (string.IsNullOrEmpty(query.ResultsTransformer) == false &&
(query.FieldsToFetch == null || query.FieldsToFetch.Length == 0))
{
query.FieldsToFetch = new[] { Constants.AllFields };
}
duration = Stopwatch.StartNew();
idsToLoad = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
var index = database.IndexDefinitionStorage.GetIndexDefinition(indexName);
if (viewGenerator == null)
{
throw new IndexDoesNotExistsException("Could not find index named: " + indexName);
}
resultEtag = database.Indexes.GetIndexEtag(index.Name, null, query.ResultsTransformer);
stale = actions.Staleness.IsIndexStale(index.IndexId, query.Cutoff, query.CutoffEtag);
if (stale == false && query.Cutoff == null && query.CutoffEtag == null)
{
var indexInstance = database.IndexStorage.GetIndexInstance(indexName);
stale = stale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
}
if (stale &&
actions.Staleness.IsIndexStaleByTask(index.IndexId, query.Cutoff) == false &&
actions.Staleness.IsReduceStale(index.IndexId) == false)
{
var forEntityNames = viewGenerator.ForEntityNames.ToList();
var lastIndexedEtag = actions.Indexing.GetIndexStats(index.IndexId).LastIndexedEtag;
if (database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
{
stale = false;
}
}
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index.IndexId);
var indexFailureInformation = actions.Indexing.GetFailureRate(index.IndexId);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
docRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, query.TransformerParameters, idsToLoad);
var fieldsToFetch = new FieldsToFetch(query,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func <IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, index, fieldsToFetch, ShouldSkipDuplicateChecking);
var indexQueryResults = database.IndexStorage.Query(indexName, query, shouldIncludeInResults,
fieldsToFetch, database.IndexQueryTriggers, cancellationToken, (query.ShowTimings ? (Action <double>)(time => executionTimes[QueryTimings.Parse] = time) : null));
indexQueryResults = new ActiveEnumerable <IndexQueryResult>(indexQueryResults);
if (query.ShowTimings)
{
indexQueryResults = new TimedEnumerable <IndexQueryResult>(indexQueryResults, timeInMilliseconds => executionTimes[QueryTimings.Lucene] += timeInMilliseconds);
}
var docs = from queryResult in indexQueryResults
let doc = docRetriever.RetrieveDocumentForQuery(queryResult, index, fieldsToFetch, ShouldSkipDuplicateChecking)
where doc != null
let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false)
let __ = tryRecordHighlightingAndScoreExplanation(queryResult)
select doc;
transformerErrors = new List <string>();
results = database
.Queries
.GetQueryResults(query, viewGenerator, docRetriever, docs, transformerErrors,
timeInMilliseconds => executionTimes[QueryTimings.LoadDocuments] = timeInMilliseconds,
timeInMilliseconds => executionTimes[QueryTimings.TransformResults] = timeInMilliseconds,
query.ShowTimings, cancellationToken);
Header = new QueryHeaderInformation
{
Index = indexName,
IsStale = stale,
ResultEtag = resultEtag,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
TotalResults = query.TotalSize.Value
};
}
public virtual async Task <IAsyncEnumerator <FileHeader> > GetFiles(Etag lastEtag, int take)
{
ShowProgress("Streaming documents from {0}, batch size {1}", lastEtag, take);
return(await PrimaryStore.AsyncFilesCommands.StreamFileHeadersAsync(lastEtag, pageSize : take));
}
public T GetMergedTask <T>(List <int> indexesToSkip, int[] allIndexes, HashSet <IComparable> alreadySeen)
where T : DatabaseTask
{
var type = CreateKey(typeof(T).FullName);
var tasksByType = tableStorage.Tasks.GetIndex(Tables.Tasks.Indices.ByType);
using (var iterator = tasksByType.MultiRead(Snapshot, (Slice)type))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
return(null);
}
do
{
ushort version;
var value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null)
{
continue;
}
var currentId = Etag.Parse(value.ReadBytes(TaskFields.TaskId));
var indexId = value.ReadInt(TaskFields.IndexId);
if (indexesToSkip.Contains(indexId))
{
if (Logger.IsDebugEnabled)
{
Logger.Debug("Skipping task id: {0} for index id: {1}", currentId, indexId);
}
continue;
}
if (alreadySeen.Add(currentId) == false)
{
continue;
}
if (allIndexes.Contains(indexId) == false)
{
if (Logger.IsDebugEnabled)
{
Logger.Debug("Skipping task id: {0} for non existing index id: {0}", currentId, indexId);
}
continue;
}
DatabaseTask task;
try
{
task = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
}
catch (Exception e)
{
Logger.ErrorException(
string.Format("Could not create instance of a task: {0}", value), e);
alreadySeen.Add(currentId);
continue;
}
if (Logger.IsDebugEnabled)
{
Logger.Debug("Fetched task id: {0}", currentId);
}
task.Id = currentId;
MergeSimilarTasks(task, alreadySeen, indexesToSkip, allIndexes);
return((T)task);
} while (iterator.MoveNext());
}
return(null);
}
public override void AfterPut(string key, RavenJObject document, RavenJObject metadata, Etag etag, TransactionInformation transactionInformation)
{
VersioningConfiguration versioningConfiguration;
if (TryGetVersioningConfiguration(key, metadata, out versioningConfiguration) == false)
{
return;
}
using (Database.DisableAllTriggersForCurrentThread())
{
var copyMetadata = new RavenJObject(metadata);
copyMetadata[VersioningUtil.RavenDocumentRevisionStatus] = RavenJToken.FromObject("Historical");
copyMetadata[Constants.RavenReadOnly] = true;
copyMetadata.Remove(VersioningUtil.RavenDocumentRevision);
object parentRevision;
metadata.__ExternalState.TryGetValue("Parent-Revision", out parentRevision);
if (parentRevision != null)
{
copyMetadata[VersioningUtil.RavenDocumentParentRevision] = key + "/revisions/" + parentRevision;
}
object value;
metadata.__ExternalState.TryGetValue("Next-Revision", out value);
Database.Documents.Put(key + "/revisions/" + value, null, (RavenJObject)document.CreateSnapshot(), copyMetadata,
transactionInformation);
}
}
protected void WaitForReplication(IDocumentStore store, string id, string db = null, Etag changedSince = null)
{
for (int i = 0; i < RetriesCount; i++)
{
using (var session = store.OpenSession(db))
{
var e = session.Load <object>(id);
if (e == null)
{
if (changedSince != null)
{
if (session.Advanced.GetEtagFor(e) != changedSince)
{
break;
}
}
Thread.Sleep(100);
continue;
}
break;
}
}
}
private void MergeSimilarTasks(DatabaseTask task, HashSet <IComparable> alreadySeen, List <int> indexesToSkip, int[] allIndexes)
{
string tree;
Slice slice;
var type = task.GetType().FullName;
if (task.SeparateTasksByIndex)
{
tree = Tables.Tasks.Indices.ByIndexAndType;
slice = (Slice)CreateKey(task.Index, type);
}
else
{
tree = Tables.Tasks.Indices.ByType;
slice = (Slice)CreateKey(type);
}
using (var iterator = tableStorage.Tasks.GetIndex(tree).MultiRead(Snapshot, slice))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
return;
}
var totalKeysToProcess = task.NumberOfKeys;
do
{
if (totalKeysToProcess >= 5 * 1024)
{
break;
}
ushort version;
var value = LoadStruct(tableStorage.Tasks, iterator.CurrentKey, writeBatch.Value, out version);
if (value == null)
{
continue;
}
var currentId = Etag.Parse(iterator.CurrentKey.ToString());
var indexId = value.ReadInt(TaskFields.IndexId);
if (task.SeparateTasksByIndex == false && indexesToSkip.Contains(indexId))
{
//need to check this only when not separating tasks by index
if (Logger.IsDebugEnabled)
{
Logger.Debug("Skipping task id: {0} for index id: {1}", currentId, indexId);
}
continue;
}
if (alreadySeen.Add(currentId) == false)
{
continue;
}
if (task.SeparateTasksByIndex == false && allIndexes.Contains(indexId) == false)
{
//need to check this only when not separating tasks by index
if (Logger.IsDebugEnabled)
{
Logger.Debug("Skipping task id: {0} for non existing index id: {0}", currentId, indexId);
}
continue;
}
DatabaseTask existingTask;
try
{
existingTask = DatabaseTask.ToTask(value.ReadString(TaskFields.Type), value.ReadBytes(TaskFields.SerializedTask));
}
catch (Exception e)
{
Logger.ErrorException(string.Format("Could not create instance of a task: {0}", value), e);
alreadySeen.Add(currentId);
continue;
}
totalKeysToProcess += existingTask.NumberOfKeys;
task.Merge(existingTask);
if (Logger.IsDebugEnabled)
{
Logger.Debug("Merged task id: {0} with task id: {1}", currentId, task.Id);
}
} while (iterator.MoveNext());
}
}
public override void AfterPut(string key, RavenJObject document, RavenJObject metadata, Etag etag, TransactionInformation transactionInformation)
{
VersioningConfiguration versioningConfiguration;
if (TryGetVersioningConfiguration(key, metadata, out versioningConfiguration) == false)
return;
using (Database.DisableAllTriggersForCurrentThread())
{
var copyMetadata = new RavenJObject(metadata);
copyMetadata[VersioningUtil.RavenDocumentRevisionStatus] = RavenJToken.FromObject("Historical");
copyMetadata[Constants.RavenReadOnly] = true;
copyMetadata.Remove(VersioningUtil.RavenDocumentRevision);
object parentRevision;
metadata.__ExternalState.TryGetValue("Parent-Revision", out parentRevision);
if (parentRevision != null)
{
copyMetadata[VersioningUtil.RavenDocumentParentRevision] = key + "/revisions/" + parentRevision;
copyMetadata[VersioningUtil.RavenDocumentParentRevision] = key + "/revisions/" + parentRevision;
}
object value;
metadata.__ExternalState.TryGetValue("Next-Revision", out value);
Database.Put(key + "/revisions/" + value, null, (RavenJObject)document.CreateSnapshot(), copyMetadata,
transactionInformation);
}
}
