public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
{
VersioningConfiguration versioningConfiguration;
if (metadata.ContainsKey(Constants.RavenCreateVersion))
{
metadata.__ExternalState[Constants.RavenCreateVersion] = metadata[Constants.RavenCreateVersion];
metadata.Remove(Constants.RavenCreateVersion);
}
if (metadata.ContainsKey(Constants.RavenIgnoreVersioning))
{
metadata.__ExternalState[Constants.RavenIgnoreVersioning] = metadata[Constants.RavenIgnoreVersioning];
metadata.Remove(Constants.RavenIgnoreVersioning);
return;
}
if (TryGetVersioningConfiguration(key, metadata, out versioningConfiguration) == false)
return;
var revision = GetNextRevisionNumber(key);
using (Database.DisableAllTriggersForCurrentThread())
{
RemoveOldRevisions(key, revision, versioningConfiguration, transactionInformation);
}
metadata.__ExternalState["Next-Revision"] = revision;
metadata.__ExternalState["Parent-Revision"] = metadata.Value<string>(VersioningUtil.RavenDocumentRevision);
metadata[VersioningUtil.RavenDocumentRevisionStatus] = RavenJToken.FromObject("Current");
metadata[VersioningUtil.RavenDocumentRevision] = RavenJToken.FromObject(revision);
}
private static DateTimeOffset GetLastModified(RavenJObject metadata)
{
if (metadata.ContainsKey(Constants.LastModified))
return metadata.Value<DateTimeOffset>(Constants.LastModified);
if (metadata.ContainsKey(Constants.RavenLastModified))
return metadata.Value<DateTimeOffset>(Constants.RavenLastModified);
throw new InvalidOperationException("Could not find last modification date in metadata");
}
public static RavenJToken HandleConflictDocuments(RavenJObject metadata)
{
if (metadata == null)
return null;
if (metadata.ContainsKey(Constants.RavenReplicationConflictDocument))
metadata.Add(Constants.RavenReplicationConflictDocumentForcePut, true);
if (metadata.ContainsKey(Constants.RavenReplicationConflict))
metadata.Add(Constants.RavenReplicationConflictSkipResolution, true);
return metadata;
}
public override void OnPut(string name, RavenJObject headers)
{
if (headers.ContainsKey(Constants.RavenCreateVersion))
{
headers.__ExternalState[Constants.RavenCreateVersion] = headers[Constants.RavenCreateVersion];
headers.Remove(Constants.RavenCreateVersion);
}
FileSystem.Storage.Batch(accessor =>
{
VersioningConfiguration versioningConfiguration;
if (TryGetVersioningConfiguration(name, headers, accessor, out versioningConfiguration) == false)
return;
var revision = GetNextRevisionNumber(name, accessor);
using (FileSystem.DisableAllTriggersForCurrentThread())
{
RemoveOldRevisions(name, revision, versioningConfiguration);
}
headers.__ExternalState["Next-Revision"] = revision;
headers.__ExternalState["Parent-Revision"] = headers.Value<string>(VersioningUtil.RavenFileRevision);
headers[VersioningUtil.RavenFileRevisionStatus] = RavenJToken.FromObject("Current");
headers[VersioningUtil.RavenFileRevision] = RavenJToken.FromObject(revision);
});
}
public bool FilterDocuments(string destinationId, string key, RavenJObject metadata)
{
if (key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)) // don't replicate system docs
{
if (key.StartsWith("Raven/Hilo/", StringComparison.InvariantCultureIgnoreCase) == false) // except for hilo documents
return false;
}
if (metadata.ContainsKey(Constants.NotForReplication) && metadata.Value<bool>(Constants.NotForReplication)) // not explicitly marked to skip
return false;
if (metadata[Constants.RavenReplicationConflict] != null) // don't replicate conflicted documents, that just propagate the conflict
return false;
if (metadata.Value<string>(Constants.RavenReplicationSource) == destinationId) // prevent replicating back to source
return false;
switch (ReplicationOptionsBehavior)
{
case TransitiveReplicationOptions.None:
var value = metadata.Value<string>(Constants.RavenReplicationSource);
var replicateDoc = value == null || (value == CurrentDatabaseId);
return replicateDoc;
}
return true;
}
public SynchronizationWorkItem DetermineWork(string file, RavenJObject localMetadata, RavenJObject destinationMetadata, string localServerUrl, out NoSyncReason reason)
{
reason = NoSyncReason.Unknown;
if (localMetadata == null)
{
reason = NoSyncReason.SourceFileNotExist;
return null;
}
if (destinationMetadata != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflictResolution] == null)
{
reason = NoSyncReason.DestinationFileConflicted;
return null;
}
if (localMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null)
{
reason = NoSyncReason.SourceFileConflicted;
return null;
}
if (localMetadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
if (localMetadata.ContainsKey(SynchronizationConstants.RavenRenameFile))
{
var rename = localMetadata.Value<string>(SynchronizationConstants.RavenRenameFile);
if (destinationMetadata != null)
return new RenameWorkItem(file, rename, localServerUrl, storage);
return new ContentUpdateWorkItem(rename, localServerUrl, storage, sigGenerator);
// we have a rename tombstone but file does not exists on destination
}
return new DeleteWorkItem(file, localServerUrl, storage);
}
if (destinationMetadata != null && Historian.IsDirectChildOfCurrent(localMetadata, destinationMetadata))
{
reason = NoSyncReason.ContainedInDestinationHistory;
return null;
}
// file exists on dest and has the same content
if (destinationMetadata != null && localMetadata.Value<string>("Content-MD5") == destinationMetadata.Value<string>("Content-MD5"))
{
// check metadata to detect if any synchronization is needed
if (localMetadata.Keys.Except(new[] { Constants.MetadataEtagField, Constants.RavenLastModified, Constants.LastModified })
.Any(key => !destinationMetadata.ContainsKey(key) || localMetadata[key] != destinationMetadata[key]))
{
return new MetadataUpdateWorkItem(file, localServerUrl, destinationMetadata, storage);
}
reason = NoSyncReason.SameContentAndMetadata;
return null; // the same content and metadata - no need to synchronize
}
return new ContentUpdateWorkItem(file, localServerUrl, storage, sigGenerator);
}
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
{
if (!metadata.ContainsKey("IsRoutable") || !metadata.Value<bool>("IsRoutable")) {
return;
}
RavenJToken parentIdToken;
RavenJToken slugToken;
if (document.TryGetValue("ParentId", out parentIdToken) && document.TryGetValue("Slug", out slugToken)) {
var parentId = parentIdToken.Value<string>();
var slug = slugToken.Value<string>();
string parentPath = null;
if (!String.IsNullOrEmpty(parentId)) {
var parent = Database.Get(parentId, transactionInformation);
parentPath = parent.DataAsJson["Path"].Value<string>();
}
if (String.IsNullOrEmpty(parentPath)) {
document["Path"] = slug;
} else {
document["Path"] = parentPath + "/" + slug;
}
}
base.OnPut(key, document, metadata, transactionInformation);
}
protected void AssertLocalFileExistsAndIsNotConflicted(RavenJObject sourceMetadata)
{
if (sourceMetadata == null)
throw new SynchronizationException(string.Format("File {0} does not exist", FileName));
if (sourceMetadata.ContainsKey(SynchronizationConstants.RavenSynchronizationConflict))
throw new SynchronizationException(string.Format("File {0} is conflicted", FileName));
}
public override ReadVetoResult AllowRead(string key, RavenJObject metadata, ReadOperation operation, TransactionInformation transactionInformation)
{
if (operation != ReadOperation.Index)
return ReadVetoResult.Allowed;
if (metadata.ContainsKey("Deleted") == false)
return ReadVetoResult.Allowed;
return ReadVetoResult.Ignore;
}
public static ICommandData CreateCommand(RavenJObject jsonCommand, TransactionInformation transactionInformation)
{
var key = jsonCommand["Key"].Value<string>();
switch (jsonCommand.Value<string>("Method"))
{
case "PUT":
var putCommand = new PutCommandData
{
Key = key,
Etag = GetEtagFromCommand(jsonCommand),
Document = jsonCommand["Document"] as RavenJObject,
Metadata = jsonCommand["Metadata"] as RavenJObject,
TransactionInformation = transactionInformation
};
ValidateMetadataKeys(putCommand.Metadata);
return putCommand;
case "DELETE":
return new DeleteCommandData
{
Key = key,
Etag = GetEtagFromCommand(jsonCommand),
TransactionInformation = transactionInformation
};
case "PATCH":
return new PatchCommandData
{
Key = key,
Etag = GetEtagFromCommand(jsonCommand),
TransactionInformation = transactionInformation,
Metadata = jsonCommand["Metadata"] as RavenJObject,
Patches = jsonCommand
.Value<RavenJArray>("Patches")
.Cast<RavenJObject>()
.Select(PatchRequest.FromJson)
.ToArray(),
PatchesIfMissing = jsonCommand["PatchesIfMissing"] == null ? null : jsonCommand
.Value<RavenJArray>("PatchesIfMissing")
.Cast<RavenJObject>()
.Select(PatchRequest.FromJson)
.ToArray(),
SkipPatchIfEtagMismatch = jsonCommand.ContainsKey("SkipPatchIfEtagMismatch") && jsonCommand.Value<bool>("SkipPatchIfEtagMismatch")
};
case "EVAL":
var debug = jsonCommand["DebugMode"].Value<bool>();
return new ScriptedPatchCommandData
{
Key = key,
Metadata = jsonCommand["Metadata"] as RavenJObject,
Etag = GetEtagFromCommand(jsonCommand),
TransactionInformation = transactionInformation,
Patch = ScriptedPatchRequest.FromJson(jsonCommand.Value<RavenJObject>("Patch")),
PatchIfMissing = jsonCommand["PatchIfMissing"] == null ? null : ScriptedPatchRequest.FromJson(jsonCommand.Value<RavenJObject>("PatchIfMissing")),
DebugMode = debug
};
default:
throw new ArgumentException("Batching only supports PUT, PATCH, EVAL and DELETE.");
}
}
public static List<HistoryItem> DeserializeHistory(RavenJObject metadata)
{
var history = new List<HistoryItem>();
if (metadata.ContainsKey(SynchronizationConstants.RavenSynchronizationHistory))
{
var array = (RavenJArray) metadata[SynchronizationConstants.RavenSynchronizationHistory];
var items = array.Values<RavenJObject>().Select(x => JsonExtensions.JsonDeserialization<HistoryItem>(x));
return new List<HistoryItem>(items);
}
return history;
}
public void InitializeMetadata(string name, RavenJObject metadata)
{
if (metadata.ContainsKey(Constants.RavenCreateVersion))
{
metadata.__ExternalState[Constants.RavenCreateVersion] = metadata[Constants.RavenCreateVersion];
metadata.Remove(Constants.RavenCreateVersion);
}
if (metadata.ContainsKey(Constants.RavenIgnoreVersioning))
{
metadata.__ExternalState[Constants.RavenIgnoreVersioning] = metadata[Constants.RavenIgnoreVersioning];
metadata.Remove(Constants.RavenIgnoreVersioning);
return;
}
fileSystem.Storage.Batch(accessor =>
{
FileVersioningConfiguration versioningConfiguration;
if (TryGetVersioningConfiguration(name, metadata, accessor, out versioningConfiguration) == false)
return;
long revision;
if (metadata.__ExternalState.ContainsKey("Synchronization-Next-Revision"))
revision = (long) metadata.__ExternalState["Synchronization-Next-Revision"];
else
revision = GetNextRevisionNumber(name, accessor);
RemoveOldRevisions(name, revision, versioningConfiguration);
metadata.__ExternalState["Next-Revision"] = revision;
metadata.__ExternalState["Parent-Revision"] = metadata.Value<string>(VersioningUtil.RavenFileRevision);
metadata[VersioningUtil.RavenFileRevisionStatus] = RavenJToken.FromObject("Current");
metadata[VersioningUtil.RavenFileRevision] = RavenJToken.FromObject(revision);
});
}
public bool FilterDocuments(string destinationId, string key, RavenJObject metadata, out string reason)
{
if (IsSystemDocumentId(key))
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because it is a system document", key, destinationId);
Log.Debug(reason);
return false;
}
if (metadata.ContainsKey(Constants.NotForReplication) && metadata.Value<bool>(Constants.NotForReplication))
// not explicitly marked to skip
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because it was marked as not for replication", key, destinationId);
Log.Debug(reason);
return false;
}
if (metadata[Constants.RavenReplicationConflict] != null)
// don't replicate conflicted documents, that just propagate the conflict
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because it a conflict document", key, destinationId);
Log.Debug(reason);
return false;
}
if (OriginsFromDestination(destinationId, metadata)) // prevent replicating back to source
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because the destination server is the same server it originated from", key, destinationId);
Log.Debug(reason);
return false;
}
switch (ReplicationOptionsBehavior)
{
case TransitiveReplicationOptions.None:
var value = metadata.Value<string>(Constants.RavenReplicationSource);
if (value != null && (value != CurrentDatabaseId))
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because it was not created on the current server, and TransitiveReplicationOptions = none", key, destinationId);
Log.Debug(reason);
return false;
}
break;
}
reason = string.Format("Will replicate '{0}' to '{1}'", key, destinationId);
Log.Debug(reason);
return true;
}
public override void AfterPut(string key, RavenJObject document, RavenJObject metadata, Etag etag, TransactionInformation transactionInformation)
{
if (!metadata.__ExternalState.ContainsKey(RavenDocumentRevision))
return;
using (Database.DisableAllTriggersForCurrentThread())
{
var revisionMetadata = new RavenJObject(metadata);
revisionMetadata[RavenDocumentRevisionStatus] = RavenJToken.FromObject("Historical");
revisionMetadata.Remove(RavenDocumentRevision);
if (revisionMetadata.ContainsKey(RavenDocumentEnsureUniqueConstraints))
revisionMetadata.Remove(RavenDocumentEnsureUniqueConstraints);
var revisionCopy = new RavenJObject(document);
var revisionkey = key + RevisionSegment + metadata.__ExternalState[RavenDocumentRevision];
Database.Documents.Put(revisionkey, null, revisionCopy, revisionMetadata, transactionInformation);
}
}
public override void AfterPut(string key, RavenJObject document, RavenJObject metadata, Guid etag, TransactionInformation transactionInformation)
{
// leave raven system docs alone
if (key.StartsWith("Raven/"))
return;
// when there's already a Created date written, this is not the original insert
if (metadata.ContainsKey("Created"))
return;
// get the timestamp set for the last-modified date
var timestamp = metadata.Value<DateTime>(Constants.LastModified);
// copy the metadata and add the timestamp
var newMetadata = (RavenJObject)metadata.CreateSnapshot();
newMetadata.Add("Created", timestamp);
// update the metadata in the document
using (Database.DisableAllTriggersForCurrentThread())
Database.PutDocumentMetadata(key, newMetadata);
}
public bool FilterDocuments(string destinationId, string key, RavenJObject metadata)
{
if (IsSystemDocumentId(key))
return false;
if (metadata.ContainsKey(Constants.NotForReplication) && metadata.Value<bool>(Constants.NotForReplication)) // not explicitly marked to skip
return false;
if (metadata[Constants.RavenReplicationConflict] != null) // don't replicate conflicted documents, that just propagate the conflict
return false;
if (metadata.Value<string>(Constants.RavenReplicationSource) == destinationId) // prevent replicating back to source
return false;
switch (ReplicationOptionsBehavior)
{
case TransitiveReplicationOptions.None:
var value = metadata.Value<string>(Constants.RavenReplicationSource);
var replicateDoc = value == null || (value == CurrentDatabaseId);
return replicateDoc;
}
return true;
}
public override void OnPut(string key,
RavenJObject document,
RavenJObject metadata,
TransactionInformation transactionInformation)
{
Contract.Assume(!string.IsNullOrWhiteSpace(key));
RavenJToken versionToken;
if (!document.TryGetValue("Revision", out versionToken) || key.Contains(RevisionSegment))
return;
var newRevision = versionToken.Value<int>();
var currentRevision = metadata.ContainsKey(RavenDocumentRevision) ? metadata[RavenDocumentRevision].Value<int>() : 0;
metadata[RavenDocumentRevisionStatus] = RavenJToken.FromObject("Current");
//if we have a higher revision number than the existing then put a new revision
if (newRevision > currentRevision)
{
metadata[RavenDocumentRevision] = RavenJToken.FromObject(versionToken.Value<int>());
metadata.__ExternalState[RavenDocumentRevision] = metadata[RavenDocumentRevision];
}
}
public Item(ItemClass iClass, RavenJObject val)
{
Attributes = new Dictionary<string, List<float>>();
Mods = new List<ItemMod>();
Class = iClass;
Name = val["name"].Value<string>();
if (Name == "")
Name = val["typeLine"].Value<string>();
Type = val["typeLine"].Value<string>();
if (val.ContainsKey("properties"))
foreach (RavenJObject obj in (RavenJArray) val["properties"])
{
var values = new List<float>();
string s = "";
foreach (RavenJArray jva in (RavenJArray) obj["values"])
{
s += " " + jva[0].Value<string>();
}
s = s.TrimStart();
if (s == "")
{
Keywords = new List<string>();
string[] sl = obj["name"].Value<string>().Split(',');
foreach (string i in sl)
Keywords.Add(i.Trim());
continue;
}
foreach (Match m in numberfilter.Matches(s))
{
if (m.Value == "") values.Add(float.NaN);
else values.Add(float.Parse(m.Value, CultureInfo.InvariantCulture));
}
string cs = obj["name"].Value<string>() + ": " + (numberfilter.Replace(s, "#"));
Attributes.Add(cs, values);
}
if (val.ContainsKey("explicitMods"))
foreach (string s in val["explicitMods"].Values<string>())
{
List<ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter);
Mods.AddRange(mods);
}
if (val.ContainsKey("implicitMods"))
foreach (string s in val["implicitMods"].Values<string>())
{
List<ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter);
Mods.AddRange(mods);
}
if (val.ContainsKey("craftedMods"))
foreach (string s in val["craftedMods"].Values<string>())
{
List<ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter);
Mods.AddRange(mods);
}
if (iClass == ItemClass.Gem)
{
switch (val["colour"].Value<string>())
{
case "S":
Keywords.Add("Strength");
break;
case "D":
Keywords.Add("Dexterity");
break;
case "I":
Keywords.Add("Intelligence");
break;
}
}
else
{
Gems = new List<Item>();
}
var Sockets = new List<int>();
if (val.ContainsKey("sockets"))
foreach (RavenJObject obj in (RavenJArray) val["sockets"])
{
Sockets.Add(obj["group"].Value<int>());
}
if (val.ContainsKey("socketedItems"))
{
int socket = 0;
foreach (RavenJObject obj in (RavenJArray) val["socketedItems"])
{
var item = new Item(ItemClass.Gem, obj);
item.SocketGroup = Sockets[socket++];
Gems.Add(item);
}
}
}
//-------------------------------------------------------------------------------
public Item(ItemClass iClass, RavenJObject val, Dictionary<string, float> skillCastTimes)
{
this.skillCastTimes = skillCastTimes;
Mods = new List<Mod>();
Requirements = new Dictionary<string, List<float>>();
Properties = new Dictionary<string, List<float>>();
Class = iClass;
WeaponType = "";
Gems = new List<SkillGem>();
TypeLine = val["typeLine"].Value<string>();
Name = val["name"].Value<string>();
if (val.ContainsKey("icon"))
{
IconUrl = val["icon"].Value<string>();
}
if (val.ContainsKey("properties"))
{
if (Class == ItemClass.MainHand || Class == ItemClass.OffHand)
{
var nameObj = (((RavenJArray)val["properties"])[0] as RavenJObject)["name"];
WeaponType = nameObj != null ? nameObj.ToString() : "";
}
else if (Class == ItemClass.Gem)
{
var nameObj = (((RavenJArray)val["properties"])[0] as RavenJObject)["name"];
KeyWords = nameObj != null ? nameObj.ToString() : "";
}
ReadProperties(val);
}
else
if (Class == ItemClass.OffHand)
WeaponType = "Quiver";
if (val.ContainsKey("requirements") && Class == ItemClass.Gem)
{
foreach (RavenJObject obj in (RavenJArray)val["requirements"])
{
string name = obj["name"].Value<string>();
List<float> values = new List<float>();
foreach (RavenJArray jva in (RavenJArray)obj["values"])
{
values.Add(jva[0].Value<float>());
}
Requirements.Add(name, values);
}
}
//if item has explicit Mods we add it to it's Mods list
if (val.ContainsKey("explicitMods"))
foreach (string s in val["explicitMods"].Values<string>())
{
Mod modifier = new Mod(s.Replace("Additional ", ""), this.Class, WeaponType == "Quiver");
Mods.Add(modifier);
}
//if item has implicit Mods we add it to it's Mods list
if (val.ContainsKey("implicitMods"))
foreach (string s in val["implicitMods"].Values<string>())
{
Mod modifier = new Mod(s.Replace("Additional ", ""), this.Class, WeaponType == "Quiver");
Mods.Add(modifier);
}
if (val.ContainsKey("sockets"))
{
Sockets = new Dictionary<int, List<SocketColor>>();
ReadSockets(val);
}
if (val.ContainsKey("socketedItems"))
ReadGems(val);
if (val.ContainsKey("secDescrText"))
Description = val["secDescrText"].Value<string>();
if (val.ContainsKey("support"))
IsSupport = val["support"].Value<bool>();
}
public MetadataUpdateResult UpdateFileMetadata(string filename, RavenJObject metadata, Etag etag)
{
Api.JetSetCurrentIndex(session, Files, "by_name");
Api.MakeKey(session, Files, filename, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Files, SeekGrbit.SeekEQ) == false)
throw new FileNotFoundException(filename);
using (var update = new Update(session, Files, JET_prep.Replace))
{
var existingEtag = EnsureFileEtagMatch(filename, etag);
metadata.Remove(Constants.MetadataEtagField);
var existingMetadata = RetrieveMetadata();
if (!metadata.ContainsKey("Content-MD5") && existingMetadata.ContainsKey("Content-MD5"))
metadata["Content-MD5"] = existingMetadata["Content-MD5"];
if (!metadata.ContainsKey(Constants.FileSystem.RavenFsSize) && existingMetadata.ContainsKey(Constants.FileSystem.RavenFsSize))
metadata[Constants.FileSystem.RavenFsSize] = existingMetadata[Constants.FileSystem.RavenFsSize];
var newEtag = uuidGenerator.CreateSequentialUuid();
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["etag"], newEtag.TransformToValueForEsentSorting());
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["metadata"], ToQueryString(metadata), Encoding.Unicode);
update.Save();
return new MetadataUpdateResult
{
PrevEtag = existingEtag,
Etag = newEtag
};
}
}
public FileUpdateResult UpdateFileMetadata(string filename, RavenJObject metadata, Etag etag)
{
var key = CreateKey(filename);
var keySlice = (Slice)key;
ushort version;
var file = LoadJson(storage.Files, keySlice, writeBatch.Value, out version);
if (file == null)
throw new FileNotFoundException(filename);
var existingEtag = EnsureDocumentEtagMatch(filename, etag, file);
var newEtag = uuidGenerator.CreateSequentialUuid();
metadata.Remove(RavenConstants.MetadataEtagField);
var existingMetadata = (RavenJObject) file["metadata"];
if (!metadata.ContainsKey("Content-MD5") && existingMetadata.ContainsKey("Content-MD5"))
metadata["Content-MD5"] = existingMetadata["Content-MD5"];
if (!metadata.ContainsKey(RavenConstants.FileSystem.RavenFsSize) && existingMetadata.ContainsKey(RavenConstants.FileSystem.RavenFsSize))
metadata[RavenConstants.FileSystem.RavenFsSize] = existingMetadata[RavenConstants.FileSystem.RavenFsSize];
file["etag"] = newEtag.ToByteArray();
file["metadata"] = metadata;
storage.Files.Add(writeBatch.Value, keySlice, file, version);
var filesByEtag = storage.Files.GetIndex(Tables.Files.Indices.ByEtag);
filesByEtag.Delete(writeBatch.Value, CreateKey(existingEtag));
filesByEtag.Add(writeBatch.Value, (Slice)CreateKey(newEtag), key);
return new FileUpdateResult()
{
PrevEtag = existingEtag,
Etag = newEtag
};
}
public async Task PutAsync(string name, Etag etag, RavenJObject metadata, Func <Task <Stream> > streamAsync, PutOperationOptions options)
{
FileUpdateResult putResult = null;
try
{
FileSystem.MetricsCounters.FilesPerSecond.Mark();
name = FileHeader.Canonize(name);
if (options.PreserveTimestamps)
{
if (!metadata.ContainsKey(Constants.RavenCreationDate))
{
if (metadata.ContainsKey(Constants.CreationDate))
{
metadata[Constants.RavenCreationDate] = metadata[Constants.CreationDate];
}
else
{
throw new InvalidOperationException("Preserve Timestamps requires that the client includes the Raven-Creation-Date header.");
}
}
Historian.UpdateLastModified(metadata, options.LastModified.HasValue ? options.LastModified.Value : DateTimeOffset.UtcNow);
}
else
{
metadata[Constants.RavenCreationDate] = DateTimeOffset.UtcNow;
Historian.UpdateLastModified(metadata);
}
// TODO: To keep current filesystems working. We should remove when adding a new migration.
metadata[Constants.CreationDate] = metadata[Constants.RavenCreationDate].Value <DateTimeOffset>().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture);
Historian.Update(name, metadata);
long?size = -1;
Storage.Batch(accessor =>
{
FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name);
AssertPutOperationNotVetoed(name, metadata);
SynchronizationTask.Cancel(name);
var contentLength = options.ContentLength;
var contentSize = options.ContentSize;
if (contentLength == 0 || contentSize.HasValue == false)
{
size = contentLength;
if (options.TransferEncodingChunked)
{
size = null;
}
}
else
{
size = contentSize;
}
FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(name, metadata));
using (FileSystem.DisableAllTriggersForCurrentThread())
{
IndicateFileToDelete(name, etag);
}
putResult = accessor.PutFile(name, size, metadata);
FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(name, size, metadata));
Search.Index(name, metadata, putResult.Etag);
});
Log.Debug("Inserted a new file '{0}' with ETag {1}", name, putResult.Etag);
using (var contentStream = await streamAsync().ConfigureAwait(false))
using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, FileSystem.PutTriggers, contentStream, name, metadata))
{
await readFileToDatabase.Execute().ConfigureAwait(false);
if (size != null && readFileToDatabase.TotalSizeRead != size)
{
throw new HttpResponseException(HttpStatusCode.BadRequest);
}
if (options.PreserveTimestamps == false)
{
Historian.UpdateLastModified(metadata); // update with the final file size.
}
Log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name);
metadata["Content-MD5"] = readFileToDatabase.FileHash;
FileUpdateResult updateMetadata = null;
Storage.Batch(accessor => updateMetadata = accessor.UpdateFileMetadata(name, metadata, null));
long totalSizeRead = readFileToDatabase.TotalSizeRead;
metadata["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture);
Search.Index(name, metadata, updateMetadata.Etag);
Publisher.Publish(new FileChangeNotification {
Action = FileChangeAction.Add, File = name
});
Log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, updateMetadata.Etag);
}
}
catch (Exception ex)
{
if (putResult != null)
{
using (FileSystem.DisableAllTriggersForCurrentThread())
{
IndicateFileToDelete(name, null);
}
}
Log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex);
throw;
}
}
internal static void EnsureReplicationInformationInMetadata(RavenJObject metadata, DocumentDatabase database)
{
Debug.Assert(database != null);
if (metadata == null)
return;
if (metadata.ContainsKey(Constants.RavenReplicationSource))
return;
metadata[Constants.RavenReplicationHistory] = new RavenJArray();
metadata[Constants.RavenReplicationVersion] = 0;
metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(database.TransactionalStorage.Id);
}
public async Task PutAsync(string name, Etag etag, RavenJObject metadata, Func<Task<Stream>> streamAsync, PutOperationOptions options)
{
FileUpdateResult putResult = null;
try
{
FileSystem.MetricsCounters.FilesPerSecond.Mark();
name = FileHeader.Canonize(name);
if (options.PreserveTimestamps)
{
if (!metadata.ContainsKey(Constants.RavenCreationDate))
{
if (metadata.ContainsKey(Constants.CreationDate))
metadata[Constants.RavenCreationDate] = metadata[Constants.CreationDate];
else
throw new InvalidOperationException("Preserve Timestamps requires that the client includes the Raven-Creation-Date header.");
}
Historian.UpdateLastModified(metadata, options.LastModified.HasValue ? options.LastModified.Value : DateTimeOffset.UtcNow);
}
else
{
metadata[Constants.RavenCreationDate] = DateTimeOffset.UtcNow;
Historian.UpdateLastModified(metadata);
}
// TODO: To keep current filesystems working. We should remove when adding a new migration.
metadata[Constants.CreationDate] = metadata[Constants.RavenCreationDate].Value<DateTimeOffset>().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture);
Historian.Update(name, metadata);
long? size = -1;
Storage.Batch(accessor =>
{
FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name);
AssertPutOperationNotVetoed(name, metadata);
SynchronizationTask.Cancel(name);
var contentLength = options.ContentLength;
var contentSize = options.ContentSize;
if (contentLength == 0 || contentSize.HasValue == false)
{
size = contentLength;
if (options.TransferEncodingChunked)
size = null;
}
else
{
size = contentSize;
}
FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(name, metadata));
using (FileSystem.DisableAllTriggersForCurrentThread())
{
IndicateFileToDelete(name, etag);
}
putResult = accessor.PutFile(name, size, metadata);
FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(name, size, metadata));
Search.Index(name, metadata, putResult.Etag);
});
Log.Debug("Inserted a new file '{0}' with ETag {1}", name, putResult.Etag);
using (var contentStream = await streamAsync().ConfigureAwait(false))
using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, FileSystem.PutTriggers, contentStream, name, metadata))
{
await readFileToDatabase.Execute().ConfigureAwait(false);
if (size != null && readFileToDatabase.TotalSizeRead != size)
{
throw new HttpResponseException(HttpStatusCode.BadRequest);
}
if (options.PreserveTimestamps == false)
Historian.UpdateLastModified(metadata); // update with the final file size.
Log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name);
metadata["Content-MD5"] = readFileToDatabase.FileHash;
FileUpdateResult updateMetadata = null;
Storage.Batch(accessor => updateMetadata = accessor.UpdateFileMetadata(name, metadata, null));
long totalSizeRead = readFileToDatabase.TotalSizeRead;
metadata["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture);
Search.Index(name, metadata, updateMetadata.Etag);
Publisher.Publish(new FileChangeNotification { Action = FileChangeAction.Add, File = name });
Log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, updateMetadata.Etag);
}
}
catch (Exception ex)
{
if (putResult != null)
{
using (FileSystem.DisableAllTriggersForCurrentThread())
{
IndicateFileToDelete(name, null);
}
}
Log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex);
throw;
}
}
public void PutFile(string filename, long? totalSize, RavenJObject metadata, bool tombstone = false)
{
using (var update = new Update(session, Files, JET_prep.Insert))
{
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["name"], filename, Encoding.Unicode);
if (totalSize != null)
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["total_size"], BitConverter.GetBytes(totalSize.Value));
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["uploaded_size"], BitConverter.GetBytes(0));
if (!metadata.ContainsKey("ETag"))
throw new InvalidOperationException(string.Format("Metadata of file {0} does not contain 'ETag' key", filename));
var innerEsentMetadata = new RavenJObject(metadata);
var etag = innerEsentMetadata.Value<Guid>("ETag");
innerEsentMetadata.Remove("ETag");
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["etag"], etag.TransformToValueForEsentSorting());
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["metadata"], ToQueryString(innerEsentMetadata), Encoding.Unicode);
update.Save();
}
if (!tombstone)
{
if (Api.TryMoveFirst(session, Details) == false)
throw new InvalidOperationException("Could not find system metadata row");
Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["file_count"], 1);
}
}
public bool FilterDocuments(string destinationId, string key, RavenJObject metadata, out string reason)
{
if (IsSystemDocumentId(key))
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because it is a system document", key, destinationId);
if (Log.IsDebugEnabled)
{
Log.Debug(reason);
}
return(false);
}
if (metadata.ContainsKey(Constants.NotForReplication) && metadata.Value <bool>(Constants.NotForReplication))
// not explicitly marked to skip
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because it was marked as not for replication", key, destinationId);
if (Log.IsDebugEnabled)
{
Log.Debug(reason);
}
return(false);
}
if (metadata[Constants.RavenReplicationConflict] != null)
// don't replicate conflicted documents, that just propagate the conflict
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because it a conflict document", key, destinationId);
if (Log.IsDebugEnabled)
{
Log.Debug(reason);
}
return(false);
}
if (OriginsFromDestination(destinationId, metadata)) // prevent replicating back to source
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because the destination server is the same server it originated from", key, destinationId);
if (Log.IsDebugEnabled)
{
Log.Debug(reason);
}
return(false);
}
switch (ReplicationOptionsBehavior)
{
case TransitiveReplicationOptions.None:
var value = metadata.Value <string>(Constants.RavenReplicationSource);
if (value != null && (value != CurrentDatabaseId))
{
reason = string.Format("Will not replicate document '{0}' to '{1}' because it was not created on the current server, and TransitiveReplicationOptions = none", key, destinationId);
if (Log.IsDebugEnabled)
{
Log.Debug(reason);
}
return(false);
}
break;
}
reason = string.Format("Will replicate '{0}' to '{1}'", key, destinationId);
if (Log.IsDebugEnabled)
{
Log.Debug(reason);
}
return(true);
}
public SynchronizationWorkItem DetermineWork(string file, RavenJObject localMetadata, RavenJObject destinationMetadata, string localServerUrl, out NoSyncReason reason)
{
reason = NoSyncReason.Unknown;
if (localMetadata == null)
{
reason = NoSyncReason.SourceFileNotExist;
return(null);
}
if (destinationMetadata != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflictResolution] == null)
{
reason = NoSyncReason.DestinationFileConflicted;
return(null);
}
if (localMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null)
{
reason = NoSyncReason.SourceFileConflicted;
return(null);
}
if (localMetadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
if (localMetadata.ContainsKey(SynchronizationConstants.RavenRenameFile))
{
var rename = localMetadata.Value <string>(SynchronizationConstants.RavenRenameFile);
if (destinationMetadata != null)
{
return(new RenameWorkItem(file, rename, localServerUrl, storage));
}
return(new ContentUpdateWorkItem(rename, localServerUrl, storage, sigGenerator));
// we have a rename tombstone but file does not exists on destination
}
return(new DeleteWorkItem(file, localServerUrl, storage));
}
if (destinationMetadata != null && Historian.IsDirectChildOfCurrent(localMetadata, destinationMetadata))
{
reason = NoSyncReason.ContainedInDestinationHistory;
return(null);
}
// file exists on dest and has the same content
if (destinationMetadata != null && localMetadata.Value <string>("Content-MD5") == destinationMetadata.Value <string>("Content-MD5"))
{
// check metadata to detect if any synchronization is needed
if (localMetadata.Keys.Except(new[] { Constants.MetadataEtagField, Constants.RavenLastModified, Constants.LastModified })
.Any(key => !destinationMetadata.ContainsKey(key) || localMetadata[key] != destinationMetadata[key]))
{
return(new MetadataUpdateWorkItem(file, localServerUrl, destinationMetadata, storage));
}
reason = NoSyncReason.SameContentAndMetadata;
return(null); // the same content and metadata - no need to synchronize
}
return(new ContentUpdateWorkItem(file, localServerUrl, storage, sigGenerator));
}
protected void WriteHeaders(RavenJObject headers, Etag etag, HttpResponseMessage msg)
{
foreach (var header in headers)
{
if (header.Key.StartsWith("@"))
{
continue;
}
switch (header.Key)
{
case "Content-Type":
var headerValue = header.Value.Value <string>();
string charset = null;
if (headerValue.Contains("charset="))
{
var splits = headerValue.Split(';');
headerValue = splits[0];
charset = splits[1].Split('=')[1];
}
msg.Content.Headers.ContentType = new MediaTypeHeaderValue(headerValue)
{
CharSet = charset
};
break;
default:
if (header.Value.Type == JTokenType.Date)
{
if (header.Key.StartsWith("Raven-"))
{
var iso8601 = GetDateString(header.Value, "o");
msg.Content.Headers.Add(header.Key, iso8601);
}
else
{
var rfc1123 = GetDateString(header.Value, "r");
msg.Content.Headers.Add(header.Key, rfc1123);
if (!headers.ContainsKey("Raven-" + header.Key))
{
var iso8601 = GetDateString(header.Value, "o");
msg.Content.Headers.Add("Raven-" + header.Key, iso8601);
}
}
}
else if (header.Value.Type == JTokenType.Boolean)
{
msg.Content.Headers.Add(header.Key, header.Value.ToString());
}
else
{
//headers do not need url decoding because they might contain special symbols (like + symbol in clr type)
var value = UnescapeStringIfNeeded(header.Value.ToString(Formatting.None), shouldDecodeUrl: false);
msg.Content.Headers.Add(header.Key, value);
}
break;
}
}
if (headers["@Http-Status-Code"] != null)
{
msg.StatusCode = (HttpStatusCode)headers.Value <int>("@Http-Status-Code");
msg.Content.Headers.Add("Temp-Status-Description", headers.Value <string>("@Http-Status-Description"));
}
WriteETag(etag, msg);
}
public Item(ItemClass iClass, RavenJObject val)
{
Type = "";
Attributes = new Dictionary<string, List<float>>();
Mods = new List<Mod>();
Class = iClass;
if (iClass != ItemClass.Gem)
{
Gems = new List<Item>();
}
Name = val["typeLine"].Value<string>();
if (val.ContainsKey("properties"))
foreach (RavenJObject obj in (RavenJArray)val["properties"])
{
List<float> values = new List<float>();
string s = "";
foreach (RavenJArray jva in (RavenJArray)obj["values"])
{
s += " "+jva[0].Value<string>() ;
}
if (s == "") continue;
foreach (Match m in numberfilter.Matches(s))
{
if (m.Value == "") values.Add(float.NaN);
else values.Add(float.Parse(m.Value, System.Globalization.CultureInfo.InvariantCulture));
}
string cs = obj["name"].Value<string>() + ": " + (numberfilter.Replace(s, "#"));
Attributes.Add(cs, values);
}
if (val.ContainsKey("explicitMods"))
foreach (string s in val["explicitMods"].Values<string>())
{
var mods = Mod.CreateMods(s.Replace("Additional ", ""), this.Class);
Mods.AddRange(mods);
}
if (val.ContainsKey("implicitMods"))
foreach (string s in val["implicitMods"].Values<string>())
{
var mods = Mod.CreateMods(s.Replace("Additional ", ""), this.Class);
Mods.AddRange(mods);
}
}
private void ReplicateDelete(string id, RavenJObject metadata, TExternal incoming)
{
TInternal existingItem;
Guid existingEtag;
var existingMetadata = TryGetExisting(id, out existingItem, out existingEtag);
if (existingMetadata == null)
{
log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring", id, Src);
return;
}
if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict)) // locally conflicted
{
log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src);
var savedConflictedItemId = SaveConflictedItem(id, metadata, incoming, existingEtag);
AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem);
return;
}
if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker)) //deleted locally as well
{
log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories", id, Src);
var existingHistory = existingMetadata.Value <RavenJArray>(Constants.RavenReplicationHistory) ?? new RavenJArray();
var newHistory = metadata.Value <RavenJArray>(Constants.RavenReplicationHistory) ?? new RavenJArray();
foreach (var item in newHistory)
{
existingHistory.Add(item);
}
if (metadata.ContainsKey(Constants.RavenReplicationVersion) &&
metadata.ContainsKey(Constants.RavenReplicationSource))
{
existingHistory.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, metadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, metadata[Constants.RavenReplicationSource] }
});
}
while (existingHistory.Length > Constants.ChangeHistoryLength)
{
existingHistory.RemoveAt(0);
}
MarkAsDeleted(id, metadata);
return;
}
if (IsDirectChildOfCurrent(metadata, existingMetadata)) // not modified
{
log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src);
DeleteItem(id, existingEtag);
MarkAsDeleted(id, metadata);
return;
}
Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSyncronization(() =>
Database.RaiseNotifications(new DocumentChangeNotification
{
Name = id,
Type = DocumentChangeTypes.ReplicationConflict
}));
var newConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag);
log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src);
// we have a new conflict move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag);
CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata);
}
public void Replicate(string id, RavenJObject metadata, TExternal incoming)
{
if (metadata.Value <bool>(Constants.RavenDeleteMarker))
{
ReplicateDelete(id, metadata, incoming);
return;
}
TInternal existingItem;
Etag existingEtag;
bool deleted;
RavenJObject existingMetadata;
try
{
existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted);
}
catch (Exception e)
{
log.ErrorException(string.Format("Replication - fetching existing item failed. (key = {0})", id), e);
throw new InvalidOperationException("Replication - fetching existing item failed. (key = " + id + ")", e);
}
if (existingMetadata == null)
{
AddWithoutConflict(id, null, metadata, incoming);
if (log.IsDebugEnabled)
{
log.Debug("New item {0} replicated successfully from {1}", id, Src);
}
return;
}
// we just got the same version from the same source - request playback again?
// at any rate, not an error, moving on
if (existingMetadata.Value <string>(Constants.RavenReplicationSource) ==
metadata.Value <string>(Constants.RavenReplicationSource)
&&
existingMetadata.Value <long>(Constants.RavenReplicationVersion) ==
metadata.Value <long>(Constants.RavenReplicationVersion))
{
return;
}
var existingDocumentIsInConflict = existingMetadata[Constants.RavenReplicationConflict] != null;
if (existingDocumentIsInConflict == false &&
// if the current document is not in conflict, we can continue without having to keep conflict semantics
(Historian.IsDirectChildOfCurrent(metadata, existingMetadata)))
// this update is direct child of the existing doc, so we are fine with overwriting this
{
var etag = deleted == false ? existingEtag : null;
AddWithoutConflict(id, etag, metadata, incoming);
if (log.IsDebugEnabled)
{
log.Debug("Existing item {0} replicated successfully from {1}", id, Src);
}
return;
}
// this is the case where the incoming metadata is an older version of the metadata we have.
// this can happen when multiple sources send data with diffrent latencies
if (existingDocumentIsInConflict == false && Historian.IsDirectChildOfCurrent(existingMetadata, metadata))
{
return;
}
RavenJObject resolvedMetadataToSave;
TExternal resolvedItemToSave;
if (TryResolveConflict(id, metadata, incoming, existingItem, out resolvedMetadataToSave, out resolvedItemToSave))
{
if (metadata.ContainsKey("Raven-Remove-Document-Marker") &&
metadata.Value <bool>("Raven-Remove-Document-Marker"))
{
if (resolvedMetadataToSave.ContainsKey(Constants.RavenEntityName))
{
metadata[Constants.RavenEntityName] = resolvedMetadataToSave[Constants.RavenEntityName];
}
DeleteItem(id, null);
MarkAsDeleted(id, metadata);
}
else
{
var etag = deleted == false ? existingEtag : null;
var resolvedItemJObject = resolvedItemToSave as RavenJObject;
if (resolvedItemJObject != null)
{
ExecuteRemoveConflictOnPutTrigger(id, metadata, resolvedItemJObject);
}
AddWithoutConflict(id, etag, resolvedMetadataToSave, resolvedItemToSave);
}
return;
}
//this is expensive but worth trying if we can avoid conflicts
if (TryResolveConflictByCheckingIfIdentical(id, metadata, incoming, existingItem, out resolvedMetadataToSave))
{
//The metadata here is merged (changed), it needs to be pushed.
AddWithoutConflict(id, null, resolvedMetadataToSave, incoming);
return;
}
CreatedConflict createdConflict;
var newDocumentConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag);
if (existingDocumentIsInConflict) // the existing document is in conflict
{
if (log.IsDebugEnabled)
{
log.Debug("Conflicted item {0} has a new version from {1}, adding to conflicted documents", id, Src);
}
createdConflict = AppendToCurrentItemConflicts(id, newDocumentConflictId, existingMetadata, existingItem);
}
else
{
if (log.IsDebugEnabled)
{
log.Debug("Existing item {0} is in conflict with replicated version from {1}, marking item as conflicted", id, Src);
}
// we have a new conflict
// move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + GetReplicationIdentifierForCurrentDatabase();
createdConflict = CreateConflict(id, newDocumentConflictId, existingDocumentConflictId, existingItem,
existingMetadata);
}
Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() =>
Database.Notifications.RaiseNotifications(new ReplicationConflictNotification()
{
Id = id,
Etag = createdConflict.Etag,
ItemType = ReplicationConflict,
OperationType = ReplicationOperationTypes.Put,
Conflicts = createdConflict.ConflictedIds
}));
}
public void UpdateFileMetadata(string filename, RavenJObject metadata)
{
Api.JetSetCurrentIndex(session, Files, "by_name");
Api.MakeKey(session, Files, filename, Encoding.Unicode, MakeKeyGrbit.NewKey);
if (Api.TrySeek(session, Files, SeekGrbit.SeekEQ) == false)
throw new FileNotFoundException(filename);
using (var update = new Update(session, Files, JET_prep.Replace))
{
if (!metadata.ContainsKey("ETag"))
{
throw new InvalidOperationException("Metadata of file {0} does not contain 'ETag' key " + filename);
}
var innerEsentMetadata = new RavenJObject(metadata);
var etag = innerEsentMetadata.Value<Guid>("ETag");
innerEsentMetadata.Remove("ETag");
var existingMetadata = RetrieveMetadata();
if (existingMetadata.ContainsKey("Content-MD5"))
{
innerEsentMetadata["Content-MD5"] = existingMetadata["Content-MD5"];
}
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["etag"], etag.TransformToValueForEsentSorting());
Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["metadata"], ToQueryString(innerEsentMetadata), Encoding.Unicode);
update.Save();
}
}
private void ReplicateDelete(string id, RavenJObject newMetadata, TExternal incoming)
{
TInternal existingItem;
Etag existingEtag;
bool deleted;
var existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted);
if (existingMetadata == null)
{
if (log.IsDebugEnabled)
{
log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring.", id, Src);
}
return;
}
if (existingMetadata.ContainsKey(Constants.RavenEntityName))
{
newMetadata[Constants.RavenEntityName] = existingMetadata[Constants.RavenEntityName];
}
RavenJObject currentReplicationEntry = null;
if (newMetadata.ContainsKey(Constants.RavenReplicationVersion) &&
newMetadata.ContainsKey(Constants.RavenReplicationSource))
{
currentReplicationEntry = new RavenJObject
{
{ Constants.RavenReplicationVersion, newMetadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, newMetadata[Constants.RavenReplicationSource] }
};
}
var existingHistory = ReplicationData.GetOrCreateHistory(existingMetadata);
if (currentReplicationEntry != null &&
existingHistory.Any(x => RavenJTokenEqualityComparer.Default.Equals(
((RavenJObject)x)[Constants.RavenReplicationSource], currentReplicationEntry[Constants.RavenReplicationSource]) &&
((RavenJObject)x)[Constants.RavenReplicationVersion].Value <long>() >= currentReplicationEntry[Constants.RavenReplicationVersion].Value <long>()))
{
if (log.IsDebugEnabled)
{
log.Debug("Replicated delete for {0} already exist in item history, ignoring", id);
}
return;
}
if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker)) //deleted locally as well
{
if (log.IsDebugEnabled)
{
log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories.", id, Src);
}
var newHistory = ReplicationData.GetOrCreateHistory(newMetadata);
if (currentReplicationEntry != null)
{
newHistory.Add(currentReplicationEntry);
}
//Merge histories
ReplicationData.SetHistory(newMetadata, Historian.MergeReplicationHistories(newHistory, existingHistory, id));
newMetadata[Constants.RavenReplicationMergedHistory] = true;
MarkAsDeleted(id, newMetadata);
return;
}
if (Historian.IsDirectChildOfCurrent(newMetadata, existingMetadata)) // not modified
{
if (log.IsDebugEnabled)
{
log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src);
}
DeleteItem(id, existingEtag);
MarkAsDeleted(id, newMetadata);
return;
}
CreatedConflict createdConflict;
if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict)) // locally conflicted
{
if (log.IsDebugEnabled)
{
log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src);
}
var savedConflictedItemId = SaveConflictedItem(id, newMetadata, incoming, existingEtag);
createdConflict = AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem);
}
else
{
RavenJObject resolvedMetadataToSave;
TExternal resolvedItemToSave;
if (TryResolveConflict(id, newMetadata, incoming, existingItem, out resolvedMetadataToSave, out resolvedItemToSave))
{
AddWithoutConflict(id, existingEtag, resolvedMetadataToSave, resolvedItemToSave);
return;
}
var newConflictId = SaveConflictedItem(id, newMetadata, incoming, existingEtag);
if (log.IsDebugEnabled)
{
log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src);
}
// we have a new conflict move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + GetReplicationIdentifierForCurrentDatabase();
createdConflict = CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata);
}
Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() =>
Database.Notifications.RaiseNotifications(new ReplicationConflictNotification()
{
Id = id,
Etag = createdConflict.Etag,
Conflicts = createdConflict.ConflictedIds,
ItemType = ReplicationConflictTypes.DocumentReplicationConflict,
OperationType = ReplicationOperationTypes.Delete
}));
}
public void PutFile(string filename, long? totalSize, RavenJObject metadata, bool tombstone = false)
{
var filesByEtag = storage.Files.GetIndex(Tables.Files.Indices.ByEtag);
var key = CreateKey(filename);
if (!metadata.ContainsKey("ETag"))
throw new InvalidOperationException(string.Format("Metadata of file {0} does not contain 'ETag' key", filename));
ushort version;
var existingFile = LoadJson(storage.Files, key, writeBatch.Value, out version);
var innerMetadata = new RavenJObject(metadata);
var etag = innerMetadata.Value<Guid>("ETag");
innerMetadata.Remove("ETag");
var file = new RavenJObject
{
{ "name", filename },
{ "total_size", totalSize },
{ "uploaded_size", 0 },
{ "etag", new RavenJValue(etag) },
{ "metadata", innerMetadata }
};
storage.Files.Add(writeBatch.Value, key, file, version);
if (existingFile != null)
{
filesByEtag.Delete(writeBatch.Value, CreateKey(existingFile.Value<Guid>("etag")));
}
filesByEtag.Add(writeBatch.Value, CreateKey(etag), key);
if (tombstone)
return;
var fileCount = storage.Files.GetIndex(Tables.Files.Indices.Count);
fileCount.Add(writeBatch.Value, key, key);
}
protected void InsertItems(string tableName, string pkName, IEnumerable<ItemToReplicate> dataForTable)
{
tableName = tableName.ToLowerInvariant(); // type names have to be lowercase
foreach (var itemToReplicate in dataForTable)
{
var o = new RavenJObject();
if (database != null)
database.WorkContext.CancellationToken.ThrowIfCancellationRequested();
foreach (var column in itemToReplicate.Columns.Where(column => column.Key != pkName))
{
o[column.Key] = column.Value;
}
if ("_id".Equals(pkName))
{
bulkCommands.Add("{\"index\":{\"_index\":\"" + targetIndexName + "\",\"_type\":\"" + tableName + "\",\"_id\":\"" + itemToReplicate.DocumentId + "\"}}");
}
else
{
o[pkName] = itemToReplicate.DocumentId;
bulkCommands.Add("{\"index\":{\"_index\":\"" + targetIndexName + "\",\"_type\":\"" + tableName + "\"}}");
}
// Setup timestamps, converting from a Javascript notation to an ES/Kibana notation
if (o.ContainsKey("$timestamp"))
{
o["@timestamp"] = o["$timestamp"];
o.Remove("$timestamp");
}
else
{
o["@timestamp"] = DateTime.UtcNow;
}
bulkCommands.Add(o.ToString(Formatting.None));
}
}
public AddDocumentResult AddDocument(string key, Etag etag, RavenJObject data, RavenJObject metadata, InvokeSource source = InvokeSource.Default)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
var byteCount = Encoding.Unicode.GetByteCount(key);
if (byteCount >= 2048)
{
throw new ArgumentException(string.Format("The key must be a maximum of 2,048 bytes in Unicode, 1,024 characters, key is: '{0}'", key), "key");
}
try
{
Api.JetSetCurrentIndex(session, Documents, "by_key");
Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
var isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ);
Etag existingEtag = null;
if (isUpdate)
{
existingEtag = EnsureDocumentEtagMatch(key, etag, "PUT");
}
else
{
if (etag != null && etag != Etag.Empty) // expected something to be there.
{
throw new ConcurrencyException("PUT attempted on document '" + key +
"' using a non current etag (document deleted)")
{
ExpectedETag = etag
}
}
;
if (Api.TryMoveFirst(session, Details))
{
Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["document_count"], 1);
}
}
Etag newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents);
DateTime savedAt;
try
{
using (var update = new Update(session, Documents, isUpdate ? JET_prep.Replace : JET_prep.Insert))
{
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode);
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"]))
{
if (isUpdate)
{
columnStream.SetLength(0); // empty the existing value, since we are going to overwrite the entire thing
}
using (Stream stream = new BufferedStream(columnStream))
using (
var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current))
)
{
data.WriteTo(finalStream);
finalStream.Flush();
}
}
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"],
newEtag.TransformToValueForEsentSorting());
var keepLastModified = source == InvokeSource.FromConflictAtReplication && metadata.ContainsKey(Constants.LastModified);
savedAt = keepLastModified ? metadata.Value <DateTime>(Constants.LastModified) : SystemTime.UtcNow;
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt.ToBinary());
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"]))
{
if (isUpdate)
{
columnStream.SetLength(0);
}
using (Stream stream = new BufferedStream(columnStream))
{
metadata.WriteTo(stream);
stream.Flush();
}
}
update.Save();
}
}
catch (EsentErrorException e)
{
if (e.Error == JET_err.KeyDuplicate || e.Error == JET_err.WriteConflict)
{
throw new ConcurrencyException("PUT attempted on document '" + key + "' concurrently", e);
}
throw;
}
if (logger.IsDebugEnabled)
{
logger.Debug("Inserted a new document with key '{0}', update: {1}, ",
key, isUpdate);
}
if (existingEtag != null)
{
cacher.RemoveCachedDocument(key, existingEtag);
}
return(new AddDocumentResult
{
Etag = newEtag,
PrevEtag = existingEtag,
SavedAt = savedAt,
Updated = isUpdate
});
}
catch (EsentKeyDuplicateException e)
{
throw new ConcurrencyException("Illegal duplicate key " + key, e);
}
}
private void ReplicateDelete(string id, RavenJObject metadata, TExternal incoming)
{
TInternal existingItem;
Etag existingEtag;
bool deleted;
var existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted);
if (existingMetadata == null)
{
log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring", id, Src);
return;
}
if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker)) //deleted locally as well
{
log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories", id, Src);
var existingHistory = new RavenJArray(ReplicationData.GetHistory(existingMetadata));
var newHistory = new RavenJArray(ReplicationData.GetHistory(metadata));
foreach (var item in newHistory)
{
existingHistory.Add(item);
}
if (metadata.ContainsKey(Constants.RavenReplicationVersion) &&
metadata.ContainsKey(Constants.RavenReplicationSource))
{
existingHistory.Add(new RavenJObject
{
{ Constants.RavenReplicationVersion, metadata[Constants.RavenReplicationVersion] },
{ Constants.RavenReplicationSource, metadata[Constants.RavenReplicationSource] }
});
}
while (existingHistory.Length > Constants.ChangeHistoryLength)
{
existingHistory.RemoveAt(0);
}
MarkAsDeleted(id, metadata);
return;
}
if (Historian.IsDirectChildOfCurrent(metadata, existingMetadata)) // not modified
{
log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src);
DeleteItem(id, existingEtag);
MarkAsDeleted(id, metadata);
return;
}
CreatedConflict createdConflict;
if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict)) // locally conflicted
{
log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src);
var savedConflictedItemId = SaveConflictedItem(id, metadata, incoming, existingEtag);
createdConflict = AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem);
}
else
{
RavenJObject resolvedMetadataToSave;
TExternal resolvedItemToSave;
if (TryResolveConflict(id, metadata, incoming, existingItem, out resolvedMetadataToSave, out resolvedItemToSave))
{
AddWithoutConflict(id, existingEtag, resolvedMetadataToSave, resolvedItemToSave);
return;
}
var newConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag);
log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src);
// we have a new conflict move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag);
createdConflict = CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata);
}
Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() =>
Database.Notifications.RaiseNotifications(new ReplicationConflictNotification()
{
Id = id,
Etag = createdConflict.Etag,
Conflicts = createdConflict.ConflictedIds,
ItemType = ReplicationConflictTypes.DocumentReplicationConflict,
OperationType = ReplicationOperationTypes.Delete
}));
}
public void UpdateFileMetadata(string filename, RavenJObject metadata)
{
var key = CreateKey(filename);
ushort version;
var file = LoadJson(storage.Files, key, writeBatch.Value, out version);
if (file == null)
throw new FileNotFoundException(filename);
if (!metadata.ContainsKey("ETag"))
throw new InvalidOperationException(string.Format("Metadata of file {0} does not contain 'ETag' key", filename));
var innerMetadata = new RavenJObject(metadata);
var etag = innerMetadata.Value<Guid>("ETag");
innerMetadata.Remove("ETag");
var existingMetadata = (RavenJObject) file["metadata"];
if (existingMetadata.ContainsKey("Content-MD5"))
innerMetadata["Content-MD5"] = existingMetadata["Content-MD5"];
var oldEtag = file.Value<Guid>("etag");
file["etag"] = new RavenJValue(etag);
file["metadata"] = innerMetadata;
storage.Files.Add(writeBatch.Value, key, file, version);
var filesByEtag = storage.Files.GetIndex(Tables.Files.Indices.ByEtag);
filesByEtag.Delete(writeBatch.Value, CreateKey(oldEtag));
filesByEtag.Add(writeBatch.Value, CreateKey(etag), key);
}
public void Replicate(string id, RavenJObject metadata, TExternal incoming)
{
if (metadata.Value <bool>(Constants.RavenDeleteMarker))
{
ReplicateDelete(id, metadata, incoming);
return;
}
TInternal existingItem;
Etag existingEtag;
bool deleted;
RavenJObject existingMetadata;
try
{
existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted);
}
catch (Exception e)
{
log.Error("Replication - fetching existing item failed. (key = {0})", id);
throw new InvalidOperationException("Replication - fetching existing item failed. (key = " + id + ")", e);
}
if (existingMetadata == null)
{
log.Debug("New item {0} replicated successfully from {1}", id, Src);
AddWithoutConflict(id, null, metadata, incoming);
return;
}
// we just got the same version from the same source - request playback again?
// at any rate, not an error, moving on
if (existingMetadata.Value <string>(Constants.RavenReplicationSource) ==
metadata.Value <string>(Constants.RavenReplicationSource)
&&
existingMetadata.Value <long>(Constants.RavenReplicationVersion) ==
metadata.Value <long>(Constants.RavenReplicationVersion))
{
return;
}
var existingDocumentIsInConflict = existingMetadata[Constants.RavenReplicationConflict] != null;
if (existingDocumentIsInConflict == false &&
// if the current document is not in conflict, we can continue without having to keep conflict semantics
(Historian.IsDirectChildOfCurrent(metadata, existingMetadata)))
// this update is direct child of the existing doc, so we are fine with overwriting this
{
log.Debug("Existing item {0} replicated successfully from {1}", id, Src);
var etag = deleted == false ? existingEtag : null;
AddWithoutConflict(id, etag, metadata, incoming);
return;
}
RavenJObject resolvedMetadataToSave;
TExternal resolvedItemToSave;
if (TryResolveConflict(id, metadata, incoming, existingItem, out resolvedMetadataToSave, out resolvedItemToSave))
{
if (metadata.ContainsKey("Raven-Remove-Document-Marker") &&
metadata.Value <bool>("Raven-Remove-Document-Marker"))
{
DeleteItem(id, null);
MarkAsDeleted(id, metadata);
}
else
{
var etag = deleted == false ? existingEtag : null;
AddWithoutConflict(id, etag, resolvedMetadataToSave, resolvedItemToSave);
}
return;
}
CreatedConflict createdConflict;
var newDocumentConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag);
if (existingDocumentIsInConflict) // the existing document is in conflict
{
log.Debug("Conflicted item {0} has a new version from {1}, adding to conflicted documents", id, Src);
createdConflict = AppendToCurrentItemConflicts(id, newDocumentConflictId, existingMetadata, existingItem);
}
else
{
log.Debug("Existing item {0} is in conflict with replicated version from {1}, marking item as conflicted", id, Src);
// we have a new conflict
// move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag);
createdConflict = CreateConflict(id, newDocumentConflictId, existingDocumentConflictId, existingItem,
existingMetadata);
}
Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() =>
Database.Notifications.RaiseNotifications(new ReplicationConflictNotification()
{
Id = id,
Etag = createdConflict.Etag,
ItemType = ReplicationConflict,
OperationType = ReplicationOperationTypes.Put,
Conflicts = createdConflict.ConflictedIds
}));
}
public Item(ItemClass iClass, RavenJObject val)
{
Attributes = new Dictionary <string, List <float> >();
Mods = new List <ItemMod>();
Class = iClass;
Name = val["name"].Value <string>();
if (Name == "")
{
Name = val["typeLine"].Value <string>();
}
Type = val["typeLine"].Value <string>();
if (val.ContainsKey("properties"))
{
foreach (RavenJObject obj in (RavenJArray)val["properties"])
{
var values = new List <float>();
string s = "";
foreach (RavenJArray jva in (RavenJArray)obj["values"])
{
s += " " + jva[0].Value <string>();
}
s = s.TrimStart();
if (s == "")
{
Keywords = new List <string>();
string[] sl = obj["name"].Value <string>().Split(',');
foreach (string i in sl)
{
Keywords.Add(i.Trim());
}
continue;
}
foreach (Match m in numberfilter.Matches(s))
{
if (m.Value == "")
{
values.Add(float.NaN);
}
else
{
values.Add(float.Parse(m.Value, CultureInfo.InvariantCulture));
}
}
string cs = obj["name"].Value <string>() + ": " + (numberfilter.Replace(s, "#"));
Attributes.Add(cs, values);
}
}
if (val.ContainsKey("explicitMods"))
{
foreach (string s in val["explicitMods"].Values <string>())
{
List <ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter);
Mods.AddRange(mods);
}
}
if (val.ContainsKey("implicitMods"))
{
foreach (string s in val["implicitMods"].Values <string>())
{
List <ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter);
Mods.AddRange(mods);
}
}
if (val.ContainsKey("craftedMods"))
{
foreach (string s in val["craftedMods"].Values <string>())
{
List <ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter);
Mods.AddRange(mods);
}
}
if (iClass == ItemClass.Gem)
{
switch (val["colour"].Value <string>())
{
case "S":
Keywords.Add("Strength");
break;
case "D":
Keywords.Add("Dexterity");
break;
case "I":
Keywords.Add("Intelligence");
break;
}
}
else
{
Gems = new List <Item>();
}
var Sockets = new List <int>();
if (val.ContainsKey("sockets"))
{
foreach (RavenJObject obj in (RavenJArray)val["sockets"])
{
Sockets.Add(obj["group"].Value <int>());
}
}
if (val.ContainsKey("socketedItems"))
{
int socket = 0;
foreach (RavenJObject obj in (RavenJArray)val["socketedItems"])
{
var item = new Item(ItemClass.Gem, obj);
item.SocketGroup = Sockets[socket++];
Gems.Add(item);
}
}
}
