public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { VersioningConfiguration versioningConfiguration; if (metadata.ContainsKey(Constants.RavenCreateVersion)) { metadata.__ExternalState[Constants.RavenCreateVersion] = metadata[Constants.RavenCreateVersion]; metadata.Remove(Constants.RavenCreateVersion); } if (metadata.ContainsKey(Constants.RavenIgnoreVersioning)) { metadata.__ExternalState[Constants.RavenIgnoreVersioning] = metadata[Constants.RavenIgnoreVersioning]; metadata.Remove(Constants.RavenIgnoreVersioning); return; } if (TryGetVersioningConfiguration(key, metadata, out versioningConfiguration) == false) return; var revision = GetNextRevisionNumber(key); using (Database.DisableAllTriggersForCurrentThread()) { RemoveOldRevisions(key, revision, versioningConfiguration, transactionInformation); } metadata.__ExternalState["Next-Revision"] = revision; metadata.__ExternalState["Parent-Revision"] = metadata.Value<string>(VersioningUtil.RavenDocumentRevision); metadata[VersioningUtil.RavenDocumentRevisionStatus] = RavenJToken.FromObject("Current"); metadata[VersioningUtil.RavenDocumentRevision] = RavenJToken.FromObject(revision); }
private static DateTimeOffset GetLastModified(RavenJObject metadata) { if (metadata.ContainsKey(Constants.LastModified)) return metadata.Value<DateTimeOffset>(Constants.LastModified); if (metadata.ContainsKey(Constants.RavenLastModified)) return metadata.Value<DateTimeOffset>(Constants.RavenLastModified); throw new InvalidOperationException("Could not find last modification date in metadata"); }
public static RavenJToken HandleConflictDocuments(RavenJObject metadata) { if (metadata == null) return null; if (metadata.ContainsKey(Constants.RavenReplicationConflictDocument)) metadata.Add(Constants.RavenReplicationConflictDocumentForcePut, true); if (metadata.ContainsKey(Constants.RavenReplicationConflict)) metadata.Add(Constants.RavenReplicationConflictSkipResolution, true); return metadata; }
public override void OnPut(string name, RavenJObject headers) { if (headers.ContainsKey(Constants.RavenCreateVersion)) { headers.__ExternalState[Constants.RavenCreateVersion] = headers[Constants.RavenCreateVersion]; headers.Remove(Constants.RavenCreateVersion); } FileSystem.Storage.Batch(accessor => { VersioningConfiguration versioningConfiguration; if (TryGetVersioningConfiguration(name, headers, accessor, out versioningConfiguration) == false) return; var revision = GetNextRevisionNumber(name, accessor); using (FileSystem.DisableAllTriggersForCurrentThread()) { RemoveOldRevisions(name, revision, versioningConfiguration); } headers.__ExternalState["Next-Revision"] = revision; headers.__ExternalState["Parent-Revision"] = headers.Value<string>(VersioningUtil.RavenFileRevision); headers[VersioningUtil.RavenFileRevisionStatus] = RavenJToken.FromObject("Current"); headers[VersioningUtil.RavenFileRevision] = RavenJToken.FromObject(revision); }); }
public bool FilterDocuments(string destinationId, string key, RavenJObject metadata) { if (key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)) // don't replicate system docs { if (key.StartsWith("Raven/Hilo/", StringComparison.InvariantCultureIgnoreCase) == false) // except for hilo documents return false; } if (metadata.ContainsKey(Constants.NotForReplication) && metadata.Value<bool>(Constants.NotForReplication)) // not explicitly marked to skip return false; if (metadata[Constants.RavenReplicationConflict] != null) // don't replicate conflicted documents, that just propagate the conflict return false; if (metadata.Value<string>(Constants.RavenReplicationSource) == destinationId) // prevent replicating back to source return false; switch (ReplicationOptionsBehavior) { case TransitiveReplicationOptions.None: var value = metadata.Value<string>(Constants.RavenReplicationSource); var replicateDoc = value == null || (value == CurrentDatabaseId); return replicateDoc; } return true; }
public SynchronizationWorkItem DetermineWork(string file, RavenJObject localMetadata, RavenJObject destinationMetadata, string localServerUrl, out NoSyncReason reason) { reason = NoSyncReason.Unknown; if (localMetadata == null) { reason = NoSyncReason.SourceFileNotExist; return null; } if (destinationMetadata != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflictResolution] == null) { reason = NoSyncReason.DestinationFileConflicted; return null; } if (localMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null) { reason = NoSyncReason.SourceFileConflicted; return null; } if (localMetadata[SynchronizationConstants.RavenDeleteMarker] != null) { if (localMetadata.ContainsKey(SynchronizationConstants.RavenRenameFile)) { var rename = localMetadata.Value<string>(SynchronizationConstants.RavenRenameFile); if (destinationMetadata != null) return new RenameWorkItem(file, rename, localServerUrl, storage); return new ContentUpdateWorkItem(rename, localServerUrl, storage, sigGenerator); // we have a rename tombstone but file does not exists on destination } return new DeleteWorkItem(file, localServerUrl, storage); } if (destinationMetadata != null && Historian.IsDirectChildOfCurrent(localMetadata, destinationMetadata)) { reason = NoSyncReason.ContainedInDestinationHistory; return null; } // file exists on dest and has the same content if (destinationMetadata != null && localMetadata.Value<string>("Content-MD5") == destinationMetadata.Value<string>("Content-MD5")) { // check metadata to detect if any synchronization is needed if (localMetadata.Keys.Except(new[] { Constants.MetadataEtagField, Constants.RavenLastModified, Constants.LastModified }) .Any(key => !destinationMetadata.ContainsKey(key) || localMetadata[key] != destinationMetadata[key])) { return new MetadataUpdateWorkItem(file, localServerUrl, destinationMetadata, storage); } reason = NoSyncReason.SameContentAndMetadata; return null; // the same content and metadata - no need to synchronize } return new ContentUpdateWorkItem(file, localServerUrl, storage, sigGenerator); }
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { if (!metadata.ContainsKey("IsRoutable") || !metadata.Value<bool>("IsRoutable")) { return; } RavenJToken parentIdToken; RavenJToken slugToken; if (document.TryGetValue("ParentId", out parentIdToken) && document.TryGetValue("Slug", out slugToken)) { var parentId = parentIdToken.Value<string>(); var slug = slugToken.Value<string>(); string parentPath = null; if (!String.IsNullOrEmpty(parentId)) { var parent = Database.Get(parentId, transactionInformation); parentPath = parent.DataAsJson["Path"].Value<string>(); } if (String.IsNullOrEmpty(parentPath)) { document["Path"] = slug; } else { document["Path"] = parentPath + "/" + slug; } } base.OnPut(key, document, metadata, transactionInformation); }
protected void AssertLocalFileExistsAndIsNotConflicted(RavenJObject sourceMetadata) { if (sourceMetadata == null) throw new SynchronizationException(string.Format("File {0} does not exist", FileName)); if (sourceMetadata.ContainsKey(SynchronizationConstants.RavenSynchronizationConflict)) throw new SynchronizationException(string.Format("File {0} is conflicted", FileName)); }
public override ReadVetoResult AllowRead(string key, RavenJObject metadata, ReadOperation operation, TransactionInformation transactionInformation) { if (operation != ReadOperation.Index) return ReadVetoResult.Allowed; if (metadata.ContainsKey("Deleted") == false) return ReadVetoResult.Allowed; return ReadVetoResult.Ignore; }
public static ICommandData CreateCommand(RavenJObject jsonCommand, TransactionInformation transactionInformation) { var key = jsonCommand["Key"].Value<string>(); switch (jsonCommand.Value<string>("Method")) { case "PUT": var putCommand = new PutCommandData { Key = key, Etag = GetEtagFromCommand(jsonCommand), Document = jsonCommand["Document"] as RavenJObject, Metadata = jsonCommand["Metadata"] as RavenJObject, TransactionInformation = transactionInformation }; ValidateMetadataKeys(putCommand.Metadata); return putCommand; case "DELETE": return new DeleteCommandData { Key = key, Etag = GetEtagFromCommand(jsonCommand), TransactionInformation = transactionInformation }; case "PATCH": return new PatchCommandData { Key = key, Etag = GetEtagFromCommand(jsonCommand), TransactionInformation = transactionInformation, Metadata = jsonCommand["Metadata"] as RavenJObject, Patches = jsonCommand .Value<RavenJArray>("Patches") .Cast<RavenJObject>() .Select(PatchRequest.FromJson) .ToArray(), PatchesIfMissing = jsonCommand["PatchesIfMissing"] == null ? null : jsonCommand .Value<RavenJArray>("PatchesIfMissing") .Cast<RavenJObject>() .Select(PatchRequest.FromJson) .ToArray(), SkipPatchIfEtagMismatch = jsonCommand.ContainsKey("SkipPatchIfEtagMismatch") && jsonCommand.Value<bool>("SkipPatchIfEtagMismatch") }; case "EVAL": var debug = jsonCommand["DebugMode"].Value<bool>(); return new ScriptedPatchCommandData { Key = key, Metadata = jsonCommand["Metadata"] as RavenJObject, Etag = GetEtagFromCommand(jsonCommand), TransactionInformation = transactionInformation, Patch = ScriptedPatchRequest.FromJson(jsonCommand.Value<RavenJObject>("Patch")), PatchIfMissing = jsonCommand["PatchIfMissing"] == null ? null : ScriptedPatchRequest.FromJson(jsonCommand.Value<RavenJObject>("PatchIfMissing")), DebugMode = debug }; default: throw new ArgumentException("Batching only supports PUT, PATCH, EVAL and DELETE."); } }
public static List<HistoryItem> DeserializeHistory(RavenJObject metadata) { var history = new List<HistoryItem>(); if (metadata.ContainsKey(SynchronizationConstants.RavenSynchronizationHistory)) { var array = (RavenJArray) metadata[SynchronizationConstants.RavenSynchronizationHistory]; var items = array.Values<RavenJObject>().Select(x => JsonExtensions.JsonDeserialization<HistoryItem>(x)); return new List<HistoryItem>(items); } return history; }
public void InitializeMetadata(string name, RavenJObject metadata) { if (metadata.ContainsKey(Constants.RavenCreateVersion)) { metadata.__ExternalState[Constants.RavenCreateVersion] = metadata[Constants.RavenCreateVersion]; metadata.Remove(Constants.RavenCreateVersion); } if (metadata.ContainsKey(Constants.RavenIgnoreVersioning)) { metadata.__ExternalState[Constants.RavenIgnoreVersioning] = metadata[Constants.RavenIgnoreVersioning]; metadata.Remove(Constants.RavenIgnoreVersioning); return; } fileSystem.Storage.Batch(accessor => { FileVersioningConfiguration versioningConfiguration; if (TryGetVersioningConfiguration(name, metadata, accessor, out versioningConfiguration) == false) return; long revision; if (metadata.__ExternalState.ContainsKey("Synchronization-Next-Revision")) revision = (long) metadata.__ExternalState["Synchronization-Next-Revision"]; else revision = GetNextRevisionNumber(name, accessor); RemoveOldRevisions(name, revision, versioningConfiguration); metadata.__ExternalState["Next-Revision"] = revision; metadata.__ExternalState["Parent-Revision"] = metadata.Value<string>(VersioningUtil.RavenFileRevision); metadata[VersioningUtil.RavenFileRevisionStatus] = RavenJToken.FromObject("Current"); metadata[VersioningUtil.RavenFileRevision] = RavenJToken.FromObject(revision); }); }
public bool FilterDocuments(string destinationId, string key, RavenJObject metadata, out string reason) { if (IsSystemDocumentId(key)) { reason = string.Format("Will not replicate document '{0}' to '{1}' because it is a system document", key, destinationId); Log.Debug(reason); return false; } if (metadata.ContainsKey(Constants.NotForReplication) && metadata.Value<bool>(Constants.NotForReplication)) // not explicitly marked to skip { reason = string.Format("Will not replicate document '{0}' to '{1}' because it was marked as not for replication", key, destinationId); Log.Debug(reason); return false; } if (metadata[Constants.RavenReplicationConflict] != null) // don't replicate conflicted documents, that just propagate the conflict { reason = string.Format("Will not replicate document '{0}' to '{1}' because it a conflict document", key, destinationId); Log.Debug(reason); return false; } if (OriginsFromDestination(destinationId, metadata)) // prevent replicating back to source { reason = string.Format("Will not replicate document '{0}' to '{1}' because the destination server is the same server it originated from", key, destinationId); Log.Debug(reason); return false; } switch (ReplicationOptionsBehavior) { case TransitiveReplicationOptions.None: var value = metadata.Value<string>(Constants.RavenReplicationSource); if (value != null && (value != CurrentDatabaseId)) { reason = string.Format("Will not replicate document '{0}' to '{1}' because it was not created on the current server, and TransitiveReplicationOptions = none", key, destinationId); Log.Debug(reason); return false; } break; } reason = string.Format("Will replicate '{0}' to '{1}'", key, destinationId); Log.Debug(reason); return true; }
public override void AfterPut(string key, RavenJObject document, RavenJObject metadata, Etag etag, TransactionInformation transactionInformation) { if (!metadata.__ExternalState.ContainsKey(RavenDocumentRevision)) return; using (Database.DisableAllTriggersForCurrentThread()) { var revisionMetadata = new RavenJObject(metadata); revisionMetadata[RavenDocumentRevisionStatus] = RavenJToken.FromObject("Historical"); revisionMetadata.Remove(RavenDocumentRevision); if (revisionMetadata.ContainsKey(RavenDocumentEnsureUniqueConstraints)) revisionMetadata.Remove(RavenDocumentEnsureUniqueConstraints); var revisionCopy = new RavenJObject(document); var revisionkey = key + RevisionSegment + metadata.__ExternalState[RavenDocumentRevision]; Database.Documents.Put(revisionkey, null, revisionCopy, revisionMetadata, transactionInformation); } }
public override void AfterPut(string key, RavenJObject document, RavenJObject metadata, Guid etag, TransactionInformation transactionInformation) { // leave raven system docs alone if (key.StartsWith("Raven/")) return; // when there's already a Created date written, this is not the original insert if (metadata.ContainsKey("Created")) return; // get the timestamp set for the last-modified date var timestamp = metadata.Value<DateTime>(Constants.LastModified); // copy the metadata and add the timestamp var newMetadata = (RavenJObject)metadata.CreateSnapshot(); newMetadata.Add("Created", timestamp); // update the metadata in the document using (Database.DisableAllTriggersForCurrentThread()) Database.PutDocumentMetadata(key, newMetadata); }
public bool FilterDocuments(string destinationId, string key, RavenJObject metadata) { if (IsSystemDocumentId(key)) return false; if (metadata.ContainsKey(Constants.NotForReplication) && metadata.Value<bool>(Constants.NotForReplication)) // not explicitly marked to skip return false; if (metadata[Constants.RavenReplicationConflict] != null) // don't replicate conflicted documents, that just propagate the conflict return false; if (metadata.Value<string>(Constants.RavenReplicationSource) == destinationId) // prevent replicating back to source return false; switch (ReplicationOptionsBehavior) { case TransitiveReplicationOptions.None: var value = metadata.Value<string>(Constants.RavenReplicationSource); var replicateDoc = value == null || (value == CurrentDatabaseId); return replicateDoc; } return true; }
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { Contract.Assume(!string.IsNullOrWhiteSpace(key)); RavenJToken versionToken; if (!document.TryGetValue("Revision", out versionToken) || key.Contains(RevisionSegment)) return; var newRevision = versionToken.Value<int>(); var currentRevision = metadata.ContainsKey(RavenDocumentRevision) ? metadata[RavenDocumentRevision].Value<int>() : 0; metadata[RavenDocumentRevisionStatus] = RavenJToken.FromObject("Current"); //if we have a higher revision number than the existing then put a new revision if (newRevision > currentRevision) { metadata[RavenDocumentRevision] = RavenJToken.FromObject(versionToken.Value<int>()); metadata.__ExternalState[RavenDocumentRevision] = metadata[RavenDocumentRevision]; } }
public Item(ItemClass iClass, RavenJObject val) { Attributes = new Dictionary<string, List<float>>(); Mods = new List<ItemMod>(); Class = iClass; Name = val["name"].Value<string>(); if (Name == "") Name = val["typeLine"].Value<string>(); Type = val["typeLine"].Value<string>(); if (val.ContainsKey("properties")) foreach (RavenJObject obj in (RavenJArray) val["properties"]) { var values = new List<float>(); string s = ""; foreach (RavenJArray jva in (RavenJArray) obj["values"]) { s += " " + jva[0].Value<string>(); } s = s.TrimStart(); if (s == "") { Keywords = new List<string>(); string[] sl = obj["name"].Value<string>().Split(','); foreach (string i in sl) Keywords.Add(i.Trim()); continue; } foreach (Match m in numberfilter.Matches(s)) { if (m.Value == "") values.Add(float.NaN); else values.Add(float.Parse(m.Value, CultureInfo.InvariantCulture)); } string cs = obj["name"].Value<string>() + ": " + (numberfilter.Replace(s, "#")); Attributes.Add(cs, values); } if (val.ContainsKey("explicitMods")) foreach (string s in val["explicitMods"].Values<string>()) { List<ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter); Mods.AddRange(mods); } if (val.ContainsKey("implicitMods")) foreach (string s in val["implicitMods"].Values<string>()) { List<ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter); Mods.AddRange(mods); } if (val.ContainsKey("craftedMods")) foreach (string s in val["craftedMods"].Values<string>()) { List<ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter); Mods.AddRange(mods); } if (iClass == ItemClass.Gem) { switch (val["colour"].Value<string>()) { case "S": Keywords.Add("Strength"); break; case "D": Keywords.Add("Dexterity"); break; case "I": Keywords.Add("Intelligence"); break; } } else { Gems = new List<Item>(); } var Sockets = new List<int>(); if (val.ContainsKey("sockets")) foreach (RavenJObject obj in (RavenJArray) val["sockets"]) { Sockets.Add(obj["group"].Value<int>()); } if (val.ContainsKey("socketedItems")) { int socket = 0; foreach (RavenJObject obj in (RavenJArray) val["socketedItems"]) { var item = new Item(ItemClass.Gem, obj); item.SocketGroup = Sockets[socket++]; Gems.Add(item); } } }
//------------------------------------------------------------------------------- public Item(ItemClass iClass, RavenJObject val, Dictionary<string, float> skillCastTimes) { this.skillCastTimes = skillCastTimes; Mods = new List<Mod>(); Requirements = new Dictionary<string, List<float>>(); Properties = new Dictionary<string, List<float>>(); Class = iClass; WeaponType = ""; Gems = new List<SkillGem>(); TypeLine = val["typeLine"].Value<string>(); Name = val["name"].Value<string>(); if (val.ContainsKey("icon")) { IconUrl = val["icon"].Value<string>(); } if (val.ContainsKey("properties")) { if (Class == ItemClass.MainHand || Class == ItemClass.OffHand) { var nameObj = (((RavenJArray)val["properties"])[0] as RavenJObject)["name"]; WeaponType = nameObj != null ? nameObj.ToString() : ""; } else if (Class == ItemClass.Gem) { var nameObj = (((RavenJArray)val["properties"])[0] as RavenJObject)["name"]; KeyWords = nameObj != null ? nameObj.ToString() : ""; } ReadProperties(val); } else if (Class == ItemClass.OffHand) WeaponType = "Quiver"; if (val.ContainsKey("requirements") && Class == ItemClass.Gem) { foreach (RavenJObject obj in (RavenJArray)val["requirements"]) { string name = obj["name"].Value<string>(); List<float> values = new List<float>(); foreach (RavenJArray jva in (RavenJArray)obj["values"]) { values.Add(jva[0].Value<float>()); } Requirements.Add(name, values); } } //if item has explicit Mods we add it to it's Mods list if (val.ContainsKey("explicitMods")) foreach (string s in val["explicitMods"].Values<string>()) { Mod modifier = new Mod(s.Replace("Additional ", ""), this.Class, WeaponType == "Quiver"); Mods.Add(modifier); } //if item has implicit Mods we add it to it's Mods list if (val.ContainsKey("implicitMods")) foreach (string s in val["implicitMods"].Values<string>()) { Mod modifier = new Mod(s.Replace("Additional ", ""), this.Class, WeaponType == "Quiver"); Mods.Add(modifier); } if (val.ContainsKey("sockets")) { Sockets = new Dictionary<int, List<SocketColor>>(); ReadSockets(val); } if (val.ContainsKey("socketedItems")) ReadGems(val); if (val.ContainsKey("secDescrText")) Description = val["secDescrText"].Value<string>(); if (val.ContainsKey("support")) IsSupport = val["support"].Value<bool>(); }
public MetadataUpdateResult UpdateFileMetadata(string filename, RavenJObject metadata, Etag etag) { Api.JetSetCurrentIndex(session, Files, "by_name"); Api.MakeKey(session, Files, filename, Encoding.Unicode, MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Files, SeekGrbit.SeekEQ) == false) throw new FileNotFoundException(filename); using (var update = new Update(session, Files, JET_prep.Replace)) { var existingEtag = EnsureFileEtagMatch(filename, etag); metadata.Remove(Constants.MetadataEtagField); var existingMetadata = RetrieveMetadata(); if (!metadata.ContainsKey("Content-MD5") && existingMetadata.ContainsKey("Content-MD5")) metadata["Content-MD5"] = existingMetadata["Content-MD5"]; if (!metadata.ContainsKey(Constants.FileSystem.RavenFsSize) && existingMetadata.ContainsKey(Constants.FileSystem.RavenFsSize)) metadata[Constants.FileSystem.RavenFsSize] = existingMetadata[Constants.FileSystem.RavenFsSize]; var newEtag = uuidGenerator.CreateSequentialUuid(); Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["etag"], newEtag.TransformToValueForEsentSorting()); Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["metadata"], ToQueryString(metadata), Encoding.Unicode); update.Save(); return new MetadataUpdateResult { PrevEtag = existingEtag, Etag = newEtag }; } }
public FileUpdateResult UpdateFileMetadata(string filename, RavenJObject metadata, Etag etag) { var key = CreateKey(filename); var keySlice = (Slice)key; ushort version; var file = LoadJson(storage.Files, keySlice, writeBatch.Value, out version); if (file == null) throw new FileNotFoundException(filename); var existingEtag = EnsureDocumentEtagMatch(filename, etag, file); var newEtag = uuidGenerator.CreateSequentialUuid(); metadata.Remove(RavenConstants.MetadataEtagField); var existingMetadata = (RavenJObject) file["metadata"]; if (!metadata.ContainsKey("Content-MD5") && existingMetadata.ContainsKey("Content-MD5")) metadata["Content-MD5"] = existingMetadata["Content-MD5"]; if (!metadata.ContainsKey(RavenConstants.FileSystem.RavenFsSize) && existingMetadata.ContainsKey(RavenConstants.FileSystem.RavenFsSize)) metadata[RavenConstants.FileSystem.RavenFsSize] = existingMetadata[RavenConstants.FileSystem.RavenFsSize]; file["etag"] = newEtag.ToByteArray(); file["metadata"] = metadata; storage.Files.Add(writeBatch.Value, keySlice, file, version); var filesByEtag = storage.Files.GetIndex(Tables.Files.Indices.ByEtag); filesByEtag.Delete(writeBatch.Value, CreateKey(existingEtag)); filesByEtag.Add(writeBatch.Value, (Slice)CreateKey(newEtag), key); return new FileUpdateResult() { PrevEtag = existingEtag, Etag = newEtag }; }
public async Task PutAsync(string name, Etag etag, RavenJObject metadata, Func <Task <Stream> > streamAsync, PutOperationOptions options) { FileUpdateResult putResult = null; try { FileSystem.MetricsCounters.FilesPerSecond.Mark(); name = FileHeader.Canonize(name); if (options.PreserveTimestamps) { if (!metadata.ContainsKey(Constants.RavenCreationDate)) { if (metadata.ContainsKey(Constants.CreationDate)) { metadata[Constants.RavenCreationDate] = metadata[Constants.CreationDate]; } else { throw new InvalidOperationException("Preserve Timestamps requires that the client includes the Raven-Creation-Date header."); } } Historian.UpdateLastModified(metadata, options.LastModified.HasValue ? options.LastModified.Value : DateTimeOffset.UtcNow); } else { metadata[Constants.RavenCreationDate] = DateTimeOffset.UtcNow; Historian.UpdateLastModified(metadata); } // TODO: To keep current filesystems working. We should remove when adding a new migration. metadata[Constants.CreationDate] = metadata[Constants.RavenCreationDate].Value <DateTimeOffset>().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture); Historian.Update(name, metadata); long?size = -1; Storage.Batch(accessor => { FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name); AssertPutOperationNotVetoed(name, metadata); SynchronizationTask.Cancel(name); var contentLength = options.ContentLength; var contentSize = options.ContentSize; if (contentLength == 0 || contentSize.HasValue == false) { size = contentLength; if (options.TransferEncodingChunked) { size = null; } } else { size = contentSize; } FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(name, metadata)); using (FileSystem.DisableAllTriggersForCurrentThread()) { IndicateFileToDelete(name, etag); } putResult = accessor.PutFile(name, size, metadata); FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(name, size, metadata)); Search.Index(name, metadata, putResult.Etag); }); Log.Debug("Inserted a new file '{0}' with ETag {1}", name, putResult.Etag); using (var contentStream = await streamAsync().ConfigureAwait(false)) using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, FileSystem.PutTriggers, contentStream, name, metadata)) { await readFileToDatabase.Execute().ConfigureAwait(false); if (size != null && readFileToDatabase.TotalSizeRead != size) { throw new HttpResponseException(HttpStatusCode.BadRequest); } if (options.PreserveTimestamps == false) { Historian.UpdateLastModified(metadata); // update with the final file size. } Log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name); metadata["Content-MD5"] = readFileToDatabase.FileHash; FileUpdateResult updateMetadata = null; Storage.Batch(accessor => updateMetadata = accessor.UpdateFileMetadata(name, metadata, null)); long totalSizeRead = readFileToDatabase.TotalSizeRead; metadata["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture); Search.Index(name, metadata, updateMetadata.Etag); Publisher.Publish(new FileChangeNotification { Action = FileChangeAction.Add, File = name }); Log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, updateMetadata.Etag); } } catch (Exception ex) { if (putResult != null) { using (FileSystem.DisableAllTriggersForCurrentThread()) { IndicateFileToDelete(name, null); } } Log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex); throw; } }
internal static void EnsureReplicationInformationInMetadata(RavenJObject metadata, DocumentDatabase database) { Debug.Assert(database != null); if (metadata == null) return; if (metadata.ContainsKey(Constants.RavenReplicationSource)) return; metadata[Constants.RavenReplicationHistory] = new RavenJArray(); metadata[Constants.RavenReplicationVersion] = 0; metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(database.TransactionalStorage.Id); }
public async Task PutAsync(string name, Etag etag, RavenJObject metadata, Func<Task<Stream>> streamAsync, PutOperationOptions options) { FileUpdateResult putResult = null; try { FileSystem.MetricsCounters.FilesPerSecond.Mark(); name = FileHeader.Canonize(name); if (options.PreserveTimestamps) { if (!metadata.ContainsKey(Constants.RavenCreationDate)) { if (metadata.ContainsKey(Constants.CreationDate)) metadata[Constants.RavenCreationDate] = metadata[Constants.CreationDate]; else throw new InvalidOperationException("Preserve Timestamps requires that the client includes the Raven-Creation-Date header."); } Historian.UpdateLastModified(metadata, options.LastModified.HasValue ? options.LastModified.Value : DateTimeOffset.UtcNow); } else { metadata[Constants.RavenCreationDate] = DateTimeOffset.UtcNow; Historian.UpdateLastModified(metadata); } // TODO: To keep current filesystems working. We should remove when adding a new migration. metadata[Constants.CreationDate] = metadata[Constants.RavenCreationDate].Value<DateTimeOffset>().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture); Historian.Update(name, metadata); long? size = -1; Storage.Batch(accessor => { FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name); AssertPutOperationNotVetoed(name, metadata); SynchronizationTask.Cancel(name); var contentLength = options.ContentLength; var contentSize = options.ContentSize; if (contentLength == 0 || contentSize.HasValue == false) { size = contentLength; if (options.TransferEncodingChunked) size = null; } else { size = contentSize; } FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(name, metadata)); using (FileSystem.DisableAllTriggersForCurrentThread()) { IndicateFileToDelete(name, etag); } putResult = accessor.PutFile(name, size, metadata); FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(name, size, metadata)); Search.Index(name, metadata, putResult.Etag); }); Log.Debug("Inserted a new file '{0}' with ETag {1}", name, putResult.Etag); using (var contentStream = await streamAsync().ConfigureAwait(false)) using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, FileSystem.PutTriggers, contentStream, name, metadata)) { await readFileToDatabase.Execute().ConfigureAwait(false); if (size != null && readFileToDatabase.TotalSizeRead != size) { throw new HttpResponseException(HttpStatusCode.BadRequest); } if (options.PreserveTimestamps == false) Historian.UpdateLastModified(metadata); // update with the final file size. Log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name); metadata["Content-MD5"] = readFileToDatabase.FileHash; FileUpdateResult updateMetadata = null; Storage.Batch(accessor => updateMetadata = accessor.UpdateFileMetadata(name, metadata, null)); long totalSizeRead = readFileToDatabase.TotalSizeRead; metadata["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture); Search.Index(name, metadata, updateMetadata.Etag); Publisher.Publish(new FileChangeNotification { Action = FileChangeAction.Add, File = name }); Log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, updateMetadata.Etag); } } catch (Exception ex) { if (putResult != null) { using (FileSystem.DisableAllTriggersForCurrentThread()) { IndicateFileToDelete(name, null); } } Log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex); throw; } }
public void PutFile(string filename, long? totalSize, RavenJObject metadata, bool tombstone = false) { using (var update = new Update(session, Files, JET_prep.Insert)) { Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["name"], filename, Encoding.Unicode); if (totalSize != null) Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["total_size"], BitConverter.GetBytes(totalSize.Value)); Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["uploaded_size"], BitConverter.GetBytes(0)); if (!metadata.ContainsKey("ETag")) throw new InvalidOperationException(string.Format("Metadata of file {0} does not contain 'ETag' key", filename)); var innerEsentMetadata = new RavenJObject(metadata); var etag = innerEsentMetadata.Value<Guid>("ETag"); innerEsentMetadata.Remove("ETag"); Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["etag"], etag.TransformToValueForEsentSorting()); Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["metadata"], ToQueryString(innerEsentMetadata), Encoding.Unicode); update.Save(); } if (!tombstone) { if (Api.TryMoveFirst(session, Details) == false) throw new InvalidOperationException("Could not find system metadata row"); Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["file_count"], 1); } }
public bool FilterDocuments(string destinationId, string key, RavenJObject metadata, out string reason) { if (IsSystemDocumentId(key)) { reason = string.Format("Will not replicate document '{0}' to '{1}' because it is a system document", key, destinationId); if (Log.IsDebugEnabled) { Log.Debug(reason); } return(false); } if (metadata.ContainsKey(Constants.NotForReplication) && metadata.Value <bool>(Constants.NotForReplication)) // not explicitly marked to skip { reason = string.Format("Will not replicate document '{0}' to '{1}' because it was marked as not for replication", key, destinationId); if (Log.IsDebugEnabled) { Log.Debug(reason); } return(false); } if (metadata[Constants.RavenReplicationConflict] != null) // don't replicate conflicted documents, that just propagate the conflict { reason = string.Format("Will not replicate document '{0}' to '{1}' because it a conflict document", key, destinationId); if (Log.IsDebugEnabled) { Log.Debug(reason); } return(false); } if (OriginsFromDestination(destinationId, metadata)) // prevent replicating back to source { reason = string.Format("Will not replicate document '{0}' to '{1}' because the destination server is the same server it originated from", key, destinationId); if (Log.IsDebugEnabled) { Log.Debug(reason); } return(false); } switch (ReplicationOptionsBehavior) { case TransitiveReplicationOptions.None: var value = metadata.Value <string>(Constants.RavenReplicationSource); if (value != null && (value != CurrentDatabaseId)) { reason = string.Format("Will not replicate document '{0}' to '{1}' because it was not created on the current server, and TransitiveReplicationOptions = none", key, destinationId); if (Log.IsDebugEnabled) { Log.Debug(reason); } return(false); } break; } reason = string.Format("Will replicate '{0}' to '{1}'", key, destinationId); if (Log.IsDebugEnabled) { Log.Debug(reason); } return(true); }
public SynchronizationWorkItem DetermineWork(string file, RavenJObject localMetadata, RavenJObject destinationMetadata, string localServerUrl, out NoSyncReason reason) { reason = NoSyncReason.Unknown; if (localMetadata == null) { reason = NoSyncReason.SourceFileNotExist; return(null); } if (destinationMetadata != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflictResolution] == null) { reason = NoSyncReason.DestinationFileConflicted; return(null); } if (localMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null) { reason = NoSyncReason.SourceFileConflicted; return(null); } if (localMetadata[SynchronizationConstants.RavenDeleteMarker] != null) { if (localMetadata.ContainsKey(SynchronizationConstants.RavenRenameFile)) { var rename = localMetadata.Value <string>(SynchronizationConstants.RavenRenameFile); if (destinationMetadata != null) { return(new RenameWorkItem(file, rename, localServerUrl, storage)); } return(new ContentUpdateWorkItem(rename, localServerUrl, storage, sigGenerator)); // we have a rename tombstone but file does not exists on destination } return(new DeleteWorkItem(file, localServerUrl, storage)); } if (destinationMetadata != null && Historian.IsDirectChildOfCurrent(localMetadata, destinationMetadata)) { reason = NoSyncReason.ContainedInDestinationHistory; return(null); } // file exists on dest and has the same content if (destinationMetadata != null && localMetadata.Value <string>("Content-MD5") == destinationMetadata.Value <string>("Content-MD5")) { // check metadata to detect if any synchronization is needed if (localMetadata.Keys.Except(new[] { Constants.MetadataEtagField, Constants.RavenLastModified, Constants.LastModified }) .Any(key => !destinationMetadata.ContainsKey(key) || localMetadata[key] != destinationMetadata[key])) { return(new MetadataUpdateWorkItem(file, localServerUrl, destinationMetadata, storage)); } reason = NoSyncReason.SameContentAndMetadata; return(null); // the same content and metadata - no need to synchronize } return(new ContentUpdateWorkItem(file, localServerUrl, storage, sigGenerator)); }
protected void WriteHeaders(RavenJObject headers, Etag etag, HttpResponseMessage msg) { foreach (var header in headers) { if (header.Key.StartsWith("@")) { continue; } switch (header.Key) { case "Content-Type": var headerValue = header.Value.Value <string>(); string charset = null; if (headerValue.Contains("charset=")) { var splits = headerValue.Split(';'); headerValue = splits[0]; charset = splits[1].Split('=')[1]; } msg.Content.Headers.ContentType = new MediaTypeHeaderValue(headerValue) { CharSet = charset }; break; default: if (header.Value.Type == JTokenType.Date) { if (header.Key.StartsWith("Raven-")) { var iso8601 = GetDateString(header.Value, "o"); msg.Content.Headers.Add(header.Key, iso8601); } else { var rfc1123 = GetDateString(header.Value, "r"); msg.Content.Headers.Add(header.Key, rfc1123); if (!headers.ContainsKey("Raven-" + header.Key)) { var iso8601 = GetDateString(header.Value, "o"); msg.Content.Headers.Add("Raven-" + header.Key, iso8601); } } } else if (header.Value.Type == JTokenType.Boolean) { msg.Content.Headers.Add(header.Key, header.Value.ToString()); } else { //headers do not need url decoding because they might contain special symbols (like + symbol in clr type) var value = UnescapeStringIfNeeded(header.Value.ToString(Formatting.None), shouldDecodeUrl: false); msg.Content.Headers.Add(header.Key, value); } break; } } if (headers["@Http-Status-Code"] != null) { msg.StatusCode = (HttpStatusCode)headers.Value <int>("@Http-Status-Code"); msg.Content.Headers.Add("Temp-Status-Description", headers.Value <string>("@Http-Status-Description")); } WriteETag(etag, msg); }
public Item(ItemClass iClass, RavenJObject val) { Type = ""; Attributes = new Dictionary<string, List<float>>(); Mods = new List<Mod>(); Class = iClass; if (iClass != ItemClass.Gem) { Gems = new List<Item>(); } Name = val["typeLine"].Value<string>(); if (val.ContainsKey("properties")) foreach (RavenJObject obj in (RavenJArray)val["properties"]) { List<float> values = new List<float>(); string s = ""; foreach (RavenJArray jva in (RavenJArray)obj["values"]) { s += " "+jva[0].Value<string>() ; } if (s == "") continue; foreach (Match m in numberfilter.Matches(s)) { if (m.Value == "") values.Add(float.NaN); else values.Add(float.Parse(m.Value, System.Globalization.CultureInfo.InvariantCulture)); } string cs = obj["name"].Value<string>() + ": " + (numberfilter.Replace(s, "#")); Attributes.Add(cs, values); } if (val.ContainsKey("explicitMods")) foreach (string s in val["explicitMods"].Values<string>()) { var mods = Mod.CreateMods(s.Replace("Additional ", ""), this.Class); Mods.AddRange(mods); } if (val.ContainsKey("implicitMods")) foreach (string s in val["implicitMods"].Values<string>()) { var mods = Mod.CreateMods(s.Replace("Additional ", ""), this.Class); Mods.AddRange(mods); } }
private void ReplicateDelete(string id, RavenJObject metadata, TExternal incoming) { TInternal existingItem; Guid existingEtag; var existingMetadata = TryGetExisting(id, out existingItem, out existingEtag); if (existingMetadata == null) { log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring", id, Src); return; } if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict)) // locally conflicted { log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src); var savedConflictedItemId = SaveConflictedItem(id, metadata, incoming, existingEtag); AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem); return; } if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker)) //deleted locally as well { log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories", id, Src); var existingHistory = existingMetadata.Value <RavenJArray>(Constants.RavenReplicationHistory) ?? new RavenJArray(); var newHistory = metadata.Value <RavenJArray>(Constants.RavenReplicationHistory) ?? new RavenJArray(); foreach (var item in newHistory) { existingHistory.Add(item); } if (metadata.ContainsKey(Constants.RavenReplicationVersion) && metadata.ContainsKey(Constants.RavenReplicationSource)) { existingHistory.Add(new RavenJObject { { Constants.RavenReplicationVersion, metadata[Constants.RavenReplicationVersion] }, { Constants.RavenReplicationSource, metadata[Constants.RavenReplicationSource] } }); } while (existingHistory.Length > Constants.ChangeHistoryLength) { existingHistory.RemoveAt(0); } MarkAsDeleted(id, metadata); return; } if (IsDirectChildOfCurrent(metadata, existingMetadata)) // not modified { log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src); DeleteItem(id, existingEtag); MarkAsDeleted(id, metadata); return; } Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSyncronization(() => Database.RaiseNotifications(new DocumentChangeNotification { Name = id, Type = DocumentChangeTypes.ReplicationConflict })); var newConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag); log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src); // we have a new conflict move the existing doc to a conflict and create a conflict document var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag); CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata); }
public void Replicate(string id, RavenJObject metadata, TExternal incoming) { if (metadata.Value <bool>(Constants.RavenDeleteMarker)) { ReplicateDelete(id, metadata, incoming); return; } TInternal existingItem; Etag existingEtag; bool deleted; RavenJObject existingMetadata; try { existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted); } catch (Exception e) { log.ErrorException(string.Format("Replication - fetching existing item failed. (key = {0})", id), e); throw new InvalidOperationException("Replication - fetching existing item failed. (key = " + id + ")", e); } if (existingMetadata == null) { AddWithoutConflict(id, null, metadata, incoming); if (log.IsDebugEnabled) { log.Debug("New item {0} replicated successfully from {1}", id, Src); } return; } // we just got the same version from the same source - request playback again? // at any rate, not an error, moving on if (existingMetadata.Value <string>(Constants.RavenReplicationSource) == metadata.Value <string>(Constants.RavenReplicationSource) && existingMetadata.Value <long>(Constants.RavenReplicationVersion) == metadata.Value <long>(Constants.RavenReplicationVersion)) { return; } var existingDocumentIsInConflict = existingMetadata[Constants.RavenReplicationConflict] != null; if (existingDocumentIsInConflict == false && // if the current document is not in conflict, we can continue without having to keep conflict semantics (Historian.IsDirectChildOfCurrent(metadata, existingMetadata))) // this update is direct child of the existing doc, so we are fine with overwriting this { var etag = deleted == false ? existingEtag : null; AddWithoutConflict(id, etag, metadata, incoming); if (log.IsDebugEnabled) { log.Debug("Existing item {0} replicated successfully from {1}", id, Src); } return; } // this is the case where the incoming metadata is an older version of the metadata we have. // this can happen when multiple sources send data with diffrent latencies if (existingDocumentIsInConflict == false && Historian.IsDirectChildOfCurrent(existingMetadata, metadata)) { return; } RavenJObject resolvedMetadataToSave; TExternal resolvedItemToSave; if (TryResolveConflict(id, metadata, incoming, existingItem, out resolvedMetadataToSave, out resolvedItemToSave)) { if (metadata.ContainsKey("Raven-Remove-Document-Marker") && metadata.Value <bool>("Raven-Remove-Document-Marker")) { if (resolvedMetadataToSave.ContainsKey(Constants.RavenEntityName)) { metadata[Constants.RavenEntityName] = resolvedMetadataToSave[Constants.RavenEntityName]; } DeleteItem(id, null); MarkAsDeleted(id, metadata); } else { var etag = deleted == false ? existingEtag : null; var resolvedItemJObject = resolvedItemToSave as RavenJObject; if (resolvedItemJObject != null) { ExecuteRemoveConflictOnPutTrigger(id, metadata, resolvedItemJObject); } AddWithoutConflict(id, etag, resolvedMetadataToSave, resolvedItemToSave); } return; } //this is expensive but worth trying if we can avoid conflicts if (TryResolveConflictByCheckingIfIdentical(id, metadata, incoming, existingItem, out resolvedMetadataToSave)) { //The metadata here is merged (changed), it needs to be pushed. AddWithoutConflict(id, null, resolvedMetadataToSave, incoming); return; } CreatedConflict createdConflict; var newDocumentConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag); if (existingDocumentIsInConflict) // the existing document is in conflict { if (log.IsDebugEnabled) { log.Debug("Conflicted item {0} has a new version from {1}, adding to conflicted documents", id, Src); } createdConflict = AppendToCurrentItemConflicts(id, newDocumentConflictId, existingMetadata, existingItem); } else { if (log.IsDebugEnabled) { log.Debug("Existing item {0} is in conflict with replicated version from {1}, marking item as conflicted", id, Src); } // we have a new conflict // move the existing doc to a conflict and create a conflict document var existingDocumentConflictId = id + "/conflicts/" + GetReplicationIdentifierForCurrentDatabase(); createdConflict = CreateConflict(id, newDocumentConflictId, existingDocumentConflictId, existingItem, existingMetadata); } Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new ReplicationConflictNotification() { Id = id, Etag = createdConflict.Etag, ItemType = ReplicationConflict, OperationType = ReplicationOperationTypes.Put, Conflicts = createdConflict.ConflictedIds })); }
public void UpdateFileMetadata(string filename, RavenJObject metadata) { Api.JetSetCurrentIndex(session, Files, "by_name"); Api.MakeKey(session, Files, filename, Encoding.Unicode, MakeKeyGrbit.NewKey); if (Api.TrySeek(session, Files, SeekGrbit.SeekEQ) == false) throw new FileNotFoundException(filename); using (var update = new Update(session, Files, JET_prep.Replace)) { if (!metadata.ContainsKey("ETag")) { throw new InvalidOperationException("Metadata of file {0} does not contain 'ETag' key " + filename); } var innerEsentMetadata = new RavenJObject(metadata); var etag = innerEsentMetadata.Value<Guid>("ETag"); innerEsentMetadata.Remove("ETag"); var existingMetadata = RetrieveMetadata(); if (existingMetadata.ContainsKey("Content-MD5")) { innerEsentMetadata["Content-MD5"] = existingMetadata["Content-MD5"]; } Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["etag"], etag.TransformToValueForEsentSorting()); Api.SetColumn(session, Files, tableColumnsCache.FilesColumns["metadata"], ToQueryString(innerEsentMetadata), Encoding.Unicode); update.Save(); } }
private void ReplicateDelete(string id, RavenJObject newMetadata, TExternal incoming) { TInternal existingItem; Etag existingEtag; bool deleted; var existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted); if (existingMetadata == null) { if (log.IsDebugEnabled) { log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring.", id, Src); } return; } if (existingMetadata.ContainsKey(Constants.RavenEntityName)) { newMetadata[Constants.RavenEntityName] = existingMetadata[Constants.RavenEntityName]; } RavenJObject currentReplicationEntry = null; if (newMetadata.ContainsKey(Constants.RavenReplicationVersion) && newMetadata.ContainsKey(Constants.RavenReplicationSource)) { currentReplicationEntry = new RavenJObject { { Constants.RavenReplicationVersion, newMetadata[Constants.RavenReplicationVersion] }, { Constants.RavenReplicationSource, newMetadata[Constants.RavenReplicationSource] } }; } var existingHistory = ReplicationData.GetOrCreateHistory(existingMetadata); if (currentReplicationEntry != null && existingHistory.Any(x => RavenJTokenEqualityComparer.Default.Equals( ((RavenJObject)x)[Constants.RavenReplicationSource], currentReplicationEntry[Constants.RavenReplicationSource]) && ((RavenJObject)x)[Constants.RavenReplicationVersion].Value <long>() >= currentReplicationEntry[Constants.RavenReplicationVersion].Value <long>())) { if (log.IsDebugEnabled) { log.Debug("Replicated delete for {0} already exist in item history, ignoring", id); } return; } if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker)) //deleted locally as well { if (log.IsDebugEnabled) { log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories.", id, Src); } var newHistory = ReplicationData.GetOrCreateHistory(newMetadata); if (currentReplicationEntry != null) { newHistory.Add(currentReplicationEntry); } //Merge histories ReplicationData.SetHistory(newMetadata, Historian.MergeReplicationHistories(newHistory, existingHistory, id)); newMetadata[Constants.RavenReplicationMergedHistory] = true; MarkAsDeleted(id, newMetadata); return; } if (Historian.IsDirectChildOfCurrent(newMetadata, existingMetadata)) // not modified { if (log.IsDebugEnabled) { log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src); } DeleteItem(id, existingEtag); MarkAsDeleted(id, newMetadata); return; } CreatedConflict createdConflict; if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict)) // locally conflicted { if (log.IsDebugEnabled) { log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src); } var savedConflictedItemId = SaveConflictedItem(id, newMetadata, incoming, existingEtag); createdConflict = AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem); } else { RavenJObject resolvedMetadataToSave; TExternal resolvedItemToSave; if (TryResolveConflict(id, newMetadata, incoming, existingItem, out resolvedMetadataToSave, out resolvedItemToSave)) { AddWithoutConflict(id, existingEtag, resolvedMetadataToSave, resolvedItemToSave); return; } var newConflictId = SaveConflictedItem(id, newMetadata, incoming, existingEtag); if (log.IsDebugEnabled) { log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src); } // we have a new conflict move the existing doc to a conflict and create a conflict document var existingDocumentConflictId = id + "/conflicts/" + GetReplicationIdentifierForCurrentDatabase(); createdConflict = CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata); } Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new ReplicationConflictNotification() { Id = id, Etag = createdConflict.Etag, Conflicts = createdConflict.ConflictedIds, ItemType = ReplicationConflictTypes.DocumentReplicationConflict, OperationType = ReplicationOperationTypes.Delete })); }
public void PutFile(string filename, long? totalSize, RavenJObject metadata, bool tombstone = false) { var filesByEtag = storage.Files.GetIndex(Tables.Files.Indices.ByEtag); var key = CreateKey(filename); if (!metadata.ContainsKey("ETag")) throw new InvalidOperationException(string.Format("Metadata of file {0} does not contain 'ETag' key", filename)); ushort version; var existingFile = LoadJson(storage.Files, key, writeBatch.Value, out version); var innerMetadata = new RavenJObject(metadata); var etag = innerMetadata.Value<Guid>("ETag"); innerMetadata.Remove("ETag"); var file = new RavenJObject { { "name", filename }, { "total_size", totalSize }, { "uploaded_size", 0 }, { "etag", new RavenJValue(etag) }, { "metadata", innerMetadata } }; storage.Files.Add(writeBatch.Value, key, file, version); if (existingFile != null) { filesByEtag.Delete(writeBatch.Value, CreateKey(existingFile.Value<Guid>("etag"))); } filesByEtag.Add(writeBatch.Value, CreateKey(etag), key); if (tombstone) return; var fileCount = storage.Files.GetIndex(Tables.Files.Indices.Count); fileCount.Add(writeBatch.Value, key, key); }
protected void InsertItems(string tableName, string pkName, IEnumerable<ItemToReplicate> dataForTable) { tableName = tableName.ToLowerInvariant(); // type names have to be lowercase foreach (var itemToReplicate in dataForTable) { var o = new RavenJObject(); if (database != null) database.WorkContext.CancellationToken.ThrowIfCancellationRequested(); foreach (var column in itemToReplicate.Columns.Where(column => column.Key != pkName)) { o[column.Key] = column.Value; } if ("_id".Equals(pkName)) { bulkCommands.Add("{\"index\":{\"_index\":\"" + targetIndexName + "\",\"_type\":\"" + tableName + "\",\"_id\":\"" + itemToReplicate.DocumentId + "\"}}"); } else { o[pkName] = itemToReplicate.DocumentId; bulkCommands.Add("{\"index\":{\"_index\":\"" + targetIndexName + "\",\"_type\":\"" + tableName + "\"}}"); } // Setup timestamps, converting from a Javascript notation to an ES/Kibana notation if (o.ContainsKey("$timestamp")) { o["@timestamp"] = o["$timestamp"]; o.Remove("$timestamp"); } else { o["@timestamp"] = DateTime.UtcNow; } bulkCommands.Add(o.ToString(Formatting.None)); } }
public AddDocumentResult AddDocument(string key, Etag etag, RavenJObject data, RavenJObject metadata, InvokeSource source = InvokeSource.Default) { if (key == null) { throw new ArgumentNullException("key"); } var byteCount = Encoding.Unicode.GetByteCount(key); if (byteCount >= 2048) { throw new ArgumentException(string.Format("The key must be a maximum of 2,048 bytes in Unicode, 1,024 characters, key is: '{0}'", key), "key"); } try { Api.JetSetCurrentIndex(session, Documents, "by_key"); Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey); var isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ); Etag existingEtag = null; if (isUpdate) { existingEtag = EnsureDocumentEtagMatch(key, etag, "PUT"); } else { if (etag != null && etag != Etag.Empty) // expected something to be there. { throw new ConcurrencyException("PUT attempted on document '" + key + "' using a non current etag (document deleted)") { ExpectedETag = etag } } ; if (Api.TryMoveFirst(session, Details)) { Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["document_count"], 1); } } Etag newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents); DateTime savedAt; try { using (var update = new Update(session, Documents, isUpdate ? JET_prep.Replace : JET_prep.Insert)) { Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode); using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"])) { if (isUpdate) { columnStream.SetLength(0); // empty the existing value, since we are going to overwrite the entire thing } using (Stream stream = new BufferedStream(columnStream)) using ( var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current)) ) { data.WriteTo(finalStream); finalStream.Flush(); } } Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"], newEtag.TransformToValueForEsentSorting()); var keepLastModified = source == InvokeSource.FromConflictAtReplication && metadata.ContainsKey(Constants.LastModified); savedAt = keepLastModified ? metadata.Value <DateTime>(Constants.LastModified) : SystemTime.UtcNow; Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt.ToBinary()); using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"])) { if (isUpdate) { columnStream.SetLength(0); } using (Stream stream = new BufferedStream(columnStream)) { metadata.WriteTo(stream); stream.Flush(); } } update.Save(); } } catch (EsentErrorException e) { if (e.Error == JET_err.KeyDuplicate || e.Error == JET_err.WriteConflict) { throw new ConcurrencyException("PUT attempted on document '" + key + "' concurrently", e); } throw; } if (logger.IsDebugEnabled) { logger.Debug("Inserted a new document with key '{0}', update: {1}, ", key, isUpdate); } if (existingEtag != null) { cacher.RemoveCachedDocument(key, existingEtag); } return(new AddDocumentResult { Etag = newEtag, PrevEtag = existingEtag, SavedAt = savedAt, Updated = isUpdate }); } catch (EsentKeyDuplicateException e) { throw new ConcurrencyException("Illegal duplicate key " + key, e); } }
private void ReplicateDelete(string id, RavenJObject metadata, TExternal incoming) { TInternal existingItem; Etag existingEtag; bool deleted; var existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted); if (existingMetadata == null) { log.Debug("Replicating deleted item {0} from {1} that does not exist, ignoring", id, Src); return; } if (existingMetadata.Value <bool>(Constants.RavenDeleteMarker)) //deleted locally as well { log.Debug("Replicating deleted item {0} from {1} that was deleted locally. Merging histories", id, Src); var existingHistory = new RavenJArray(ReplicationData.GetHistory(existingMetadata)); var newHistory = new RavenJArray(ReplicationData.GetHistory(metadata)); foreach (var item in newHistory) { existingHistory.Add(item); } if (metadata.ContainsKey(Constants.RavenReplicationVersion) && metadata.ContainsKey(Constants.RavenReplicationSource)) { existingHistory.Add(new RavenJObject { { Constants.RavenReplicationVersion, metadata[Constants.RavenReplicationVersion] }, { Constants.RavenReplicationSource, metadata[Constants.RavenReplicationSource] } }); } while (existingHistory.Length > Constants.ChangeHistoryLength) { existingHistory.RemoveAt(0); } MarkAsDeleted(id, metadata); return; } if (Historian.IsDirectChildOfCurrent(metadata, existingMetadata)) // not modified { log.Debug("Delete of existing item {0} was replicated successfully from {1}", id, Src); DeleteItem(id, existingEtag); MarkAsDeleted(id, metadata); return; } CreatedConflict createdConflict; if (existingMetadata.Value <bool>(Constants.RavenReplicationConflict)) // locally conflicted { log.Debug("Replicating deleted item {0} from {1} that is already conflicted, adding to conflicts.", id, Src); var savedConflictedItemId = SaveConflictedItem(id, metadata, incoming, existingEtag); createdConflict = AppendToCurrentItemConflicts(id, savedConflictedItemId, existingMetadata, existingItem); } else { RavenJObject resolvedMetadataToSave; TExternal resolvedItemToSave; if (TryResolveConflict(id, metadata, incoming, existingItem, out resolvedMetadataToSave, out resolvedItemToSave)) { AddWithoutConflict(id, existingEtag, resolvedMetadataToSave, resolvedItemToSave); return; } var newConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag); log.Debug("Existing item {0} is in conflict with replicated delete from {1}, marking item as conflicted", id, Src); // we have a new conflict move the existing doc to a conflict and create a conflict document var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag); createdConflict = CreateConflict(id, newConflictId, existingDocumentConflictId, existingItem, existingMetadata); } Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new ReplicationConflictNotification() { Id = id, Etag = createdConflict.Etag, Conflicts = createdConflict.ConflictedIds, ItemType = ReplicationConflictTypes.DocumentReplicationConflict, OperationType = ReplicationOperationTypes.Delete })); }
public void UpdateFileMetadata(string filename, RavenJObject metadata) { var key = CreateKey(filename); ushort version; var file = LoadJson(storage.Files, key, writeBatch.Value, out version); if (file == null) throw new FileNotFoundException(filename); if (!metadata.ContainsKey("ETag")) throw new InvalidOperationException(string.Format("Metadata of file {0} does not contain 'ETag' key", filename)); var innerMetadata = new RavenJObject(metadata); var etag = innerMetadata.Value<Guid>("ETag"); innerMetadata.Remove("ETag"); var existingMetadata = (RavenJObject) file["metadata"]; if (existingMetadata.ContainsKey("Content-MD5")) innerMetadata["Content-MD5"] = existingMetadata["Content-MD5"]; var oldEtag = file.Value<Guid>("etag"); file["etag"] = new RavenJValue(etag); file["metadata"] = innerMetadata; storage.Files.Add(writeBatch.Value, key, file, version); var filesByEtag = storage.Files.GetIndex(Tables.Files.Indices.ByEtag); filesByEtag.Delete(writeBatch.Value, CreateKey(oldEtag)); filesByEtag.Add(writeBatch.Value, CreateKey(etag), key); }
public void Replicate(string id, RavenJObject metadata, TExternal incoming) { if (metadata.Value <bool>(Constants.RavenDeleteMarker)) { ReplicateDelete(id, metadata, incoming); return; } TInternal existingItem; Etag existingEtag; bool deleted; RavenJObject existingMetadata; try { existingMetadata = TryGetExisting(id, out existingItem, out existingEtag, out deleted); } catch (Exception e) { log.Error("Replication - fetching existing item failed. (key = {0})", id); throw new InvalidOperationException("Replication - fetching existing item failed. (key = " + id + ")", e); } if (existingMetadata == null) { log.Debug("New item {0} replicated successfully from {1}", id, Src); AddWithoutConflict(id, null, metadata, incoming); return; } // we just got the same version from the same source - request playback again? // at any rate, not an error, moving on if (existingMetadata.Value <string>(Constants.RavenReplicationSource) == metadata.Value <string>(Constants.RavenReplicationSource) && existingMetadata.Value <long>(Constants.RavenReplicationVersion) == metadata.Value <long>(Constants.RavenReplicationVersion)) { return; } var existingDocumentIsInConflict = existingMetadata[Constants.RavenReplicationConflict] != null; if (existingDocumentIsInConflict == false && // if the current document is not in conflict, we can continue without having to keep conflict semantics (Historian.IsDirectChildOfCurrent(metadata, existingMetadata))) // this update is direct child of the existing doc, so we are fine with overwriting this { log.Debug("Existing item {0} replicated successfully from {1}", id, Src); var etag = deleted == false ? existingEtag : null; AddWithoutConflict(id, etag, metadata, incoming); return; } RavenJObject resolvedMetadataToSave; TExternal resolvedItemToSave; if (TryResolveConflict(id, metadata, incoming, existingItem, out resolvedMetadataToSave, out resolvedItemToSave)) { if (metadata.ContainsKey("Raven-Remove-Document-Marker") && metadata.Value <bool>("Raven-Remove-Document-Marker")) { DeleteItem(id, null); MarkAsDeleted(id, metadata); } else { var etag = deleted == false ? existingEtag : null; AddWithoutConflict(id, etag, resolvedMetadataToSave, resolvedItemToSave); } return; } CreatedConflict createdConflict; var newDocumentConflictId = SaveConflictedItem(id, metadata, incoming, existingEtag); if (existingDocumentIsInConflict) // the existing document is in conflict { log.Debug("Conflicted item {0} has a new version from {1}, adding to conflicted documents", id, Src); createdConflict = AppendToCurrentItemConflicts(id, newDocumentConflictId, existingMetadata, existingItem); } else { log.Debug("Existing item {0} is in conflict with replicated version from {1}, marking item as conflicted", id, Src); // we have a new conflict // move the existing doc to a conflict and create a conflict document var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingEtag); createdConflict = CreateConflict(id, newDocumentConflictId, existingDocumentConflictId, existingItem, existingMetadata); } Database.TransactionalStorage.ExecuteImmediatelyOrRegisterForSynchronization(() => Database.Notifications.RaiseNotifications(new ReplicationConflictNotification() { Id = id, Etag = createdConflict.Etag, ItemType = ReplicationConflict, OperationType = ReplicationOperationTypes.Put, Conflicts = createdConflict.ConflictedIds })); }
public Item(ItemClass iClass, RavenJObject val) { Attributes = new Dictionary <string, List <float> >(); Mods = new List <ItemMod>(); Class = iClass; Name = val["name"].Value <string>(); if (Name == "") { Name = val["typeLine"].Value <string>(); } Type = val["typeLine"].Value <string>(); if (val.ContainsKey("properties")) { foreach (RavenJObject obj in (RavenJArray)val["properties"]) { var values = new List <float>(); string s = ""; foreach (RavenJArray jva in (RavenJArray)obj["values"]) { s += " " + jva[0].Value <string>(); } s = s.TrimStart(); if (s == "") { Keywords = new List <string>(); string[] sl = obj["name"].Value <string>().Split(','); foreach (string i in sl) { Keywords.Add(i.Trim()); } continue; } foreach (Match m in numberfilter.Matches(s)) { if (m.Value == "") { values.Add(float.NaN); } else { values.Add(float.Parse(m.Value, CultureInfo.InvariantCulture)); } } string cs = obj["name"].Value <string>() + ": " + (numberfilter.Replace(s, "#")); Attributes.Add(cs, values); } } if (val.ContainsKey("explicitMods")) { foreach (string s in val["explicitMods"].Values <string>()) { List <ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter); Mods.AddRange(mods); } } if (val.ContainsKey("implicitMods")) { foreach (string s in val["implicitMods"].Values <string>()) { List <ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter); Mods.AddRange(mods); } } if (val.ContainsKey("craftedMods")) { foreach (string s in val["craftedMods"].Values <string>()) { List <ItemMod> mods = ItemMod.CreateMods(this, s.Replace("Additional ", ""), numberfilter); Mods.AddRange(mods); } } if (iClass == ItemClass.Gem) { switch (val["colour"].Value <string>()) { case "S": Keywords.Add("Strength"); break; case "D": Keywords.Add("Dexterity"); break; case "I": Keywords.Add("Intelligence"); break; } } else { Gems = new List <Item>(); } var Sockets = new List <int>(); if (val.ContainsKey("sockets")) { foreach (RavenJObject obj in (RavenJArray)val["sockets"]) { Sockets.Add(obj["group"].Value <int>()); } } if (val.ContainsKey("socketedItems")) { int socket = 0; foreach (RavenJObject obj in (RavenJArray)val["socketedItems"]) { var item = new Item(ItemClass.Gem, obj); item.SocketGroup = Sockets[socket++]; Gems.Add(item); } } }