public override ReadVetoResult AllowRead(string name, RavenJObject metadata, ReadOperation operation) { if (metadata.Value<bool>(SynchronizationConstants.RavenDeleteMarker)) return ReadVetoResult.Ignore; return ReadVetoResult.Allowed; }
public void RaiseNotifications(DocumentChangeNotification obj, RavenJObject metadata) { Database.TransportState.Send(obj); var onDocumentChange = OnDocumentChange; if (onDocumentChange != null) onDocumentChange(Database, obj, metadata); }
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data) { Etag etag = uuidGenerator.CreateSequentialUuid(UuidType.MappedResults); using (var update = new Update(session, MappedResults, JET_prep.Insert)) { Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], view, Encoding.Unicode); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["document_key"], docId, Encoding.Unicode); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"], reduceKey, Encoding.Unicode); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["hashed_reduce_key"], HashReduceKey(reduceKey)); var mapBucket = IndexingUtil.MapBucket(docId); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["bucket"], mapBucket); using (Stream stream = new BufferedStream(new ColumnStream(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]))) { using (var dataStream = documentCodecs.Aggregate(stream, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds))) { data.WriteTo(dataStream); dataStream.Flush(); } } Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"], etag.TransformToValueForEsentSorting()); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"], SystemTime.UtcNow.ToBinary()); update.Save(); } }
public override bool TryResolve(string id, RavenJObject metadata, RavenJObject document, JsonDocument existingDoc, Func<string, JsonDocument> getDocument, out RavenJObject metadataToSave, out RavenJObject documentToSave) { var existingDocumentIsInConflict = existingDoc.Metadata[Constants.RavenReplicationConflict] != null; var existingDocumentIsDeleted = existingDoc.Metadata[Constants.RavenDeleteMarker] != null && existingDoc.Metadata[Constants.RavenDeleteMarker].Value<bool>(); metadataToSave = null; documentToSave = null; if (existingDocumentIsInConflict && existingDocumentIsDeleted == false) { var conflictIds = existingDoc.DataAsJson.Value<RavenJArray>("Conflicts") .Select(x => x.Value<string>()) .ToArray(); if (conflictIds.Length == 0) return false; if (conflictIds .Select(getDocument) .Where(x => x != null) .All(doc => Historian.IsDirectChildOfCurrent(metadata, doc.Metadata)) == false) return false; metadataToSave = metadata; documentToSave = document; return true; } return false; }
public override VetoResult AllowPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { return VetoResult.Allowed; }
public override ReadVetoResult AllowRead(string key, Stream data, RavenJObject metadata, ReadOperation operation) { RavenJToken value; if (metadata.TryGetValue("Raven-Delete-Marker", out value)) return ReadVetoResult.Ignore; return ReadVetoResult.Allowed; }
public override void OnPut(string key, Stream data, RavenJObject metadata) { if (key.StartsWith("Raven/")) // we don't deal with system attachment return; using (Database.DisableAllTriggersForCurrentThread()) { var attachmentMetadata = GetAttachmentMetadata(key); if (attachmentMetadata != null) { RavenJArray history = new RavenJArray(metadata.Value<RavenJArray>(Constants.RavenReplicationHistory)); metadata[Constants.RavenReplicationHistory] = history; if (attachmentMetadata.ContainsKey(Constants.RavenReplicationVersion) && attachmentMetadata.ContainsKey(Constants.RavenReplicationSource)) { history.Add(new RavenJObject { {Constants.RavenReplicationVersion, attachmentMetadata[Constants.RavenReplicationVersion]}, {Constants.RavenReplicationSource, attachmentMetadata[Constants.RavenReplicationSource]} }); } if (history.Length > Constants.ChangeHistoryLength) { history.RemoveAt(0); } } metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId()); metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id); } }
public IEnumerable<AbstractField> Index(RavenJObject document, Field.Store defaultStorage) { return from property in document where property.Key != Constants.DocumentIdFieldName from field in CreateFields(property.Key, GetPropertyValue(property.Value), defaultStorage) select field; }
public void Execute(RavenJObject document) { if (Includes == null) return; foreach (var include in Includes) { if (string.IsNullOrEmpty(include)) continue; var path = include; string prefix = null; var match = IncludePrefixRegex.Match(path); if (match.Success && match.Groups.Count >= 2) { prefix = match.Groups[1].Value; path = path.Replace(prefix, ""); prefix = prefix.Substring(1, prefix.Length - 2); } foreach (var token in document.SelectTokenWithRavenSyntaxReturningFlatStructure(path)) { ExecuteInternal(token.Item1, prefix); } } }
private RavenJToken GetTheNewResultOrWireTheDefault(RavenJToken result) { var selectToken = result.SelectToken(Name); if (selectToken != null) return selectToken; return ((RavenJObject)result)[Name] = new RavenJObject(); }
public RavenJObject ToRavenJObject(JsValue jsObject, string propertyKey = null, bool recursiveCall = false) { var objectInstance = jsObject.AsObject(); if (objectInstance.Class == "Function") { // getting a Function instance here, // means that we couldn't evaulate it using Jint return null; } var rjo = new RavenJObject(); foreach (var property in objectInstance.Properties) { if (property.Key == Constants.ReduceKeyFieldName || property.Key == Constants.DocumentIdFieldName) continue; var value = property.Value.Value; if (value.HasValue == false) continue; if (value.Value.IsRegExp()) continue; var recursive = jsObject == value; if (recursiveCall && recursive) rjo[property.Key] = null; else rjo[property.Key] = ToRavenJToken(value.Value, CreatePropertyKey(property.Key, propertyKey), recursive); } return rjo; }
public static RavenJToken DisableVersioning(RavenJObject metadata) { if (metadata != null) metadata.Add(Constants.RavenIgnoreVersioning, true); return metadata; }
public void SetCachedDocument(string key, Etag etag, RavenJObject doc, RavenJObject metadata, int size) { if (skipSettingDocumentInCache) return; var documentClone = ((RavenJObject)doc.CloneToken()); documentClone.EnsureCannotBeChangeAndEnableSnapshotting(); var metadataClone = ((RavenJObject)metadata.CloneToken()); metadataClone.EnsureCannotBeChangeAndEnableSnapshotting(); try { cachedSerializedDocuments.Set("Doc/" + key + "/" + etag, new CachedDocument { Document = documentClone, Metadata = metadataClone, Size = size }, new CacheItemPolicy { SlidingExpiration = configuration.MemoryCacheExpiration, }); } catch (OverflowException) { // this is a bug in the framework // http://connect.microsoft.com/VisualStudio/feedback/details/735033/memorycache-set-fails-with-overflowexception-exception-when-key-is-u7337-u7f01-u2117-exception-message-negating-the-minimum-value-of-a-twos-complement-number-is-invalid // in this case, we just threat it as uncachable } }
private static void EnsureQuotasBundleActivated(RavenJObject settings) { RavenJToken value; if (settings.TryGetValue(Constants.ActiveBundles, out value) == false) settings[Constants.ActiveBundles] = value = new RavenJValue(string.Empty); var activeBundles = value.Value<string>(); var bundles = activeBundles.GetSemicolonSeparatedValues(); if (bundles.Contains("Quotas", StringComparer.OrdinalIgnoreCase) == false) bundles.Add("Quotas"); int hardLimitInKb; if (int.TryParse(ConfigurationManager.AppSettings["Raven/Bundles/LiveTest/Quotas/Size/HardLimitInKB"], out hardLimitInKb) == false) hardLimitInKb = QuotasHardLimitInKb; int softMarginInKb; if (int.TryParse(ConfigurationManager.AppSettings["Raven/Bundles/LiveTest/Quotas/Size/SoftLimitInKB"], out softMarginInKb) == false) softMarginInKb = QuotasSoftMarginInKb; settings[Constants.ActiveBundles] = string.Join(";", bundles); settings[Constants.SizeHardLimitInKB] = hardLimitInKb; settings[Constants.SizeSoftLimitInKB] = softMarginInKb; settings[Constants.DocsHardLimit] = null; settings[Constants.DocsSoftLimit] = null; }
public override void AfterCommit(string key, Stream data, RavenJObject metadata, Guid etag) { // Make sure we have a filename var filename = metadata.Value<string>("Raven-Attachment-Filename"); if (string.IsNullOrEmpty(filename)) return; RavenJObject doc; try { // Extract the text in the attachment as a json document var extension = Path.GetExtension(filename); doc = Extractor.GetJson(data, extension); } catch (InvalidOperationException ex) { // If there is no ifilter installed, don't do the extraction, but still do everything else. if (!ex.Message.Contains("HRESULT: 0x80004005")) throw; // Still write a dummmy doc so we get metadata in the index. doc = new RavenJObject(); } // Write the results to a document. Include all of the attachment's metadata, and a reference back to the attachment key. var md = new RavenJObject(metadata) { { "Raven-Attachment-Key", key } }; Database.Put(key + "/text", null, doc, md, null); }
protected StorageStream(ITransactionalStorage transactionalStorage, string fileName, StorageStreamAccess storageStreamAccess, RavenJObject metadata, IndexStorage indexStorage, StorageOperationsTask operations) { TransactionalStorage = transactionalStorage; StorageStreamAccess = storageStreamAccess; Name = fileName; switch (storageStreamAccess) { case StorageStreamAccess.Read: TransactionalStorage.Batch(accessor => fileHeader = accessor.ReadFile(fileName)); if (fileHeader.TotalSize == null) { throw new FileNotFoundException("File is not uploaded yet"); } Metadata = fileHeader.Metadata; Seek(0, SeekOrigin.Begin); break; case StorageStreamAccess.CreateAndWrite: TransactionalStorage.Batch(accessor => { operations.IndicateFileToDelete(fileName); accessor.PutFile(fileName, null, metadata); indexStorage.Index(fileName, metadata); }); Metadata = metadata; break; default: throw new ArgumentOutOfRangeException("storageStreamAccess", storageStreamAccess, "Unknown value"); } }
public void Set(string name, string key, RavenJObject data, UuidType uuidType) { Api.JetSetCurrentIndex(session, Lists, "by_name_and_key"); Api.MakeKey(session, Lists, name, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, Lists, key, Encoding.Unicode, MakeKeyGrbit.None); var exists = Api.TrySeek(session, Lists, SeekGrbit.SeekEQ); using (var update = new Update(session, Lists, exists ? JET_prep.Replace : JET_prep.Insert)) { Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["name"], name, Encoding.Unicode); Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["key"], key, Encoding.Unicode); Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["etag"], uuidGenerator.CreateSequentialUuid(uuidType).TransformToValueForEsentSorting()); Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["created_at"], SystemTime.UtcNow); using (var columnStream = new ColumnStream(session, Lists, tableColumnsCache.ListsColumns["data"])) { if (exists) columnStream.SetLength(0); using (Stream stream = new BufferedStream(columnStream)) { data.WriteTo(stream); stream.Flush(); } } update.Save(); } }
public override void AfterPut(string key, RavenJObject document, RavenJObject metadata, System.Guid etag, TransactionInformation transactionInformation) { if (key.StartsWith("Raven/")) { return; } var entityName = metadata.Value<string>(Constants.RavenEntityName) + "/"; var properties = metadata.Value<RavenJArray>(Constants.EnsureUniqueConstraints); if (properties == null || properties.Length <= 0) return; var constraintMetaObject = new RavenJObject { { Constants.IsConstraintDocument, true } }; constraintMetaObject.EnsureSnapshot(); foreach (var property in properties) { var propName = ((RavenJValue)property).Value.ToString(); var uniqueValue = document.Value<string>(propName); if(uniqueValue == null) continue; string documentName = "UniqueConstraints/" + entityName + propName + "/" +Util.EscapeUniqueValue(uniqueValue); Database.Put( documentName, null, RavenJObject.FromObject(new { RelatedId = key }), (RavenJObject)constraintMetaObject.CreateSnapshot(), transactionInformation); } }
public override IEnumerable<Row> Execute(IEnumerable<Row> rows) { int count = 0; foreach (var partitionedRows in rows.Partition(Constants.BatchSize)) { List<PutCommandData> commands = new List<PutCommandData>(); foreach(var row in partitionedRows) { RavenJObject obj = new RavenJObject(); foreach(object key in row.Keys) { obj.Add((string)key, RavenJToken.FromObject(row[key])); } commands.Add(new PutCommandData() { Document = obj, Metadata = new RavenJObject { {"Raven-Entity-Name", new RavenJValue(collection)} }, Key = generateKey(obj) }); } count++; WriteCommandsTo(collection + " #" + count.ToString("00000") + ".json", commands); } yield break; }
private static void WriteJsonSnapshot(RavenJObject ravenJObject, StringWriter sw, int margin, int intdent = 0) { foreach (var item in ravenJObject) { if (intdent > 0) sw.Write(new string(' ', intdent * 4)); sw.Write(item.Key + ": "); switch (item.Value.Type) { case JTokenType.Object: sw.Write('{'); sw.Write(Environment.NewLine); WriteJsonSnapshot((RavenJObject)item.Value, sw, margin, intdent + 1); sw.Write('}'); break; case JTokenType.Null: sw.Write("null"); break; case JTokenType.String: sw.Write('"'); sw.Write(item.Value.ToString().ShortViewOfString(margin - 2)); sw.Write('"'); break; default: sw.Write(item.Value.ToString().ShortViewOfString(margin)); break; } sw.Write(Environment.NewLine); } }
public RavenJObject ConvertEntityToJson(string key, object entity, RavenJObject metadata) { foreach (var extendedDocumentConversionListener in Listeners.ConversionListeners) { extendedDocumentConversionListener.BeforeConversionToDocument(key, entity, metadata); } var entityType = entity.GetType(); var identityProperty = documentStore.Conventions.GetIdentityProperty(entityType); var objectAsJson = GetObjectAsJson(entity); if (identityProperty != null) { objectAsJson.Remove(identityProperty.Name); } SetClrType(entityType, metadata); foreach (var extendedDocumentConversionListener in Listeners.ConversionListeners) { extendedDocumentConversionListener.AfterConversionToDocument(key, entity, objectAsJson, metadata); } return objectAsJson; }
public SynchronizationWorkItem DetermineWork(string file, RavenJObject localMetadata, RavenJObject destinationMetadata, string localServerUrl, out NoSyncReason reason) { reason = NoSyncReason.Unknown; if (localMetadata == null) { reason = NoSyncReason.SourceFileNotExist; return null; } if (destinationMetadata != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null && destinationMetadata[SynchronizationConstants.RavenSynchronizationConflictResolution] == null) { reason = NoSyncReason.DestinationFileConflicted; return null; } if (localMetadata[SynchronizationConstants.RavenSynchronizationConflict] != null) { reason = NoSyncReason.SourceFileConflicted; return null; } if (localMetadata[SynchronizationConstants.RavenDeleteMarker] != null) { if (localMetadata.ContainsKey(SynchronizationConstants.RavenRenameFile)) { var rename = localMetadata.Value<string>(SynchronizationConstants.RavenRenameFile); if (destinationMetadata != null) return new RenameWorkItem(file, rename, localServerUrl, storage); return new ContentUpdateWorkItem(rename, localServerUrl, storage, sigGenerator); // we have a rename tombstone but file does not exists on destination } return new DeleteWorkItem(file, localServerUrl, storage); } if (destinationMetadata != null && Historian.IsDirectChildOfCurrent(localMetadata, destinationMetadata)) { reason = NoSyncReason.ContainedInDestinationHistory; return null; } // file exists on dest and has the same content if (destinationMetadata != null && localMetadata.Value<string>("Content-MD5") == destinationMetadata.Value<string>("Content-MD5")) { // check metadata to detect if any synchronization is needed if (localMetadata.Keys.Except(new[] { Constants.MetadataEtagField, Constants.RavenLastModified, Constants.LastModified }) .Any(key => !destinationMetadata.ContainsKey(key) || localMetadata[key] != destinationMetadata[key])) { return new MetadataUpdateWorkItem(file, localServerUrl, destinationMetadata, storage); } reason = NoSyncReason.SameContentAndMetadata; return null; // the same content and metadata - no need to synchronize } return new ContentUpdateWorkItem(file, localServerUrl, storage, sigGenerator); }
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { if (key.StartsWith("Raven/")) // we don't deal with system documents return; var doc = Database.Get(key, null); if (doc != null) { var history = doc.Metadata.Value<RavenJArray>(ReplicationConstants.RavenReplicationHistory) ?? new RavenJArray(); metadata[ReplicationConstants.RavenReplicationHistory] = history; history.Add(new RavenJObject { {ReplicationConstants.RavenReplicationVersion, doc.Metadata[ReplicationConstants.RavenReplicationVersion]}, {ReplicationConstants.RavenReplicationSource, doc.Metadata[ReplicationConstants.RavenReplicationSource]} }); if (history.Length > ReplicationConstants.ChangeHistoryLength) { history.RemoveAt(0); } } metadata[ReplicationConstants.RavenReplicationVersion] = RavenJToken.FromObject(hiLo.NextId()); metadata[ReplicationConstants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id); }
public override void OnPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { if (key.StartsWith("Raven/", StringComparison.OrdinalIgnoreCase) && // we don't deal with system documents key.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase) == false) // except for hilos return; using (Database.DisableAllTriggersForCurrentThread()) { var documentMetadata = GetDocumentMetadata(key); if (documentMetadata != null) { RavenJArray history = new RavenJArray(ReplicationData.GetHistory(documentMetadata)); metadata[Constants.RavenReplicationHistory] = history; if (documentMetadata.ContainsKey(Constants.RavenReplicationVersion) && documentMetadata.ContainsKey(Constants.RavenReplicationSource)) { history.Add(new RavenJObject { {Constants.RavenReplicationVersion, documentMetadata[Constants.RavenReplicationVersion]}, {Constants.RavenReplicationSource, documentMetadata[Constants.RavenReplicationSource]} }); } while (history.Length > Constants.ChangeHistoryLength) { history.RemoveAt(0); } } metadata[Constants.RavenReplicationVersion] = RavenJToken.FromObject(HiLo.NextId()); metadata[Constants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id); } }
public void can_update_a_doc_within_transaction_scope() { using (var documentStore = NewDocumentStore(requestedStorage: "esent")) { EnsureDtcIsSupported(documentStore); var id1 = Guid.NewGuid(); RavenJObject dummy = null; using (TransactionScope trnx = new TransactionScope()) { using (var session = documentStore.OpenSession()) { dummy = new RavenJObject(); dummy.Add("Name", "This is the object content"); dummy.Add("Id", RavenJToken.FromObject(id1)); session.Store(dummy); session.SaveChanges(); } using (var session = documentStore.OpenSession()) { session.Store(dummy); session.SaveChanges(); } trnx.Complete(); } } }
public bool FilterDocuments(string destinationId, string key, RavenJObject metadata) { if (key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase)) // don't replicate system docs { if (key.StartsWith("Raven/Hilo/", StringComparison.InvariantCultureIgnoreCase) == false) // except for hilo documents return false; } if (metadata.ContainsKey(Constants.NotForReplication) && metadata.Value<bool>(Constants.NotForReplication)) // not explicitly marked to skip return false; if (metadata[Constants.RavenReplicationConflict] != null) // don't replicate conflicted documents, that just propagate the conflict return false; if (metadata.Value<string>(Constants.RavenReplicationSource) == destinationId) // prevent replicating back to source return false; switch (ReplicationOptionsBehavior) { case TransitiveReplicationOptions.None: var value = metadata.Value<string>(Constants.RavenReplicationSource); var replicateDoc = value == null || (value == CurrentDatabaseId); return replicateDoc; } return true; }
public RenameFileOperation(string name, string rename, Etag currentEtag, RavenJObject metadataAfterOperation) { Name = name; Rename = rename; Etag = currentEtag; MetadataAfterOperation = metadataAfterOperation; }
public override void OnPut(string key, byte[] data, RavenJObject metadata) { if (key.StartsWith("Raven/")) // we don't deal with system attachment return; var attachment = Database.GetStatic(key); if (attachment != null) { var history = attachment.Metadata.Value<RavenJArray>(ReplicationConstants.RavenReplicationHistory) ?? new RavenJArray(); metadata[ReplicationConstants.RavenReplicationHistory] = history; history.Add(new RavenJObject { {ReplicationConstants.RavenReplicationVersion, attachment.Metadata[ReplicationConstants.RavenReplicationVersion]}, {ReplicationConstants.RavenReplicationSource, attachment.Metadata[ReplicationConstants.RavenReplicationSource]} }); if (history.Length > ReplicationConstants.ChangeHistoryLength) { history.RemoveAt(0); } } metadata[ReplicationConstants.RavenReplicationVersion] = RavenJToken.FromObject(hiLo.NextId()); metadata[ReplicationConstants.RavenReplicationSource] = RavenJToken.FromObject(Database.TransactionalStorage.Id); }
public void CanProcess() { var document = new RavenJObject { { "Data", new RavenJObject { {"Title", "Hi"} } } }; const string name = @"Raven.Tests.Patching.x2js.js"; var manifestResourceStream = Assembly.GetExecutingAssembly().GetManifestResourceStream(name); var code = new StreamReader(manifestResourceStream).ReadToEnd(); var jsonPatcher = new ScriptedJsonPatcher(); using (var scope = new DefaultScriptedJsonPatcherOperationScope()) { scope.CustomFunctions = new JsonDocument { DataAsJson = new RavenJObject { {"Functions", code} } }; jsonPatcher.Apply(scope, document, new ScriptedPatchRequest { Script = "this.Xml = js2x(this.Data);" }); } }
public ActionResult Create(CreateEditMenu showMenu) { try { var menu = new Menu { Name = showMenu.Name }; _session.Store(menu); var key = RavenDbKey.GenerateKey<Menu>(menu.Id); var stream = showMenu.File.InputStream; var optionalMetaData = new RavenJObject(); optionalMetaData["Format"] = showMenu.File.ContentType; _session.Advanced.DocumentStore.DatabaseCommands.PutAttachment(key, null, stream, optionalMetaData); return RedirectToAction("Index"); } catch { ModelState.AddModelError("", "Unable to save changes. Try again, and if the problem persists see your system administrator."); } return View(showMenu); }
private static T Extract <T>(RavenJObject metadata, string key, T defaultValue = default(T)) { return(Extract <T, T>(metadata, key, defaultValue, t => t)); }
private RavenJToken HandleErrors(WebException e) { var httpWebResponse = e.Response as HttpWebResponse; if (httpWebResponse == null || httpWebResponse.StatusCode == HttpStatusCode.Unauthorized || httpWebResponse.StatusCode == HttpStatusCode.NotFound || httpWebResponse.StatusCode == HttpStatusCode.Conflict) { int httpResult = -1; if (httpWebResponse != null) { httpResult = (int)httpWebResponse.StatusCode; } factory.InvokeLogRequest(owner, () => new RequestResultArgs { DurationMilliseconds = CalculateDuration(), Method = webRequest.Method, HttpResult = httpResult, Status = RequestStatus.ErrorOnServer, Result = e.Message, Url = webRequest.RequestUri.PathAndQuery, PostedData = postedData }); return(null); //throws } if (httpWebResponse.StatusCode == HttpStatusCode.NotModified && CachedRequestDetails != null) { factory.UpdateCacheTime(this); var result = factory.GetCachedResponse(this, httpWebResponse.Headers); HandleReplicationStatusChanges(httpWebResponse.Headers, primaryUrl, operationUrl); factory.InvokeLogRequest(owner, () => new RequestResultArgs { DurationMilliseconds = CalculateDuration(), Method = webRequest.Method, HttpResult = (int)httpWebResponse.StatusCode, Status = RequestStatus.Cached, Result = result.ToString(), Url = webRequest.RequestUri.PathAndQuery, PostedData = postedData }); return(result); } using (var sr = new StreamReader(e.Response.GetResponseStreamWithHttpDecompression())) { var readToEnd = sr.ReadToEnd(); factory.InvokeLogRequest(owner, () => new RequestResultArgs { DurationMilliseconds = CalculateDuration(), Method = webRequest.Method, HttpResult = (int)httpWebResponse.StatusCode, Status = RequestStatus.Cached, Result = readToEnd, Url = webRequest.RequestUri.PathAndQuery, PostedData = postedData }); if (string.IsNullOrWhiteSpace(readToEnd)) { return(null); // throws } RavenJObject ravenJObject; try { ravenJObject = RavenJObject.Parse(readToEnd); } catch (Exception) { throw new InvalidOperationException(readToEnd, e); } if (ravenJObject.ContainsKey("IndexDefinitionProperty")) { throw new IndexCompilationException(ravenJObject.Value <string>("Message")) { IndexDefinitionProperty = ravenJObject.Value <string>("IndexDefinitionProperty"), ProblematicText = ravenJObject.Value <string>("ProblematicText") }; } if (httpWebResponse.StatusCode == HttpStatusCode.BadRequest && ravenJObject.ContainsKey("Message")) { throw new BadRequestException(ravenJObject.Value <string>("Message"), e); } if (ravenJObject.ContainsKey("Error")) { var sb = new StringBuilder(); foreach (var prop in ravenJObject) { if (prop.Key == "Error") { continue; } sb.Append(prop.Key).Append(": ").AppendLine(prop.Value.ToString(Formatting.Indented)); } sb.AppendLine() .AppendLine(ravenJObject.Value <string>("Error")); throw new InvalidOperationException(sb.ToString(), e); } throw new InvalidOperationException(readToEnd, e); } }
/// <summary> /// Filters the headers from unwanted headers /// </summary> /// <param name="self">The self.</param> /// <returns></returns>public static RavenJObject FilterHeadersToObject(this System.Collections.Specialized.NameValueCollection self, bool isServerDocument) public static RavenJObject FilterHeadersToObject(this RavenJObject self) { return(FilterHeadersToObject(self, HeadersToIgnoreClient, PrefixesInHeadersToIgnoreClient)); }
private void WriteMetadata(RavenJObject metadata) { if (metadata == null || metadata.Count == 0) { return; } foreach (var prop in metadata) { if (prop.Value == null) { continue; } if (prop.Value.Type == JTokenType.Object || prop.Value.Type == JTokenType.Array) { continue; } var headerName = prop.Key; var value = prop.Value.Value <object>().ToString(); if (headerName == Constants.MetadataEtagField) { headerName = "If-None-Match"; if (!value.StartsWith("\"")) { value = "\"" + value; } if (!value.EndsWith("\"")) { value = value + "\""; } } bool isRestricted; try { isRestricted = WebHeaderCollection.IsRestricted(headerName); } catch (Exception e) { throw new InvalidOperationException("Could not figure out how to treat header: " + headerName, e); } // Restricted headers require their own special treatment, otherwise an exception will // be thrown. // See http://msdn.microsoft.com/en-us/library/78h415ay.aspx if (isRestricted) { switch (headerName) { /*case "Date": * case "Referer": * case "Content-Length": * case "Expect": * case "Range": * case "Transfer-Encoding": * case "User-Agent": * case "Proxy-Connection": * case "Host": // Host property is not supported by 3.5 * break;*/ case "Content-Type": headers["Content-Type"] = value; break; case "If-Modified-Since": DateTime tmp; DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out tmp); httpClient.DefaultRequestHeaders.IfModifiedSince = tmp; break; case "Accept": httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(value)); break; case "Connection": httpClient.DefaultRequestHeaders.Connection.Add(value); break; } } else { headers[headerName] = value; } } }
private async Task <RavenJToken> CheckForErrorsAndReturnCachedResultIfAnyAsync(bool readErrorString) { if (Response.IsSuccessStatusCode == false) { if (Response.StatusCode == HttpStatusCode.Unauthorized || Response.StatusCode == HttpStatusCode.NotFound || Response.StatusCode == HttpStatusCode.Conflict) { factory.InvokeLogRequest(owner, () => new RequestResultArgs { DurationMilliseconds = CalculateDuration(), Method = Method, HttpResult = (int)Response.StatusCode, Status = RequestStatus.ErrorOnServer, Result = Response.StatusCode.ToString(), Url = Url, PostedData = postedData }); throw ErrorResponseException.FromResponseMessage(Response, readErrorString); } if (Response.StatusCode == HttpStatusCode.NotModified && CachedRequestDetails != null) { factory.UpdateCacheTime(this); var result = factory.GetCachedResponse(this, ResponseHeaders); // here we explicitly need to get Response.Headers, and NOT ResponseHeaders because we are // getting the value _right now_ from the secondary, and don't care about the 304, the force check // is still valid HandleReplicationStatusChanges(ResponseHeaders, primaryUrl, operationUrl); factory.InvokeLogRequest(owner, () => new RequestResultArgs { DurationMilliseconds = CalculateDuration(), Method = Method, HttpResult = (int)Response.StatusCode, Status = RequestStatus.Cached, Result = result.ToString(), Url = Url, PostedData = postedData }); return(result); } using (var sr = new StreamReader(await Response.GetResponseStreamWithHttpDecompression().ConfigureAwait(false))) { var readToEnd = sr.ReadToEnd(); factory.InvokeLogRequest(owner, () => new RequestResultArgs { DurationMilliseconds = CalculateDuration(), Method = Method, HttpResult = (int)Response.StatusCode, Status = RequestStatus.Cached, Result = readToEnd, Url = Url, PostedData = postedData }); if (string.IsNullOrWhiteSpace(readToEnd)) { throw ErrorResponseException.FromResponseMessage(Response); } RavenJObject ravenJObject; try { ravenJObject = RavenJObject.Parse(readToEnd); } catch (Exception e) { throw new ErrorResponseException(Response, readToEnd, e); } if (ravenJObject.ContainsKey("IndexDefinitionProperty")) { throw new IndexCompilationException(ravenJObject.Value <string>("Message")) { IndexDefinitionProperty = ravenJObject.Value <string>("IndexDefinitionProperty"), ProblematicText = ravenJObject.Value <string>("ProblematicText") }; } if (Response.StatusCode == HttpStatusCode.BadRequest && ravenJObject.ContainsKey("Message")) { throw new BadRequestException(ravenJObject.Value <string>("Message"), ErrorResponseException.FromResponseMessage(Response)); } if (ravenJObject.ContainsKey("Error")) { var sb = new StringBuilder(); foreach (var prop in ravenJObject) { if (prop.Key == "Error") { continue; } sb.Append(prop.Key).Append(": ").AppendLine(prop.Value.ToString(Formatting.Indented)); } if (sb.Length > 0) { sb.AppendLine(); } sb.Append(ravenJObject.Value <string>("Error")); throw new ErrorResponseException(Response, sb.ToString(), readToEnd); } throw new ErrorResponseException(Response, readToEnd); } } return(null); }
private void AddError(int index, string indexName, string key, string error, string component) { var increment = Interlocked.Increment(ref errorsCounter); var indexingError = new IndexingError { Id = increment, Document = key, Error = error, Index = index, IndexName = indexName, Action = component, Timestamp = SystemTime.UtcNow }; indexingErrors.Enqueue(indexingError); IndexingError ignored = null; lock (indexingErrorLocks.GetOrAdd(index, new object())) { using (TransactionalStorage.DisableBatchNesting()) TransactionalStorage.Batch(accessor => { accessor.Lists.Set("Raven/Indexing/Errors/" + indexName, indexingError.Id.ToString(CultureInfo.InvariantCulture), RavenJObject.FromObject(indexingError), UuidType.Indexing); if (indexingErrors.Count <= 50) return; if (indexingErrors.TryDequeue(out ignored) == false) return; if (index != ignored.Index || (SystemTime.UtcNow - ignored.Timestamp).TotalSeconds <= 10) return; accessor.Lists.RemoveAllOlderThan("Raven/Indexing/Errors/" + ignored.IndexName, ignored.Timestamp); }); } if (ignored == null || index == ignored.Index) return; if ((SystemTime.UtcNow - ignored.Timestamp).TotalSeconds <= 10) return; lock (indexingErrorLocks.GetOrAdd(ignored.Index, new object())) { using (TransactionalStorage.DisableBatchNesting()) TransactionalStorage.Batch(accessor => { accessor.Lists.RemoveAllOlderThan("Raven/Indexing/Errors/" + ignored.IndexName, ignored.Timestamp); }); } }
///<summary> /// Translate a collection of RavenJObject to JsonDocuments ///</summary> public static JsonDocument ToJsonDocument(this RavenJObject response) { return(RavenJObjectsToJsonDocuments(new[] { response }).First()); }
public void AfterConversionToDocument(string key, object entity, RavenJObject document, RavenJObject metadata) { }
public bool BeforeStore(string key, object entityInstance, RavenJObject metadata, RavenJObject original) { return(false); }
public void BeforeConversionToEntity(string key, RavenJObject document, RavenJObject metadata) { }
/// <summary> /// Sends a patch request for a specific document which may or may not currently exist /// </summary> /// <param name="key">Id of the document to patch</param> /// <param name="patchesToExisting">Array of patch requests to apply to an existing document</param> /// <param name="patchesToDefault">Array of patch requests to apply to a default document when the document is missing</param> /// <param name="defaultMetadata">The metadata for the default document when the document is missing</param> public RavenJObject Patch(string key, PatchRequest[] patchesToExisting, PatchRequest[] patchesToDefault, RavenJObject defaultMetadata) { return(asyncServerClient.PatchAsync(key, patchesToExisting, patchesToDefault, defaultMetadata).ResultUnwrap()); }
public void BeforeConversionToDocument(string key, object entity, RavenJObject metadata) { }
public void PutAttachment(string key, Etag etag, Stream data, RavenJObject metadata) { asyncServerClient.PutAttachmentAsync(key, etag, data, metadata).WaitUnwrap(); }
/// <summary> /// Sends a patch request for a specific document which may or may not currently exist /// </summary> /// <param name="key">Id of the document to patch</param> /// <param name="patchExisting">The patch request to use (using JavaScript) to an existing document</param> /// <param name="patchDefault">The patch request to use (using JavaScript) to a default document when the document is missing</param> /// <param name="defaultMetadata">The metadata for the default document when the document is missing</param> public RavenJObject Patch(string key, ScriptedPatchRequest patchExisting, ScriptedPatchRequest patchDefault, RavenJObject defaultMetadata) { return(asyncServerClient.PatchAsync(key, patchExisting, patchDefault, defaultMetadata).ResultUnwrap()); }
public async Task <HttpResponseMessage> Restore() { if (EnsureSystemDatabase() == false) { return(GetMessageWithString("Restore is only possible from the system database", HttpStatusCode.BadRequest)); } var restoreStatus = new RestoreStatus { Messages = new List <string>() }; var restoreRequest = await ReadJsonObjectAsync <RestoreRequest>(); DatabaseDocument databaseDocument = null; var databaseDocumentPath = Path.Combine(restoreRequest.BackupLocation, "Database.Document"); if (File.Exists(databaseDocumentPath)) { var databaseDocumentText = File.ReadAllText(databaseDocumentPath); databaseDocument = RavenJObject.Parse(databaseDocumentText).JsonDeserialization <DatabaseDocument>(); } var databaseName = !string.IsNullOrWhiteSpace(restoreRequest.DatabaseName) ? restoreRequest.DatabaseName : databaseDocument == null ? null : databaseDocument.Id; if (string.IsNullOrWhiteSpace(databaseName)) { var errorMessage = (databaseDocument == null || String.IsNullOrWhiteSpace(databaseDocument.Id)) ? "Database.Document file is invalid - database name was not found and not supplied in the request (Id property is missing or null). This is probably a bug - should never happen." : "A database name must be supplied if the restore location does not contain a valid Database.Document file"; restoreStatus.Messages.Add(errorMessage); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenRestoreStatusDocumentKey, null, RavenJObject.FromObject(new { restoreStatus }), new RavenJObject(), null); return(GetMessageWithString(errorMessage, HttpStatusCode.BadRequest)); } if (databaseName == Constants.SystemDatabase) { return(GetMessageWithString("Cannot do an online restore for the <system> database", HttpStatusCode.BadRequest)); } var existingDatabase = Database.Documents.GetDocumentMetadata("Raven/Databases/" + databaseName, null); if (existingDatabase != null) { return(GetMessageWithString("Cannot do an online restore for an existing database, delete the database " + databaseName + " and restore again.", HttpStatusCode.BadRequest)); } var ravenConfiguration = new RavenConfiguration { DatabaseName = databaseName, IsTenantDatabase = true }; if (databaseDocument != null) { foreach (var setting in databaseDocument.Settings) { ravenConfiguration.Settings[setting.Key] = setting.Value; } } if (File.Exists(Path.Combine(restoreRequest.BackupLocation, BackupMethods.Filename))) { ravenConfiguration.DefaultStorageTypeName = typeof(Raven.Storage.Voron.TransactionalStorage).AssemblyQualifiedName; } else if (Directory.Exists(Path.Combine(restoreRequest.BackupLocation, "new"))) { ravenConfiguration.DefaultStorageTypeName = typeof(Raven.Storage.Esent.TransactionalStorage).AssemblyQualifiedName; } ravenConfiguration.CustomizeValuesForTenant(databaseName); ravenConfiguration.Initialize(); string documentDataDir; ravenConfiguration.DataDirectory = ResolveTenantDataDirectory(restoreRequest.DatabaseLocation, databaseName, out documentDataDir); restoreRequest.DatabaseLocation = ravenConfiguration.DataDirectory; DatabasesLandlord.SystemDatabase.Documents.Delete(RestoreStatus.RavenRestoreStatusDocumentKey, null, null); bool defrag; if (bool.TryParse(GetQueryStringValue("defrag"), out defrag)) { restoreRequest.Defrag = defrag; } var task = Task.Factory.StartNew(() => { MaintenanceActions.Restore(ravenConfiguration, restoreRequest, msg => { restoreStatus.Messages.Add(msg); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenRestoreStatusDocumentKey, null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null); }); if (databaseDocument == null) { return; } databaseDocument.Settings[Constants.RavenDataDir] = documentDataDir; if (restoreRequest.IndexesLocation != null) { databaseDocument.Settings[Constants.RavenIndexPath] = restoreRequest.IndexesLocation; } if (restoreRequest.JournalsLocation != null) { databaseDocument.Settings[Constants.RavenTxJournalPath] = restoreRequest.JournalsLocation; } databaseDocument.Id = databaseName; DatabasesLandlord.Protect(databaseDocument); DatabasesLandlord.SystemDatabase.Documents.Put("Raven/Databases/" + databaseName, null, RavenJObject.FromObject(databaseDocument), new RavenJObject(), null); restoreStatus.Messages.Add("The new database was created"); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenRestoreStatusDocumentKey, null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null); }, TaskCreationOptions.LongRunning); long id; Database.Tasks.AddTask(task, new object(), new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.RestoreDatabase, Payload = "Restoring database " + databaseName + " from " + restoreRequest.BackupLocation }, out id); return(GetMessageWithObject(new { OperationId = id })); }
public void UpdateAttachmentMetadata(string key, Etag etag, RavenJObject metadata) { asyncServerClient.UpdateAttachmentMetadataAsync(key, etag, metadata).WaitUnwrap(); }
protected override Task <RavenJObject> TransformDocument(RavenJObject document, string transformScript) { return(new CompletedTask <RavenJObject>(document)); }
/// <summary> /// Puts the document with the specified key in the database /// </summary> /// <param name="key">The key.</param> /// <param name="etag">The etag.</param> /// <param name="document">The document.</param> /// <param name="metadata">The metadata.</param> /// <returns></returns> public PutResult Put(string key, Etag etag, RavenJObject document, RavenJObject metadata) { return(asyncServerClient.PutAsync(key, etag, document, metadata).ResultUnwrap()); }
/// <summary> /// Invoked before the store request is sent to the server. /// </summary> /// <param name="key">The key.</param> /// <param name="entityInstance">The entity instance.</param> /// <param name="metadata">The metadata.</param> /// <param name="original">The original document that was loaded from the server</param> /// <returns> /// Whatever the entity instance was modified and requires us re-serialize it. /// Returning true would force re-serialization of the entity, returning false would /// mean that any changes to the entityInstance would be ignored in the current SaveChanges call. /// </returns> public bool BeforeStore(string key, object entityInstance, RavenJObject metadata, RavenJObject original) { // Ensure id if (entityInstance is IModel) { var model = (IModel)entityInstance; if (model.Id == Guid.Empty) { model.Id = Guid.NewGuid(); } } // Track changes if (entityInstance is Data.IChanges) { var model = (Data.IChanges)entityInstance; var now = DateTime.Now; // Set updated date model.Updated = now; // Set created date for new models if (model.Created == DateTime.MinValue) { model.Created = now; } } // Call events if (entityInstance is Models.Model) { ((Models.Model)entityInstance).OnSave(); } return(true); }
private void ProcessPatchResult(IHttpContext context, string docId, PatchResult patchResult, object debug, RavenJObject document) { switch (patchResult) { case PatchResult.DocumentDoesNotExists: context.SetStatusToNotFound(); break; case PatchResult.Patched: context.Response.AddHeader("Location", Database.Configuration.GetFullUrl("/docs/" + docId)); context.WriteJson(new { Patched = true, Debug = debug }); break; case PatchResult.Tested: context.WriteJson(new { Patched = false, Debug = debug, Document = document }); break; default: throw new ArgumentOutOfRangeException("Value " + patchResult + " is not understood"); } }
/// <summary> /// Allow the trigger to perform any logic just before the document is saved to disk. /// Any modifications the trigger makes to the document or the metadata will be persisted /// to disk. /// </summary><remarks> /// If the trigger need to access the previous state of the document, the trigger should /// implement <seealso cref="IRequiresDocumentDatabaseInitialization" /> and use the provided /// <seealso cref="DocumentDatabase" /> instance to Get it. The returned result would be the old /// document (if it exists) or null. /// Any call to the provided <seealso cref="DocumentDatabase" /> instance will be done under the /// same transaction as the PUT operation. /// </remarks><param name="key">The document key</param><param name="jsonReplicationDocument">The new document about to be put into Raven</param><param name="metadata">The new document metadata</param><param name="transactionInformation">The current transaction, if it exists</param> public virtual void OnPut(string key, RavenJObject jsonReplicationDocument, RavenJObject metadata) { }
/// <summary> /// Invoked after the store request is sent to the server. /// </summary> /// <param name="key">The key.</param> /// <param name="entityInstance">The entity instance.</param> /// <param name="metadata">The metadata.</param> public void AfterStore(string key, object entityInstance, RavenJObject metadata) { }
public override VetoResult AllowPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation) { if (key.StartsWith("Raven/")) { return(VetoResult.Allowed); } var entityName = metadata.Value <string>(Constants.RavenEntityName); if (string.IsNullOrEmpty(entityName)) { return(VetoResult.Allowed); } entityName += "/"; var properties = metadata.Value <RavenJArray>(Constants.EnsureUniqueConstraints); if (properties == null || properties.Length <= 0) { return(VetoResult.Allowed); } var invalidFields = new StringBuilder(); foreach (var property in properties) { var propName = property.Value <string>(); var prefix = "UniqueConstraints/" + entityName + property + "/"; var prop = document[propName]; if (prop == null || prop.Type == JTokenType.Null) { continue; } var array = prop as RavenJArray; var checkKeys = array != null?array.Select(p => p.Value <string>()) : new[] { prop.Value <string>() }; foreach (var checkKey in checkKeys) { var checkDoc = Database.Get(prefix + Util.EscapeUniqueValue(checkKey), transactionInformation); if (checkDoc == null) { continue; } var checkId = checkDoc.DataAsJson.Value <string>("RelatedId"); if (checkId != key) { invalidFields.Append(property + ", "); } } } if (invalidFields.Length > 0) { invalidFields.Length = invalidFields.Length - 2; return(VetoResult.Deny("Ensure unique constraint violated for fields: " + invalidFields)); } return(VetoResult.Allowed); }
/// <summary> /// Allow the trigger to perform any logic _after_ the transaction was committed. /// For example, by notifying interested parties. /// </summary><remarks> /// This method SHOULD NOT modify either the document or the metadata /// </remarks><param name="key">The document key</param><param name="document">The document that was put into Raven</param><param name="metadata">The document metadata</param> /// <param name="etag">The etag of the just put document</param> public virtual void AfterCommit(string key, RavenJObject document, RavenJObject metadata, Etag etag) { }
public Task <HttpResponseMessage> ResetSqlReplication(string sqlReplicationName) { var task = Database.StartupTasks.OfType <SqlReplicationTask>().FirstOrDefault(); if (task == null) { return(GetMessageWithObjectAsTask(new { Error = "SQL Replication bundle is not installed" }, HttpStatusCode.NotFound)); } SqlReplicationStatistics stats; task.Statistics.TryRemove(sqlReplicationName, out stats); var jsonDocument = Database.Documents.Get(SqlReplicationTask.RavenSqlReplicationStatus, null); if (jsonDocument != null) { var replicationStatus = jsonDocument.DataAsJson.JsonDeserialization <SqlReplicationStatus>(); replicationStatus.LastReplicatedEtags.RemoveAll(x => x.Name == sqlReplicationName); Database.Documents.Put(SqlReplicationTask.RavenSqlReplicationStatus, null, RavenJObject.FromObject(replicationStatus), new RavenJObject(), null); } return(GetEmptyMessageAsTask(HttpStatusCode.NoContent)); }
/// <summary> /// Ask the trigger whatever the PUT should be vetoed. /// If the trigger vote to veto the PUT, it needs to provide a human readable /// explanation why the PUT was rejected. /// </summary> /// <remarks> /// This method SHOULD NOT modify either the document or the metadata. /// </remarks> /// <param name="key">The document key</param> /// <param name="document">The new document about to be put into Raven</param> /// <param name="metadata">The new document metadata</param> /// <returns>Whatever the put was vetoed or not</returns> public virtual VetoResult AllowPut(string key, RavenJObject document, RavenJObject metadata) { return(VetoResult.Allowed); }
public bool Delete(string key, Etag etag, TransactionInformation transactionInformation, out RavenJObject metadata, string[] participatingIds = null) { if (key == null) { throw new ArgumentNullException("key"); } key = key.Trim(); var deleted = false; Log.Debug("Delete a document with key: {0} and etag {1}", key, etag); RavenJObject metadataVar = null; using (Database.DocumentLock.Lock()) { TransactionalStorage.Batch(actions => { AssertDeleteOperationNotVetoed(key, transactionInformation); if (transactionInformation == null) { Database.DeleteTriggers.Apply(trigger => trigger.OnDelete(key, null)); string collection = null; Etag deletedETag; if (actions.Documents.DeleteDocument(key, etag, out metadataVar, out deletedETag)) { deleted = true; actions.Indexing.RemoveAllDocumentReferencesFrom(key); WorkContext.MarkDeleted(key); Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, actions, participatingIds); collection = metadataVar.Value <string>(Constants.RavenEntityName); DeleteDocumentFromIndexesForCollection(key, collection, actions); if (deletedETag != null) { Database.Prefetcher.AfterDelete(key, deletedETag); } Database.DeleteTriggers.Apply(trigger => trigger.AfterDelete(key, null)); } TransactionalStorage .ExecuteImmediatelyOrRegisterForSynchronization(() => { Database.DeleteTriggers.Apply(trigger => trigger.AfterCommit(key)); if (string.IsNullOrEmpty(collection) == false) { Database.LastCollectionEtags.Update(collection); } Database.Notifications.RaiseNotifications(new DocumentChangeNotification { Id = key, Type = DocumentChangeTypes.Delete, TypeName = (metadataVar != null) ? metadataVar.Value <string>(Constants.RavenClrType) : null, CollectionName = (metadataVar != null) ? metadataVar.Value <string>(Constants.RavenEntityName) : null }, metadataVar); }); } else { var doc = actions.Documents.DocumentMetadataByKey(key); Database.InFlightTransactionalState.DeleteDocumentInTransaction(transactionInformation, key, etag, doc == null ? Etag.Empty : doc.Etag, UuidGenerator); deleted = doc != null; } WorkContext.ShouldNotifyAboutWork(() => "DEL " + key); }); metadata = metadataVar; return(deleted); } }
public async Task <HttpResponseMessage> LoadCsvFile() { if (!Request.Content.IsMimeMultipartContent()) { throw new Exception(); // divided by zero } var provider = new MultipartMemoryStreamProvider(); await Request.Content.ReadAsMultipartAsync(provider); foreach (var file in provider.Contents) { var filename = file.Headers.ContentDisposition.FileName.Trim('\"'); var stream = await file.ReadAsStreamAsync(); using (var csvReader = new TextFieldParser(stream)) { csvReader.SetDelimiters(","); var headers = csvReader.ReadFields(); var entity = Inflector.Pluralize(CSharpClassName.ConvertToValidClassName(Path.GetFileNameWithoutExtension(filename))); if (entity.Length > 0 && char.IsLower(entity[0])) { entity = char.ToUpper(entity[0]) + entity.Substring(1); } var totalCount = 0; var batch = new List <RavenJObject>(); var columns = headers.Where(x => x.StartsWith("@") == false).ToArray(); batch.Clear(); while (csvReader.EndOfData == false) { var record = csvReader.ReadFields(); var document = new RavenJObject(); string id = null; RavenJObject metadata = null; for (int index = 0; index < columns.Length; index++) { var column = columns[index]; if (string.IsNullOrEmpty(column)) { continue; } if (string.Equals("id", column, StringComparison.OrdinalIgnoreCase)) { id = record[index]; } else if (string.Equals(Constants.RavenEntityName, column, StringComparison.OrdinalIgnoreCase)) { metadata = metadata ?? new RavenJObject(); metadata[Constants.RavenEntityName] = record[index]; id = id ?? record[index] + "/"; } else if (string.Equals(Constants.RavenClrType, column, StringComparison.OrdinalIgnoreCase)) { metadata = metadata ?? new RavenJObject(); metadata[Constants.RavenClrType] = record[index]; id = id ?? record[index] + "/"; } else { document[column] = SetValueInDocument(record[index]); } } metadata = metadata ?? new RavenJObject { { "Raven-Entity-Name", entity } }; document.Add("@metadata", metadata); metadata.Add("@id", id ?? Guid.NewGuid().ToString()); batch.Add(document); totalCount++; if (batch.Count >= CsvImportBatchSize) { await FlushBatch(batch); batch.Clear(); } } if (batch.Count > 0) { await FlushBatch(batch); } } } return(GetEmptyMessage()); }
public PutResult Put(string key, Etag etag, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation, string[] participatingIds = null) { WorkContext.MetricsCounters.DocsPerSecond.Mark(); key = string.IsNullOrWhiteSpace(key) ? Guid.NewGuid().ToString() : key.Trim(); RemoveReservedProperties(document); RemoveMetadataReservedProperties(metadata); var newEtag = Etag.Empty; using (Database.DocumentLock.Lock()) { TransactionalStorage.Batch(actions => { if (key.EndsWith("/")) { key += GetNextIdentityValueWithoutOverwritingOnExistingDocuments(key, actions); } AssertPutOperationNotVetoed(key, metadata, document, transactionInformation); if (transactionInformation == null) { if (Database.InFlightTransactionalState.IsModified(key)) { throw new ConcurrencyException("PUT attempted on : " + key + " while it is being locked by another transaction"); } Database.PutTriggers.Apply(trigger => trigger.OnPut(key, document, metadata, null)); var addDocumentResult = actions.Documents.AddDocument(key, etag, document, metadata); newEtag = addDocumentResult.Etag; Database.Indexes.CheckReferenceBecauseOfDocumentUpdate(key, actions, participatingIds); metadata[Constants.LastModified] = addDocumentResult.SavedAt; metadata.EnsureSnapshot( "Metadata was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?"); document.EnsureSnapshot( "Document was written to the database, cannot modify the document after it was written (changes won't show up in the db). Did you forget to call CreateSnapshot() to get a clean copy?"); actions.AfterStorageCommitBeforeWorkNotifications(new JsonDocument { Metadata = metadata, Key = key, DataAsJson = document, Etag = newEtag, LastModified = addDocumentResult.SavedAt, SkipDeleteFromIndex = addDocumentResult.Updated == false }, documents => { if (Database.IndexDefinitionStorage.IndexesCount == 0 || Database.WorkContext.RunIndexing == false) { return; } Database.Prefetcher.AfterStorageCommitBeforeWorkNotifications(PrefetchingUser.Indexer, documents); }); Database.PutTriggers.Apply(trigger => trigger.AfterPut(key, document, metadata, newEtag, null)); TransactionalStorage .ExecuteImmediatelyOrRegisterForSynchronization(() => { Database.PutTriggers.Apply(trigger => trigger.AfterCommit(key, document, metadata, newEtag)); var newDocumentChangeNotification = new DocumentChangeNotification { Id = key, Type = DocumentChangeTypes.Put, TypeName = metadata.Value <string>(Constants.RavenClrType), CollectionName = metadata.Value <string>(Constants.RavenEntityName), Etag = newEtag }; Database.Notifications.RaiseNotifications(newDocumentChangeNotification, metadata); }); WorkContext.ShouldNotifyAboutWork(() => "PUT " + key); } else { var doc = actions.Documents.DocumentMetadataByKey(key); newEtag = Database.InFlightTransactionalState.AddDocumentInTransaction(key, etag, document, metadata, transactionInformation, doc == null ? Etag.Empty : doc.Etag, UuidGenerator); } }); Log.Debug("Put document {0} with etag {1}", key, newEtag); return(new PutResult { Key = key, ETag = newEtag }); } }
public async Task <HttpResponseMessage> ImportDatabase(int batchSize, bool includeExpiredDocuments, bool stripReplicationInformation, ItemType operateOnTypes, string filtersPipeDelimited, string transformScript) { if (!Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } string tempPath = Path.GetTempPath(); var fullTempPath = tempPath + Constants.TempUploadsDirectoryName; if (File.Exists(fullTempPath)) { File.Delete(fullTempPath); } if (Directory.Exists(fullTempPath) == false) { Directory.CreateDirectory(fullTempPath); } var streamProvider = new MultipartFileStreamProvider(fullTempPath); await Request.Content.ReadAsMultipartAsync(streamProvider).ConfigureAwait(false); var uploadedFilePath = streamProvider.FileData[0].LocalFileName; string fileName = null; var fileContent = streamProvider.Contents.SingleOrDefault(); if (fileContent != null) { fileName = fileContent.Headers.ContentDisposition.FileName.Replace("\"", string.Empty); } var status = new ImportOperationStatus(); var cts = new CancellationTokenSource(); var task = Task.Run(async() => { try { using (var fileStream = File.Open(uploadedFilePath, FileMode.Open, FileAccess.Read)) { var dataDumper = new DatabaseDataDumper(Database); dataDumper.Progress += s => status.LastProgress = s; var smugglerOptions = dataDumper.Options; smugglerOptions.BatchSize = batchSize; smugglerOptions.ShouldExcludeExpired = !includeExpiredDocuments; smugglerOptions.StripReplicationInformation = stripReplicationInformation; smugglerOptions.OperateOnTypes = operateOnTypes; smugglerOptions.TransformScript = transformScript; smugglerOptions.CancelToken = cts; // Filters are passed in without the aid of the model binder. Instead, we pass in a list of FilterSettings using a string like this: pathHere;;;valueHere;;;true|||againPathHere;;;anotherValue;;;false // Why? Because I don't see a way to pass a list of a values to a WebAPI method that accepts a file upload, outside of passing in a simple string value and parsing it ourselves. if (filtersPipeDelimited != null) { smugglerOptions.Filters.AddRange(filtersPipeDelimited .Split(new string[] { "|||" }, StringSplitOptions.RemoveEmptyEntries) .Select(f => f.Split(new string[] { ";;;" }, StringSplitOptions.RemoveEmptyEntries)) .Select(o => new FilterSetting { Path = o[0], Values = new List <string> { o[1] }, ShouldMatch = bool.Parse(o[2]) })); } await dataDumper.ImportData(new SmugglerImportOptions <RavenConnectionStringOptions> { FromStream = fileStream }); } } catch (Exception e) { status.Faulted = true; status.State = RavenJObject.FromObject(new { Error = e.ToString() }); if (cts.Token.IsCancellationRequested) { status.State = RavenJObject.FromObject(new { Error = "Task was cancelled" }); cts.Token.ThrowIfCancellationRequested(); //needed for displaying the task status as canceled and not faulted } if (e is InvalidDataException) { status.ExceptionDetails = e.Message; } else if (e is Imports.Newtonsoft.Json.JsonReaderException) { status.ExceptionDetails = "Failed to load JSON Data. Please make sure you are importing .ravendump file, exported by smuggler (aka database export). If you are importing a .ravnedump file then the file may be corrupted"; } else { status.ExceptionDetails = e.ToString(); } throw; } finally { status.Completed = true; File.Delete(uploadedFilePath); } }, cts.Token); long id; Database.Tasks.AddTask(task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.ImportDatabase, Payload = fileName, }, out id, cts); return(GetMessageWithObject(new { OperationId = id })); }