public void Create(string fileName, ConflictItem conflict) { RavenJObject metadata = null; storage.Batch( accessor => { metadata = accessor.GetFile(fileName, 0, 0).Metadata; accessor.SetConfig(RavenFileNameHelper.ConflictConfigNameForFile(fileName), JsonExtensions.ToJObject(conflict)); metadata[SynchronizationConstants.RavenSynchronizationConflict] = true; accessor.UpdateFileMetadata(fileName, metadata); }); if (metadata != null) { index.Index(fileName, metadata); } }
public static IndexDefinition CreateIndexDefinition() { // this is how the index looks like in JSON var jsonIndexDefinition = @" { ""Map"" : "" from e in docs.Events select new { Tag = \""Event\"", _lat = SpatialIndex.Lat(e.Latitude), _lng = SpatialIndex.Lng(e.Longitude), _tier_2 = SpatialIndex.Tier(2, e.Latitude, e.Longitude), _tier_3 = SpatialIndex.Tier(3, e.Latitude, e.Longitude), _tier_4 = SpatialIndex.Tier(4, e.Latitude, e.Longitude), _tier_5 = SpatialIndex.Tier(5, e.Latitude, e.Longitude), _tier_6 = SpatialIndex.Tier(6, e.Latitude, e.Longitude), _tier_7 = SpatialIndex.Tier(7, e.Latitude, e.Longitude), _tier_8 = SpatialIndex.Tier(8, e.Latitude, e.Longitude), _tier_9 = SpatialIndex.Tier(9, e.Latitude, e.Longitude), _tier_10 = SpatialIndex.Tier(10, e.Latitude, e.Longitude), _tier_11 = SpatialIndex.Tier(11, e.Latitude, e.Longitude), _tier_12 = SpatialIndex.Tier(12, e.Latitude, e.Longitude), _tier_13 = SpatialIndex.Tier(13, e.Latitude, e.Longitude), _tier_14 = SpatialIndex.Tier(14, e.Latitude, e.Longitude), _tier_15 = SpatialIndex.Tier(15, e.Latitude, e.Longitude) }"", ""Stores"" :{ ""latitude"" : ""Yes"", ""longitude"" : ""Yes"", ""_tier_2"" : ""Yes"", ""_tier_3"" : ""Yes"", ""_tier_4"" : ""Yes"", ""_tier_5"" : ""Yes"", ""_tier_6"" : ""Yes"", ""_tier_7"" : ""Yes"", ""_tier_8"" : ""Yes"", ""_tier_9"" : ""Yes"", ""_tier_10"" : ""Yes"", ""_tier_11"" : ""Yes"", ""_tier_12"" : ""Yes"", ""_tier_13"" : ""Yes"", ""_tier_14"" : ""Yes"", ""_tier_15"" : ""Yes"" }, ""Indexes"" :{ ""Tag"" : ""NotAnalyzed"", ""latitude"" : ""NotAnalyzed"", ""longitude"" : ""NotAnalyzed"", ""_tier_2"" : ""NotAnalyzedNoNorms"", ""_tier_3"" : ""NotAnalyzedNoNorms"", ""_tier_4"" : ""NotAnalyzedNoNorms"", ""_tier_5"" : ""NotAnalyzedNoNorms"", ""_tier_6"" : ""NotAnalyzedNoNorms"", ""_tier_7"" : ""NotAnalyzedNoNorms"", ""_tier_8"" : ""NotAnalyzedNoNorms"", ""_tier_9"" : ""NotAnalyzedNoNorms"", ""_tier_10"" : ""NotAnalyzedNoNorms"", ""_tier_11"" : ""NotAnalyzedNoNorms"", ""_tier_12"" : ""NotAnalyzedNoNorms"", ""_tier_13"" : ""NotAnalyzedNoNorms"", ""_tier_14"" : ""NotAnalyzedNoNorms"", ""_tier_15"" : ""NotAnalyzedNoNorms"" } }"; using (var stringReader = new StringReader(jsonIndexDefinition)) using (var jsonReader = new JsonTextReader(stringReader)) { return(JsonExtensions.CreateDefaultJsonSerializer().Deserialize <IndexDefinition>(jsonReader)); } }
/// <summary> /// Writes this token to a <see cref="JsonWriter"/>. /// </summary> /// <param name="writer">A <see cref="JsonWriter"/> into which this method will write.</param> /// <param name="converters">A collection of <see cref="JsonConverter"/> which will be used when writing the token.</param> public override void WriteTo(JsonWriter writer, params JsonConverter[] converters) { switch (_valueType) { case JTokenType.Comment: writer.WriteComment(_value.ToString()); return; case JTokenType.Raw: writer.WriteRawValue((_value != null) ? _value.ToString() : null); return; case JTokenType.Null: writer.WriteNull(); return; case JTokenType.Undefined: writer.WriteUndefined(); return; } JsonConverter matchingConverter; if (_value != null && ((matchingConverter = GetMatchingConverter(converters, _value.GetType())) != null)) { matchingConverter.WriteJson(writer, _value, JsonExtensions.CreateDefaultJsonSerializer()); return; } switch (_valueType) { case JTokenType.Integer: writer.WriteValue(Convert.ToInt64(_value, CultureInfo.InvariantCulture)); return; case JTokenType.Float: if (_value is decimal) { writer.WriteValue((decimal)_value); return; } if (_value is float) { writer.WriteValue((float)_value); return; } writer.WriteValue(Convert.ToDouble(_value, CultureInfo.InvariantCulture)); return; case JTokenType.String: writer.WriteValue((_value != null) ? _value.ToString() : null); return; case JTokenType.Boolean: writer.WriteValue(Convert.ToBoolean(_value, CultureInfo.InvariantCulture)); return; case JTokenType.Date: #if !PocketPC && !NET20 if (_value is DateTimeOffset) { writer.WriteValue((DateTimeOffset)_value); } else #endif writer.WriteValue(Convert.ToDateTime(_value, CultureInfo.InvariantCulture)); return; case JTokenType.Bytes: writer.WriteValue((byte[])_value); return; case JTokenType.Guid: case JTokenType.Uri: case JTokenType.TimeSpan: writer.WriteValue((_value != null) ? _value.ToString() : null); return; } throw MiscellaneousUtils.CreateArgumentOutOfRangeException("TokenType", _valueType, "Unexpected token type."); }
public static void SetConfigurationValue <T>(this IStorageActionsAccessor accessor, string key, T objectToSave) { accessor.SetConfig(key, JsonExtensions.ToJObject(objectToSave)); }
private void SaveSynchronizationReport(string fileName, IStorageActionsAccessor accessor, SynchronizationReport report) { var name = RavenFileNameHelper.SyncResultNameForFile(fileName); accessor.SetConfig(name, JsonExtensions.ToJObject(report)); }
private void TimerCallback(bool fullBackup) { if (currentTask != null) { return; } if (Database.Disposed) { Dispose(); return; } // we have shared lock for both incremental and full backup. lock (this) { if (currentTask != null) { return; } currentTask = Task.Factory.StartNew(async() => { var documentDatabase = Database; if (documentDatabase == null) { return; } using (LogContext.WithResource(documentDatabase.Name)) { try { OperationState exportResult; bool performAnotherRun = false; do { var dataDumper = new DatabaseDataDumper(documentDatabase, new SmugglerDatabaseOptions() { Limit = backupLimit }); var localBackupConfigs = exportConfigs; var localBackupStatus = exportStatus; if (localBackupConfigs == null) { return; } if (localBackupConfigs.Disabled) { return; } var backupLocally = string.IsNullOrEmpty(localBackupConfigs.LocalFolderName) == false; if (fullBackup == false && backupLocally) { if (string.IsNullOrEmpty(localBackupStatus.LastFullLocalBackupFolder) || Directory.Exists(localBackupStatus.LastFullLocalBackupFolder) == false || Directory.GetFiles(localBackupStatus.LastFullLocalBackupFolder, "*", SearchOption.TopDirectoryOnly).Length == 0) { // there must be a full backup before we can start an incremental backup // and the last full backup folder must exist and not empty fullBackup = true; } } if (fullBackup == false) { var currentEtags = dataDumper.Operations.FetchCurrentMaxEtags(); // No-op if nothing has changed if (currentEtags.LastDocsEtag == localBackupStatus.LastDocsEtag && currentEtags.LastAttachmentsEtag == localBackupStatus.LastAttachmentsEtag && currentEtags.LastDocDeleteEtag == localBackupStatus.LastDocsDeletionEtag && currentEtags.LastAttachmentsDeleteEtag == localBackupStatus.LastAttachmentDeletionEtag) { return; } } var formattedDate = SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture); string backupPath; string lastFullLocalBackupFolder = null; switch (backupLocally) { case false: // will need a temp folder to store the backup file backupPath = Path.Combine(documentDatabase.Configuration.DataDirectory, "PeriodicExport-Temp"); break; default: backupPath = lastFullLocalBackupFolder = GetLocalBackupPath(fullBackup, formattedDate, localBackupConfigs.LocalFolderName, localBackupStatus.LastFullLocalBackupFolder); break; } if (Directory.Exists(backupPath) == false) { Directory.CreateDirectory(backupPath); } if (fullBackup) { // create filename for full dump var tempBackupPath = Path.Combine(backupPath, formattedDate + ".ravendb-full-dump"); var counter = 1; while (File.Exists(tempBackupPath)) { tempBackupPath = Path.Combine(backupPath, formattedDate + " - " + (counter++) + ".ravendb-full-dump"); } backupPath = tempBackupPath; } var smugglerOptions = dataDumper.Options; if (fullBackup == false) { smugglerOptions.StartDocsEtag = localBackupStatus.LastDocsEtag; smugglerOptions.StartAttachmentsEtag = localBackupStatus.LastAttachmentsEtag; smugglerOptions.StartDocsDeletionEtag = localBackupStatus.LastDocsDeletionEtag; smugglerOptions.StartAttachmentsDeletionEtag = localBackupStatus.LastAttachmentDeletionEtag; smugglerOptions.Incremental = true; smugglerOptions.ExportDeletions = true; } exportResult = await dataDumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions> { ToFile = backupPath }).ConfigureAwait(false); if (fullBackup == false) { // No-op if nothing has changed if (exportResult.LastDocsEtag == localBackupStatus.LastDocsEtag && exportResult.LastAttachmentsEtag == localBackupStatus.LastAttachmentsEtag && exportResult.LastDocDeleteEtag == localBackupStatus.LastDocsDeletionEtag && exportResult.LastAttachmentsDeleteEtag == localBackupStatus.LastAttachmentDeletionEtag) { logger.Info("Periodic export returned prematurely, nothing has changed since last export"); return; } } try { await UploadToServer(exportResult.FilePath, localBackupConfigs, fullBackup).ConfigureAwait(false); } finally { // if user did not specify local folder we delete temporary file. if (backupLocally == false) { IOExtensions.DeleteFile(exportResult.FilePath); } } localBackupStatus.LastAttachmentsEtag = exportResult.LastAttachmentsEtag; localBackupStatus.LastDocsEtag = exportResult.LastDocsEtag; localBackupStatus.LastDocsDeletionEtag = exportResult.LastDocDeleteEtag; localBackupStatus.LastAttachmentDeletionEtag = exportResult.LastAttachmentsDeleteEtag; if (fullBackup) { localBackupStatus.LastFullLocalBackupFolder = lastFullLocalBackupFolder; localBackupStatus.LastFullBackup = SystemTime.UtcNow; } else { localBackupStatus.LastBackup = SystemTime.UtcNow; } var ravenJObject = JsonExtensions.ToJObject(localBackupStatus); ravenJObject.Remove("Id"); var putResult = documentDatabase.Documents.Put(PeriodicExportStatus.RavenDocumentKey, null, ravenJObject, new RavenJObject(), null); // this result in exportStatus being refreshed localBackupStatus = exportStatus; if (localBackupStatus != null) { if (localBackupStatus.LastDocsEtag.IncrementBy(1) == putResult.ETag) // the last etag is with just us { localBackupStatus.LastDocsEtag = putResult.ETag; // so we can skip it for the next time } } if (backupLimit != int.MaxValue) { backupLimit = int.MaxValue; performAnotherRun = true; } else { performAnotherRun = false; } } while (performAnotherRun); } catch (ObjectDisposedException) { // shutting down, probably } catch (OperationCanceledException) { // shutting down, probably } catch (Exception e) { backupLimit = 100; logger.ErrorException("Error when performing periodic export", e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = e.Message, Title = "Error in Periodic Export", Exception = e.ToString(), UniqueKey = "Periodic Export Error", }); } } }) .Unwrap(); currentTask.ContinueWith(_ => { currentTask = null; }); } }
public object Deserialize(Type type, MemoryStream source) { string data = Encoding.UTF8.GetString(source.ToArray()); return(JsonExtensions.DeserializeObject(type, JSON.Parse(data))); }
public Task <HttpResponseMessage> ExportDatabase([FromBody] ExportData smugglerOptionsJson) { var result = GetEmptyMessage(); var taskId = smugglerOptionsJson.ProgressTaskId; var requestString = smugglerOptionsJson.DownloadOptions; SmugglerDatabaseOptions smugglerOptions; using (var jsonReader = new RavenJsonTextReader(new StringReader(requestString))) { var serializer = JsonExtensions.CreateDefaultJsonSerializer(); smugglerOptions = (SmugglerDatabaseOptions)serializer.Deserialize(jsonReader, typeof(SmugglerDatabaseOptions)); } var fileName = string.IsNullOrEmpty(smugglerOptions.NoneDefaultFileName) || (smugglerOptions.NoneDefaultFileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) ? $"Dump of {DatabaseName}, {DateTime.Now.ToString("yyyy-MM-dd HH-mm", CultureInfo.InvariantCulture)}" : smugglerOptions.NoneDefaultFileName; //create PushStreamContent object that will be called when the output stream will be ready. result.Content = new PushStreamContent(async(outputStream, content, arg3) => { var status = new DataDumperOperationStatus(); var tcs = new TaskCompletionSource <object>(); var sp = Stopwatch.StartNew(); try { Database.Tasks.AddTask(tcs.Task, status, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.ExportDatabase, Description = "Exporting database, file name: " + fileName }, taskId, smugglerOptions.CancelToken, skipStatusCheck: true); var dataDumper = new DatabaseDataDumper(Database, smugglerOptions); dataDumper.Progress += s => status.MarkProgress(s); await dataDumper.ExportData( new SmugglerExportOptions <RavenConnectionStringOptions> { ToStream = outputStream }).ConfigureAwait(false); const string message = "Completed export"; status.MarkCompleted(message, sp.Elapsed); } catch (OperationCanceledException e) { status.MarkCanceled(e.Message); } catch (Exception e) { status.ExceptionDetails = e.ToString(); status.MarkFaulted(e.ToString()); throw; } finally { tcs.SetResult("Completed"); outputStream.Close(); } }); result.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = fileName + ".ravendbdump" }; return(new CompletedTask <HttpResponseMessage>(result)); }
public HttpResponseMessage Patch(string name, string rename) { name = FileHeader.Canonize(name); rename = FileHeader.Canonize(rename); try { ConcurrencyAwareExecutor.Execute(() => Storage.Batch(accessor => { AssertFileIsNotBeingSynced(name, accessor, true); var metadata = accessor.GetFile(name, 0, 0).Metadata; if (metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker)) { throw new FileNotFoundException(); } var existingHeader = accessor.ReadFile(rename); if (existingHeader != null && !existingHeader.Metadata.ContainsKey(SynchronizationConstants.RavenDeleteMarker)) { throw new HttpResponseException( Request.CreateResponse(HttpStatusCode.Forbidden, new InvalidOperationException("Cannot rename because file " + rename + " already exists"))); } Historian.UpdateLastModified(metadata); var operation = new RenameFileOperation { FileSystem = FileSystem.Name, Name = name, Rename = rename, MetadataAfterOperation = metadata }; accessor.SetConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation)); accessor.PulseTransaction(); // commit rename operation config StorageOperationsTask.RenameFile(operation); }), ConcurrencyResponseException); } catch (FileNotFoundException) { log.Debug("Cannot rename a file '{0}' to '{1}' because a file was not found", name, rename); return(GetEmptyMessage(HttpStatusCode.NotFound)); } log.Debug("File '{0}' was renamed to '{1}'", name, rename); StartSynchronizeDestinationsInBackground(); return(GetMessageWithString("", HttpStatusCode.NoContent)); }
/// <summary> /// jsonからわざリストを取得する /// </summary> public static MoveData[] LoadMoves() { return(JsonExtensions.DeserializeFromFile <MoveData[]>(FilePath.MovesDataPath) ?? Array.Empty <MoveData>()); }
private void GetOldEvents <T>(Expression <Func <MessagesContext, DbSet <T> > > func, string settings) where T : MessageEvent { List <T> ids; var compile = func.Compile(); do { using var scope = ServiceProvider.CreateScope(); using var ef = scope.ServiceProvider.GetService <DbContextManager <MessagesContext> >().Get("messages"); var table = compile.Invoke(ef); var ae = table .Join(ef.Tenants, r => r.TenantId, r => r.Id, (audit, tenant) => audit) .Select(r => new { r.Id, r.Date, r.TenantId, ef = r }) .Where(r => r.Date < DateTime.UtcNow.AddDays(-Convert.ToDouble( ef.WebstudioSettings .Where(a => a.TenantId == r.TenantId && a.Id == TenantAuditSettings.Guid) .Select(r => JsonExtensions.JsonValue(nameof(r.Data).ToLower(), settings)) .FirstOrDefault() ?? TenantAuditSettings.MaxLifeTime.ToString()))) .Take(1000); ids = ae.Select(r => r.ef).ToList(); if (!ids.Any()) { return; } table.RemoveRange(ids); ef.SaveChanges(); } while (ids.Any()); }
/// <summary> /// jsonからポケモンリストを取得する /// </summary> public static PokemonData[] LoadForms() { return(JsonExtensions.DeserializeFromFile <PokemonData[]>(FilePath.PokemonFormDataPath) ?? Array.Empty <PokemonData>()); }
/// <summary> /// jsonから特性リストを取得する /// </summary> public static AbilityData[] LoadAbilities() { return(JsonExtensions.DeserializeFromFile <AbilityData[]>(FilePath.AbilitiesDataPath) ?? Array.Empty <AbilityData>()); }
public void CanHandleCaching() { using (GetNewServer()) using (var docStore = new DocumentStore { Url = "http://localhost:8079" }.Initialize()) { using (var session = docStore.OpenSession()) { session.Store(new User { Name = "Ayende" }); session.Store(new User { Name = "Oren" }); session.SaveChanges(); } var request = (HttpWebRequest)WebRequest.Create("http://localhost:8079/multi_get"); request.Method = "POST"; using (var stream = request.GetRequestStream()) { var streamWriter = new StreamWriter(stream); JsonExtensions.CreateDefaultJsonSerializer().Serialize(streamWriter, new[] { new GetRequest { Url = "/docs/users/1" }, new GetRequest { Url = "/docs/users/2" }, }); streamWriter.Flush(); stream.Flush(); } GetResponse[] results; using (var resp = request.GetResponse()) using (var stream = resp.GetResponseStream()) { var result = new StreamReader(stream).ReadToEnd(); results = JsonConvert.DeserializeObject <GetResponse[]>(result, Default.Converters); Assert.True(results[0].Headers.ContainsKey("ETag")); Assert.True(results[1].Headers.ContainsKey("ETag")); } request = (HttpWebRequest)WebRequest.Create("http://localhost:8079/multi_get"); request.Method = "POST"; using (var stream = request.GetRequestStream()) { var streamWriter = new StreamWriter(stream); JsonExtensions.CreateDefaultJsonSerializer().Serialize(streamWriter, new[] { new GetRequest { Url = "/docs/users/1", Headers = { { "If-None-Match", results[0].Headers["ETag"] } } }, new GetRequest { Url = "/docs/users/2", Headers = { { "If-None-Match", results[1].Headers["ETag"] } } }, }); streamWriter.Flush(); stream.Flush(); } using (var resp = request.GetResponse()) using (var stream = resp.GetResponseStream()) { var result = new StreamReader(stream).ReadToEnd(); results = JsonConvert.DeserializeObject <GetResponse[]>(result, Default.Converters); Assert.Equal(304, results[0].Status); Assert.Equal(304, results[1].Status); } } }
public virtual async Task OnAuthorizationAsync(AuthorizationFilterContext context) { if (context == null) { throw new ArgumentNullException("context"); } AuthorizationPolicy effectivePolicy = Policy; if (effectivePolicy == null) { if (PolicyProvider == null) { throw new InvalidOperationException("An AuthorizationPolicy cannot be created without a valid instance of IAuthorizationPolicyProvider."); } effectivePolicy = await AuthorizationPolicy.CombineAsync(PolicyProvider, AuthorizeData); } if (effectivePolicy != null) { MvcPrincipal newPrincipal = null; string currentScheme = effectivePolicy.AuthenticationSchemes.FirstOrDefault(); if (!string.IsNullOrEmpty(currentScheme)) { if (!(context.HttpContext.User.Identity is MvcIdentity) || !context.HttpContext.User.Identity.IsAuthenticated) { string cookie = CookieUtility.GetCookie(currentScheme, true); if (!string.IsNullOrEmpty(cookie)) { try { string value = ServiceCollectionExtension.Decrypt(cookie); MvcIdentity identity = JsonExtensions.GetModel <MvcIdentity>(value, ""); if (identity != null) { newPrincipal = identity.GetPrincipal(); } } catch { } } } else { newPrincipal = (context.HttpContext.User as MvcPrincipal); } } if (newPrincipal == null) { context.HttpContext.User = MvcIdentity.Instance.GetPrincipal(); } else { context.HttpContext.User = newPrincipal; } if (!context.Filters.Any((IFilterMetadata item) => item is IAllowAnonymousFilter)) { if (context.HttpContext.User.Identity.IsAuthenticated) { if (AuthorizeFilter == null) { AuthorizeFilter = ServiceProviderServiceExtensions.GetService <IAuthorizeFilter>(context.HttpContext.RequestServices); } if (AuthorizeFilter != null) { await AuthorizeFilter.OnAuthorizedAsync(context, currentScheme); } } else { context.Result = new ChallengeResult(effectivePolicy.AuthenticationSchemes.ToArray()); } } } }
private static (Object[], ImportAssetsLog) ImportAssets(IAkyuiImportSettings settings, IAkyuiLoader akyuiLoader, PathGetter pathGetter, AkyuiLogger logger, IAkyuiProgress progress) { var stopWatch = Stopwatch.StartNew(); var assets = new List <Object>(); var unityAssetsParentPath = Path.GetDirectoryName(Application.dataPath) ?? ""; var assetOutputDirectoryFullPath = Path.Combine(unityAssetsParentPath, pathGetter.AssetOutputDirectoryPath); if (!Directory.Exists(assetOutputDirectoryFullPath)) { Directory.CreateDirectory(assetOutputDirectoryFullPath); } var importAssetNames = new List <string>(); var skipAssetNames = new List <string>(); var allAssets = akyuiLoader.AssetsInfo.Assets.ToList(); foreach (var trigger in settings.Triggers) { trigger.OnPreprocessAllAssets(akyuiLoader, ref allAssets); } progress.SetTotal(allAssets.Count); foreach (var tmp in allAssets) { var asset = tmp; using (progress.TaskStart(asset.FileName)) { var savePath = Path.Combine(pathGetter.AssetOutputDirectoryPath, asset.FileName); var saveFullPath = Path.Combine(unityAssetsParentPath, savePath); if (!settings.ReimportAsset && File.Exists(saveFullPath)) { var import = AssetImporter.GetAtPath(savePath); var prevUserData = JsonSerializer.Deserialize <Dictionary <string, object> >(import.userData); if (JsonExtensions.ToUint(prevUserData["hash"]) == asset.Hash) { skipAssetNames.Add(asset.FileName); assets.Add(AssetDatabase.LoadAssetAtPath <Object>(import.assetPath)); continue; } } var bytes = akyuiLoader.LoadAsset(asset.FileName); // Hashチェック後に初めて呼ぶ var userData = new Dictionary <string, object>(); userData["hash"] = asset.Hash; if (asset is SpriteAsset) { var texture = new Texture2D(2, 2); texture.LoadImage(bytes); userData["source_width"] = texture.width; userData["source_height"] = texture.height; } foreach (var trigger in settings.Triggers) { trigger.OnPreprocessAsset(akyuiLoader, ref bytes, ref asset, ref userData); } ImportAsset(asset, savePath, saveFullPath, bytes, userData, settings, logger); assets.Add(AssetDatabase.LoadAssetAtPath <Object>(savePath)); importAssetNames.Add(asset.FileName); } } var importAssets = assets.ToArray(); foreach (var trigger in settings.Triggers) { trigger.OnPostprocessAllAssets(akyuiLoader, importAssets); } return( importAssets, new ImportAssetsLog { Time = stopWatch.Elapsed.TotalSeconds, Import = importAssetNames.Count, Skip = skipAssetNames.Count } ); }
private void GivenTheMessageHasTheApproveProvidersRequest(PublishedProviderIdsRequest approveProvidersRequest) { _message.Body = Encoding.UTF8.GetBytes(JsonExtensions.AsJson(approveProvidersRequest)); }
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize, CancellationToken token, ref int nextStart, Action <RavenJObject> addDoc, string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, string skipAfter = null) { if (idPrefix == null) { throw new ArgumentNullException("idPrefix"); } idPrefix = idPrefix.Trim(); var canPerformRapidPagination = nextStart > 0 && start == nextStart; var actualStart = canPerformRapidPagination ? start : 0; var addedDocs = 0; var matchedDocs = 0; TransactionalStorage.Batch( actions => { var docsToSkip = canPerformRapidPagination ? 0 : start; int docCount; AbstractTransformer storedTransformer = null; if (transformer != null) { storedTransformer = IndexDefinitionStorage.GetTransformer(transformer); if (storedTransformer == null) { throw new InvalidOperationException("No transformer with the name: " + transformer); } } do { docCount = 0; var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter); var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters); foreach (var doc in docs) { token.ThrowIfCancellationRequested(); docCount++; var keyTest = doc.Key.Substring(idPrefix.Length); if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest)) { continue; } DocumentRetriever.EnsureIdInMetadata(doc); var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key); var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc; document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load); if (document == null) { continue; } matchedDocs++; if (matchedDocs <= docsToSkip) { continue; } token.ThrowIfCancellationRequested(); if (storedTransformer != null) { using (new CurrentTransformationScope(Database, documentRetriever)) { var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) }) .Select(x => JsonExtensions.ToJObject(x)) .ToArray(); if (transformed.Length == 0) { throw new InvalidOperationException("The transform results function failed on a document: " + document.Key); } var transformedJsonDocument = new JsonDocument { Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag), NonAuthoritativeInformation = document.NonAuthoritativeInformation, LastModified = document.LastModified, DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } }, }; addDoc(transformedJsonDocument.ToJson()); } } else { addDoc(document.ToJson()); } addedDocs++; if (addedDocs >= pageSize) { break; } } actualStart += pageSize; }while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue); }); if (addedDocs != pageSize) { nextStart = start; // will mark as last page } else if (canPerformRapidPagination) { nextStart = start + matchedDocs; } else { nextStart = actualStart; } }
public T Deserialize <T>(string data) { return(JsonExtensions.DeserializeObject <T>(data)); }
private static void DumpStacktrace(ZipArchive package) { var stacktrace = package.CreateEntry("stacktraces.txt", CompressionLevel.Optimal); var jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer(); jsonSerializer.Formatting = Formatting.Indented; using (var stacktraceStream = stacktrace.Open()) { string ravenDebugDir = null; try { if (Debugger.IsAttached) { throw new InvalidOperationException("Cannot get stacktraces when debugger is attached"); } ravenDebugDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); var ravenDebugExe = Path.Combine(ravenDebugDir, "Raven.Debug.exe"); var ravenDebugOutput = Path.Combine(ravenDebugDir, "stacktraces.txt"); Directory.CreateDirectory(ravenDebugDir); if (Environment.Is64BitProcess) { ExtractResource("Raven.Database.Util.Raven.Debug.x64.Raven.Debug.exe", ravenDebugExe); } else { ExtractResource("Raven.Database.Util.Raven.Debug.x86.Raven.Debug.exe", ravenDebugExe); } var process = new Process { StartInfo = new ProcessStartInfo { Arguments = string.Format("--pid={0} --stacktrace --output=\"{1}\"", Process.GetCurrentProcess().Id, ravenDebugOutput), FileName = ravenDebugExe, WindowStyle = ProcessWindowStyle.Normal, LoadUserProfile = true, RedirectStandardError = true, RedirectStandardOutput = true, UseShellExecute = false }, EnableRaisingEvents = true }; var output = string.Empty; process.OutputDataReceived += (sender, args) => output += args.Data; process.ErrorDataReceived += (sender, args) => output += args.Data; process.Start(); process.BeginErrorReadLine(); process.BeginOutputReadLine(); process.WaitForExit(); if (process.ExitCode != 0) { throw new InvalidOperationException("Raven.Debug exit code is: " + process.ExitCode + Environment.NewLine + "Message: " + output); } using (var stackDumpOutputStream = File.Open(ravenDebugOutput, FileMode.Open)) { stackDumpOutputStream.CopyTo(stacktraceStream); } } catch (Exception ex) { var streamWriter = new StreamWriter(stacktraceStream); jsonSerializer.Serialize(streamWriter, new { Error = "Exception occurred during getting stacktraces of the RavenDB process. Exception: " + ex }); streamWriter.Flush(); } finally { if (ravenDebugDir != null && Directory.Exists(ravenDebugDir)) { IOExtensions.DeleteDirectory(ravenDebugDir); } } stacktraceStream.Flush(); } }
public object Deserialize(Type type, JSONNode data) { //string data = Encoding.UTF8.GetString(source.ToArray()); return(JsonExtensions.DeserializeObject(type, data)); }
public HttpResponseMessage ResolveConflict(string filename, ConflictResolutionStrategy strategy) { var canonicalFilename = FileHeader.Canonize(filename); if (Log.IsDebugEnabled) { Log.Debug("Resolving conflict of a file '{0}' by using {1} strategy", filename, strategy); } switch (strategy) { case ConflictResolutionStrategy.CurrentVersion: Storage.Batch(accessor => { var localMetadata = accessor.GetFile(canonicalFilename, 0, 0).Metadata; var conflict = accessor.GetConfigurationValue <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile(canonicalFilename)); ConflictResolver.ApplyCurrentStrategy(canonicalFilename, conflict, localMetadata); accessor.UpdateFileMetadata(canonicalFilename, localMetadata, null); ConflictArtifactManager.Delete(canonicalFilename, accessor); }); Publisher.Publish(new ConflictNotification { FileName = canonicalFilename, Status = ConflictStatus.Resolved }); break; case ConflictResolutionStrategy.RemoteVersion: Storage.Batch(accessor => { var localMetadata = accessor.GetFile(canonicalFilename, 0, 0).Metadata; var conflict = accessor.GetConfig(RavenFileNameHelper.ConflictConfigNameForFile(canonicalFilename)).JsonDeserialization <ConflictItem>(); ConflictResolver.ApplyRemoteStrategy(canonicalFilename, conflict, localMetadata); accessor.UpdateFileMetadata(canonicalFilename, localMetadata, null); accessor.SetConfig(RavenFileNameHelper.ConflictConfigNameForFile(canonicalFilename), JsonExtensions.ToJObject(conflict)); // ConflictArtifactManager.Delete(canonicalFilename, accessor); - intentionally not deleting, conflict item will be removed when a remote file is put }); SynchronizationTask.Context.NotifyAboutWork(); break; default: throw new NotSupportedException(string.Format("{0} is not the valid strategy to resolve a conflict", strategy)); } return(GetEmptyMessage(HttpStatusCode.NoContent)); }
private void TimerCallback(object state) { if (currentTask != null) { return; } lock (this) { if (currentTask != null) { return; } currentTask = Task.Factory.StartNew(async() => { var documentDatabase = Database; if (documentDatabase == null) { return; } using (LogContext.WithDatabase(documentDatabase.Name)) { try { var localBackupConfigs = backupConfigs; var localBackupStatus = backupStatus; if (localBackupConfigs == null) { return; } var databaseStatistics = documentDatabase.Statistics; // No-op if nothing has changed if (databaseStatistics.LastDocEtag == localBackupStatus.LastDocsEtag && databaseStatistics.LastAttachmentEtag == localBackupStatus.LastAttachmentsEtag) { return; } var backupPath = localBackupConfigs.LocalFolderName ?? Path.Combine(documentDatabase.Configuration.DataDirectory, "PeriodicBackup-Temp"); var options = new SmugglerOptions { BackupPath = backupPath, LastDocsEtag = localBackupStatus.LastDocsEtag, LastAttachmentEtag = localBackupStatus.LastAttachmentsEtag }; var dd = new DataDumper(documentDatabase, options); string filePath; try { filePath = await dd.ExportData(null, null, true, backupStatus); } catch (SmugglerExportException e) { filePath = e.File; logger.ErrorException("Recoverable error when performing periodic backup", e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = e.Message, Title = "Recoverable Error in Periodic Backup", Exception = e.ToString(), UniqueKey = "Periodic Backup Error", }); } // No-op if nothing has changed if (options.LastDocsEtag == localBackupStatus.LastDocsEtag && options.LastAttachmentEtag == localBackupStatus.LastAttachmentsEtag) { logger.Info("Periodic backup returned prematurely, nothing has changed since last backup"); return; } try { UploadToServer(filePath, localBackupConfigs); } finally { // we delete file only with some upload option was selected if (string.IsNullOrEmpty(localBackupConfigs.LocalFolderName)) { File.Delete(filePath); } } localBackupStatus.LastAttachmentsEtag = options.LastAttachmentEtag; localBackupStatus.LastDocsEtag = options.LastDocsEtag; localBackupStatus.LastBackup = SystemTime.UtcNow; var ravenJObject = JsonExtensions.ToJObject(localBackupStatus); ravenJObject.Remove("Id"); var putResult = documentDatabase.Put(PeriodicBackupStatus.RavenDocumentKey, null, ravenJObject, new RavenJObject(), null); // this result in backupStatus being refreshed localBackupStatus = backupStatus; if (localBackupStatus != null) { if (localBackupStatus.LastDocsEtag.IncrementBy(1) == putResult.ETag) // the last etag is with just us { localBackupStatus.LastDocsEtag = putResult.ETag; // so we can skip it for the next time } } } catch (ObjectDisposedException) { // shutting down, probably } catch (Exception e) { logger.ErrorException("Error when performing periodic backup", e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = e.Message, Title = "Error in Periodic Backup", Exception = e.ToString(), UniqueKey = "Periodic Backup Error", }); } } }) .ContinueWith(_ => { currentTask = null; }); } }
protected Task <T> Put <T>(string url, object data) where T : BaseResponse { return(Put <T>(url, JsonExtensions.Serialize(data))); }
public override void WriteTo(JsonWriter writer, JsonConverterCollection converters) { switch (_valueType) { case JTokenType.Comment: writer.WriteComment(_value.ToString()); return; case JTokenType.Raw: writer.WriteRawValue((_value != null) ? _value.ToString() : null); return; case JTokenType.Null: writer.WriteNull(); return; case JTokenType.Undefined: writer.WriteUndefined(); return; } if (_value != null) { Type typeToFind = _value.GetType(); // If we are using the default converters we will try to avoid repeatedly check the same types as // GetMatchingConverter is a costly call with a very low probability to hit (less than 1% in real scenarios). JsonConverter matchingConverter = JsonConverterCache.GetMatchingConverter(converters, typeToFind); if (matchingConverter != null) { matchingConverter.WriteJson(writer, _value, JsonExtensions.CreateDefaultJsonSerializer()); return; } } switch (_valueType) { case JTokenType.Integer: writer.WriteValue(Convert.ToInt64(_value, CultureInfo.InvariantCulture)); return; case JTokenType.Float: if (_value is decimal) { writer.WriteValue((decimal)_value); return; } if (_value is float) { writer.WriteValue((float)_value); return; } writer.WriteValue(Convert.ToDouble(_value, CultureInfo.InvariantCulture)); return; case JTokenType.String: writer.WriteValue((_value != null) ? _value.ToString() : null); return; case JTokenType.Boolean: writer.WriteValue(Convert.ToBoolean(_value, CultureInfo.InvariantCulture)); return; case JTokenType.Date: #if !PocketPC && !NET20 if (_value is DateTimeOffset) { writer.WriteValue((DateTimeOffset)_value); } else #endif writer.WriteValue(Convert.ToDateTime(_value, CultureInfo.InvariantCulture)); return; case JTokenType.Bytes: writer.WriteValue((byte[])_value); return; case JTokenType.Guid: case JTokenType.Uri: case JTokenType.TimeSpan: writer.WriteValue((_value != null) ? _value.ToString() : null); return; } throw MiscellaneousUtils.CreateArgumentOutOfRangeException("TokenType", _valueType, "Unexpected token type."); }
public static Task ToTask(string taskType, byte[] task) { var type = typeof(Task).Assembly.GetType(taskType); return((Task)JsonExtensions.CreateDefaultJsonSerializer().Deserialize(new BsonReader(new MemoryStream(task)), type)); }
public void IndicateFileToDelete(string fileName) { var deletingFileName = RavenFileNameHelper.DeletingFileName(fileName); var fileExists = true; storage.Batch(accessor => { var existingFileHeader = accessor.ReadFile(fileName); if (existingFileHeader == null) { // do nothing if file does not exist fileExists = false; return; } if (existingFileHeader.Metadata[SynchronizationConstants.RavenDeleteMarker] != null) { // if it is a tombstone drop it accessor.Delete(fileName); fileExists = false; return; } var metadata = new RavenJObject(existingFileHeader.Metadata).WithDeleteMarker(); var renameSucceeded = false; int deleteVersion = 0; do { try { accessor.RenameFile(fileName, deletingFileName); renameSucceeded = true; } catch (FileExistsException) // it means that .deleting file was already existed { var deletingFileHeader = accessor.ReadFile(deletingFileName); if (deletingFileHeader != null && deletingFileHeader.Equals(existingFileHeader)) { fileExists = false; // the same file already marked as deleted no need to do it again return; } // we need to use different name to do a file rename deleteVersion++; deletingFileName = RavenFileNameHelper.DeletingFileName(fileName, deleteVersion); } } while (!renameSucceeded && deleteVersion < 128); if (renameSucceeded) { accessor.UpdateFileMetadata(deletingFileName, metadata); accessor.DecrementFileCount(deletingFileName); Log.Debug(string.Format("File '{0}' was renamed to '{1}' and marked as deleted", fileName, deletingFileName)); var configName = RavenFileNameHelper.DeleteOperationConfigNameForFile(deletingFileName); var operation = new DeleteFileOperation { OriginalFileName = fileName, CurrentFileName = deletingFileName }; accessor.SetConfig(configName, JsonExtensions.ToJObject(operation)); notificationPublisher.Publish(new ConfigurationChangeNotification { Name = configName, Action = ConfigurationChangeAction.Set }); } else { Log.Warn("Could not rename a file '{0}' when a delete operation was performed", fileName); } }); if (fileExists) { search.Delete(fileName); search.Delete(deletingFileName); } }
private void TimerCallback(bool fullBackup) { if (currentTask != null) { return; } if (Database.Disposed) { Dispose(); return; } // we have shared lock for both incremental and full backup. lock (this) { if (currentTask != null) { return; } currentTask = Task.Factory.StartNew(async() => { var documentDatabase = Database; if (documentDatabase == null) { return; } using (LogContext.WithDatabase(documentDatabase.Name)) { try { var dataDumper = new DatabaseDataDumper(documentDatabase); var localBackupConfigs = exportConfigs; var localBackupStatus = exportStatus; if (localBackupConfigs == null) { return; } if (fullBackup == false) { var currentEtags = dataDumper.Operations.FetchCurrentMaxEtags(); // No-op if nothing has changed if (currentEtags.LastDocsEtag == localBackupStatus.LastDocsEtag && currentEtags.LastAttachmentsEtag == localBackupStatus.LastAttachmentsEtag && currentEtags.LastDocDeleteEtag == localBackupStatus.LastDocsDeletionEtag && currentEtags.LastAttachmentsDeleteEtag == localBackupStatus.LastAttachmentDeletionEtag) { return; } } var backupPath = localBackupConfigs.LocalFolderName ?? Path.Combine(documentDatabase.Configuration.DataDirectory, "PeriodicExport-Temp"); if (fullBackup) { // create filename for full dump backupPath = Path.Combine(backupPath, SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + ".ravendb-full-dump"); if (File.Exists(backupPath)) { var counter = 1; while (true) { backupPath = Path.Combine(Path.GetDirectoryName(backupPath), SystemTime.UtcNow.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture) + " - " + counter + ".ravendb-full-dump"); if (File.Exists(backupPath) == false) { break; } counter++; } } } var smugglerOptions = dataDumper.Options; if (fullBackup == false) { smugglerOptions.StartDocsEtag = localBackupStatus.LastDocsEtag; smugglerOptions.StartAttachmentsEtag = localBackupStatus.LastAttachmentsEtag; smugglerOptions.StartDocsDeletionEtag = localBackupStatus.LastDocsDeletionEtag; smugglerOptions.StartAttachmentsDeletionEtag = localBackupStatus.LastAttachmentDeletionEtag; smugglerOptions.Incremental = true; smugglerOptions.ExportDeletions = true; } var exportResult = await dataDumper.ExportData(new SmugglerExportOptions <RavenConnectionStringOptions> { ToFile = backupPath }); if (fullBackup == false) { // No-op if nothing has changed if (exportResult.LastDocsEtag == localBackupStatus.LastDocsEtag && exportResult.LastAttachmentsEtag == localBackupStatus.LastAttachmentsEtag && exportResult.LastDocDeleteEtag == localBackupStatus.LastDocsDeletionEtag && exportResult.LastAttachmentsDeleteEtag == localBackupStatus.LastAttachmentDeletionEtag) { logger.Info("Periodic export returned prematurely, nothing has changed since last export"); return; } } try { if (!localBackupConfigs.Disabled) { UploadToServer(exportResult.FilePath, localBackupConfigs, fullBackup); } } finally { // if user did not specify local folder we delete temporary file. if (String.IsNullOrEmpty(localBackupConfigs.LocalFolderName)) { IOExtensions.DeleteFile(exportResult.FilePath); } } if (fullBackup) { localBackupStatus.LastFullBackup = SystemTime.UtcNow; } else { localBackupStatus.LastAttachmentsEtag = exportResult.LastAttachmentsEtag; localBackupStatus.LastDocsEtag = exportResult.LastDocsEtag; localBackupStatus.LastDocsDeletionEtag = exportResult.LastDocDeleteEtag; localBackupStatus.LastAttachmentDeletionEtag = exportResult.LastAttachmentsDeleteEtag; localBackupStatus.LastBackup = SystemTime.UtcNow; } var ravenJObject = JsonExtensions.ToJObject(localBackupStatus); ravenJObject.Remove("Id"); var putResult = documentDatabase.Documents.Put(PeriodicExportStatus.RavenDocumentKey, null, ravenJObject, new RavenJObject(), null); // this result in exportStatus being refreshed localBackupStatus = exportStatus; if (localBackupStatus != null) { if (localBackupStatus.LastDocsEtag.IncrementBy(1) == putResult.ETag) // the last etag is with just us { localBackupStatus.LastDocsEtag = putResult.ETag; // so we can skip it for the next time } } } catch (ObjectDisposedException) { // shutting down, probably } catch (Exception e) { logger.ErrorException("Error when performing periodic export", e); Database.AddAlert(new Alert { AlertLevel = AlertLevel.Error, CreatedAt = SystemTime.UtcNow, Message = e.Message, Title = "Error in Periodic Export", Exception = e.ToString(), UniqueKey = "Periodic Export Error", }); } } }) .Unwrap(); currentTask.ContinueWith(_ => { currentTask = null; }); } }
public HttpResponseMessage Patch(string name, string rename) { name = FileHeader.Canonize(name); rename = FileHeader.Canonize(rename); var etag = GetEtag(); if (rename.Length > SystemParameters.KeyMost) { Log.Debug("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename); return(GetMessageWithString(string.Format("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename), HttpStatusCode.BadRequest)); } Storage.Batch(accessor => { FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name); var existingFile = accessor.ReadFile(name); if (existingFile == null || existingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker)) { throw new FileNotFoundException(); } var renamingFile = accessor.ReadFile(rename); if (renamingFile != null && renamingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker) == false) { throw new FileExistsException("Cannot rename because file " + rename + " already exists"); } var metadata = existingFile.Metadata; if (etag != null && existingFile.Etag != etag) { throw new ConcurrencyException("Operation attempted on file '" + name + "' using a non current etag") { ActualETag = existingFile.Etag, ExpectedETag = etag } } ; Historian.UpdateLastModified(metadata); var operation = new RenameFileOperation(name, rename, existingFile.Etag, metadata); accessor.SetConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation)); accessor.PulseTransaction(); // commit rename operation config Files.ExecuteRenameOperation(operation); }); Log.Debug("File '{0}' was renamed to '{1}'", name, rename); SynchronizationTask.Context.NotifyAboutWork(); return(GetEmptyMessage(HttpStatusCode.NoContent)); }
public static dynamic SendPushNotificationP(string ExpoToken) { dynamic body = new { to = ExpoToken, title = "Notification", body = "You point is 0.", sound = "default", data = new { some = "daaaata" } }; string response = null; using (WebClient client = new WebClient()) { client.Headers.Add("accept", "application/json"); client.Headers.Add("accept-encoding", "gzip, deflate"); client.Headers.Add("Content-Type", "application/json"); response = client.UploadString("https://exp.host/--/api/v2/push/send", JsonExtensions.SerializeToJson(body)); } var json = JsonExtensions.DeserializeFromJson <dynamic>(response); return(json); }