/// <summary> /// Constructor for objects. /// </summary> public AtomEntryWriter(IObjectData objectData, CmisVersion cmisVersion, IContentStream contentStream) { if (objectData == null || objectData.Properties == null) { throw new CmisInvalidArgumentException("Object and properties must not be null!"); } if (contentStream != null && contentStream.MimeType == null) { throw new CmisInvalidArgumentException("Media type must be set if a stream is present!"); } this.objectData = objectData; this.cmisVersion = cmisVersion; this.contentStream = contentStream; if (contentStream != null && contentStream.Stream != null) { // do we need buffering here? stream = contentStream.Stream; } else { stream = null; } this.typeDef = null; this.bulkUpdate = null; }
// Registers a bulk update public void RemoveBulkUpdate(BulkUpdate bulkUpdate) { bulkUpdates.Remove(bulkUpdate); if (bulkUpdates.Count == 0) { TryToSave(); } }
// Registers a bulk update public void RegisterBulkUpdate(BulkUpdate bulkUpdate) { if (bulkUpdates.Contains(bulkUpdate)) { return; } bulkUpdates.Add(bulkUpdate); }
public void Should_serialize_false() { var source = new BulkUpdate { DoNotScript = false }; var serialized = source.ToJson(); Assert.Contains("\"doNotScript\": false", serialized); }
private void SaveButton_Click(object sender, EventArgs e) { Program app = Program.GetInstance(); ChairService chairManager = app.GetService <ChairService>("chairs"); RoomService roomManager = app.GetService <RoomService>("rooms"); // Create bulk update BulkUpdate bulkUpdate = new BulkUpdate(); bulkUpdate.Begin(); // Save room Room room = new Room((int)numberInput.Value); if (!roomManager.SaveRoom(room)) { GuiHelper.ShowError(ValidationHelper.GetErrorList(room)); return; } // Disable save button saveButton.Enabled = false; // Create chairs int rows = (int)rowInput.Value; int columns = (int)columnInput.Value; double price = (double)priceInput.Value; for (int i = 1; i <= rows; i++) { for (int j = 1; j <= columns; j++) { Chair chair = new Chair(room.id, i, j, price, "default"); if (!chairManager.SaveChair(chair)) { GuiHelper.ShowError(ValidationHelper.GetErrorList(room)); } } } // End bulk update bulkUpdate.End(); // Enable save button saveButton.Enabled = true; // Redirect to screen RoomDetail roomDetail = app.GetScreen <RoomDetail>("roomDetail"); roomDetail.SetRoom(room); app.ShowScreen(roomDetail); GuiHelper.ShowInfo("Zaal succesvol aangemaakt"); }
/// <summary> /// Constructor for bulk updates. /// </summary> public AtomEntryWriter(BulkUpdate bulkUpdate) { if (bulkUpdate == null) { throw new CmisInvalidArgumentException("Bulk update data must not be null!"); } this.bulkUpdate = bulkUpdate; this.typeDef = null; this.cmisVersion = CmisVersion.Cmis_1_1; this.objectData = null; this.contentStream = null; this.stream = null; }
/// <summary> /// Constructor for types. /// </summary> public AtomEntryWriter(ITypeDefinition type, CmisVersion cmisVersion) { if (type == null) { throw new CmisInvalidArgumentException("Type must not be null!"); } this.typeDef = type; this.cmisVersion = cmisVersion; this.objectData = null; this.contentStream = null; this.stream = null; this.bulkUpdate = null; }
public async Task UpsertAsync(ISession session, ILogger log, ReferenceCache cache) { var client = session.Contents(SchemaName); var request = new BulkUpdate { Jobs = Contents.Select(x => new BulkUpdateJob { Query = new { filter = x.Filter }, Data = x.Data }).ToList() }; var results = await client.BulkUpdateAsync(request); var i = 0; foreach (var result in results) { var content = Contents[i]; log.StepStart($"{content.File} / {content.Ref}"); if (result.ContentId != null) { log.StepSuccess(); } else if (result.Error != null) { log.StepFailed(result.Error.ToString()); } else { log.StepSkipped("Unknown Reason"); } i++; } }
public JobStatusResponse BulkUpdate(IEnumerable <long> ids, BulkUpdate info) { return(GenericPut <JobStatusResponse>($"{_tickets}/update_many.json?ids={ids.ToCsv()}", new { ticket = info })); }
public async Task ImportAsync(ISyncService sync, SyncOptions options, ISession session) { var models = GetFiles(sync.FileSystem) .Select(x => (x, sync.Read <ContentsModel>(x, log))); var schemas = await session.Schemas.GetSchemasAsync(session.App); var schemaMap = schemas.Items.ToDictionary(x => x.Name, x => x.Id); foreach (var(file, model) in models) { if (model?.Contents?.Count > 0) { model.Clear(options.Languages); foreach (var content in model.Contents) { content.MapComponents(schemaMap); } var client = session.Contents(model.Contents[0].Schema); var request = new BulkUpdate { OptimizeValidation = true, DoNotScript = true, DoNotValidate = false, DoNotValidateWorkflow = true, Jobs = model.Contents.Select(x => x.ToUpsert(schemas)).ToList() }; var contentIdAssigned = false; var contentIndex = 0; var results = await client.BulkUpdateAsync(request); foreach (var content in model.Contents) { var result = results.Find(x => x.JobIndex == contentIndex); log.StepStart($"Upserting #{contentIndex}"); if (result?.Error != null) { log.StepFailed(result.Error.ToString()); } else if (result?.ContentId != null) { if (string.IsNullOrWhiteSpace(content.Id)) { content.Id = result.ContentId; contentIdAssigned = true; } log.StepSuccess(); } else { log.StepSkipped("Unknown Reason"); } contentIndex++; } if (contentIdAssigned) { await log.DoSafeAsync($"Saving {file.Name}", async() => { await sync.WriteWithSchema(file, model, Ref); }); } } } }
public static async Task ImportAsync(this ISession session, IImportSettings setting, ILogger log, IEnumerable <DynamicData> datas) { var contents = session.Contents(setting.Schema); var totalWritten = 0; using (var logLine = log.WriteSameLine()) { var keyField = setting.KeyField; var update = new BulkUpdate { Jobs = new List <BulkUpdateJob>(), DoNotScript = false, DoNotValidate = false, Publish = !setting.Unpublished }; const string op = "eq"; foreach (var batch in datas.Batch(50)) { update.Jobs.Clear(); foreach (var data in batch) { var job = new BulkUpdateJob { Data = data, }; if (!string.IsNullOrWhiteSpace(keyField)) { if (!data.TryGetValue(keyField, out var temp) || temp is not JObject obj || !obj.TryGetValue("iv", StringComparison.Ordinal, out var value)) { throw new InvalidOperationException($"Cannot find key '{keyField}' in data."); } job.Query = new { filter = new { path = $"data.{keyField}.iv", op, value, } }; job.Type = BulkUpdateType.Upsert; } else { job.Type = BulkUpdateType.Create; } update.Jobs.Add(job); } var result = await contents.BulkUpdateAsync(update); var error = result.Find(x => x.Error != null)?.Error; if (error != null) { throw new SquidexManagementException <ErrorDto>(error.Message, error.StatusCode, null, null, error, null); } totalWritten += update.Jobs.Count; logLine.WriteLine("> Imported: {0}.", totalWritten); } } log.WriteLine("> Imported: {0}. Completed.", totalWritten); }
public async Task ImportAsync(Stream stream) { var deserializeBlock = new TransformBlock <CsvRecord, AuthorRecord>(x => { var data = new AuthorData(); var json = JObject.Parse(x.Json); if (json.TryGetValue("name", out var name)) { data.Name = GetString(name); } if (json.TryGetValue("birth_date", out var birthdate)) { data.Birthdate = GetString(birthdate); } if (json.TryGetValue("bio", out var bio)) { data.Bio = GetString(bio); } if (json.TryGetValue("personal_name", out var personalName)) { data.PersonalName = GetString(personalName); } if (json.TryGetValue("wikipedia", out var wikipedia)) { data.Wikipedia = GetString(wikipedia); } return(new AuthorRecord(x.Id, data)); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 1, BoundedCapacity = 100 }); var batchBlock = new BatchBlock <AuthorRecord>(100, new GroupingDataflowBlockOptions { BoundedCapacity = 100 }); var totalFailed = 0; var totalSuccess = 0; Console.Write("Importing (success/failed)..."); var y = Console.CursorTop; var x = Console.CursorLeft; var lockObject = new object(); var importBlock = new ActionBlock <AuthorRecord[]>(async authors => { var request = new BulkUpdate { OptimizeValidation = true, DoNotScript = true, DoNotValidate = false, DoNotValidateWorkflow = true, Jobs = authors.Select(x => { return(new BulkUpdateJob { Id = x.Id, Data = x.Author, Type = BulkUpdateType.Upsert }); }).ToList() }; var response = await client.BulkUpdateAsync(request); lock (lockObject) { totalFailed += response.Count(x => x.Error != null); totalSuccess += response.Count(x => x.Error == null); Console.SetCursorPosition(x, y); Console.Write("{0}/{1}", totalSuccess, totalFailed); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 1, BoundedCapacity = Environment.ProcessorCount * 2 }); deserializeBlock.LinkTo(batchBlock, new DataflowLinkOptions { PropagateCompletion = true }); batchBlock.LinkTo(importBlock, new DataflowLinkOptions { PropagateCompletion = true }); using (var streamReader = new StreamReader(stream)) { var configuration = new CsvConfiguration(CultureInfo.InvariantCulture) { BadDataFound = null, Delimiter = "\t" }; using (var csvReader = new CsvReader(streamReader, configuration)) { while (await csvReader.ReadAsync()) { var record = new CsvRecord( csvReader.GetField(1), csvReader.GetField(4)); await deserializeBlock.SendAsync(record); } } } deserializeBlock.Complete(); await importBlock.Completion; Console.WriteLine(); }
public JobStatusResponse BulkUpdate(List <long> ids, BulkUpdate info) { var body = new { ticket = info }; return(GenericPut <JobStatusResponse>(string.Format("{0}/update_many.json?ids={1}", _tickets, ids.ToCsv()), body)); }
public async Task <JobStatusResponse> BulkUpdateAsync(IEnumerable <long> ids, BulkUpdate info) { return(await GenericPutAsync <JobStatusResponse>($"{_tickets}/update_many.json?ids={ids.ToCsv()}", new { ticket = info })); }
public async void Update() { if (_isUpdating) { return; } lock (_sync) { if (_isUpdating) { return; } _isUpdating = true; } BulkUpdate bulkUpdate = new BulkUpdate(); try { ulong lastUpdatedSyncHeight = 0; AsyncServerStreamingCall <SyncBlockDescriptor> serverStreamingCall = _syncManagerClient.GetDeltaSyncBlocks(new ByHeightRequest { Height = _lastUpdatedSyncHeight }); while (await serverStreamingCall.ResponseStream.MoveNext()) { SyncBlockDescriptor syncBlockDescriptor = serverStreamingCall.ResponseStream.Current; if (syncBlockDescriptor.Height > _lastUpdatedSyncHeight) { bulkUpdate.SyncBlockInfos.Add(new Models.SyncBlockInfo { SyncBlockHeight = syncBlockDescriptor.Height }); if (lastUpdatedSyncHeight < syncBlockDescriptor.Height) { lastUpdatedSyncHeight = syncBlockDescriptor.Height; } } } if (_lastUpdatedSyncHeight < lastUpdatedSyncHeight) { _lastUpdatedSyncHeight = lastUpdatedSyncHeight; } ulong lastUpdatedCombinedBlockHeight = 0; AsyncServerStreamingCall <CombinedRegistryBlockInfo> serverStreamingCall2 = _syncManagerClient.GetCombinedRegistryBlocksInfoSinceHeight(new ByHeightRequest { Height = _lastUpdatedCombinedBlockHeight }); while (await serverStreamingCall2.ResponseStream.MoveNext()) { CombinedRegistryBlockInfo combinedRegistryBlockInfo = serverStreamingCall2.ResponseStream.Current; if (combinedRegistryBlockInfo.Height > _lastUpdatedCombinedBlockHeight) { bulkUpdate.CombinedBlockInfos.Add( new CombinedBlockInfo { SyncBlockHeight = combinedRegistryBlockInfo.SyncBlockHeight, BlockHeight = combinedRegistryBlockInfo.Height, CombinedRegistryBlocksCount = combinedRegistryBlockInfo.CombinedRegistryBlocksCount, RegistryFullBlockInfos = combinedRegistryBlockInfo.BlockDescriptors.Select( b => new RegistryFullBlockInfo { SyncBlockHeight = b.SyncBlockHeight, Round = b.Round, TransactionsCount = b.TransactionsCount }).ToList() }); if (lastUpdatedCombinedBlockHeight < combinedRegistryBlockInfo.Height) { lastUpdatedCombinedBlockHeight = combinedRegistryBlockInfo.Height; } } } if (_lastUpdatedCombinedBlockHeight < lastUpdatedCombinedBlockHeight) { _lastUpdatedCombinedBlockHeight = lastUpdatedCombinedBlockHeight; } } finally { _isUpdating = false; } foreach (IObserver <BulkUpdate> observer in _observers) { observer.OnNext(bulkUpdate); } }
public async Task <JobStatusResponse> BulkUpdateAsync(IEnumerable <long> ids, BulkUpdate info) { var body = new { ticket = info }; return(await GenericPutAsync <JobStatusResponse>(string.Format("{0}/update_many.json?ids={1}", _tickets, ids.ToCsv()), body)); }
public async Task ImportAsync(DirectoryInfo directoryInfo, JsonHelper jsonHelper, SyncOptions options, ISession session) { var models = GetFiles(directoryInfo) .Select(x => (x, jsonHelper.Read <ContentsModel>(x, log))); foreach (var(file, model) in models) { if (model?.Contents?.Count > 0) { model.Clear(options.Languages); var client = session.Contents(model.Contents.First().Schema); var request = new BulkUpdate { OptimizeValidation = true, DoNotScript = true, DoNotValidate = false, DoNotValidateWorkflow = true, Jobs = model.Contents.Select(x => x.ToJob()).ToList() }; var contentIdAssigned = false; var contentIndex = 0; var results = await client.BulkUpdateAsync(request); foreach (var content in model.Contents) { var result = results.FirstOrDefault(x => x.JobIndex == contentIndex); log.StepStart($"Upserting #{contentIndex}"); if (result?.Error != null) { log.StepFailed(result.Error.ToString()); } else if (result?.ContentId != null) { if (string.IsNullOrWhiteSpace(content.Id)) { content.Id = result.ContentId; contentIdAssigned = true; } log.StepSuccess(); } else { log.StepSkipped("Unknown Reason"); } contentIndex++; } if (contentIdAssigned) { await log.DoSafeAsync($"Saving {file.Name}", async() => { await jsonHelper.WriteWithSchema(directoryInfo, file.FullName, model, Ref); }); } } } }