/// <summary>
/// Constructor for objects.
/// </summary>
public AtomEntryWriter(IObjectData objectData, CmisVersion cmisVersion, IContentStream contentStream)
{
if (objectData == null || objectData.Properties == null)
{
throw new CmisInvalidArgumentException("Object and properties must not be null!");
}
if (contentStream != null && contentStream.MimeType == null)
{
throw new CmisInvalidArgumentException("Media type must be set if a stream is present!");
}
this.objectData = objectData;
this.cmisVersion = cmisVersion;
this.contentStream = contentStream;
if (contentStream != null && contentStream.Stream != null)
{
// do we need buffering here?
stream = contentStream.Stream;
}
else
{
stream = null;
}
this.typeDef = null;
this.bulkUpdate = null;
}
// Registers a bulk update
public void RemoveBulkUpdate(BulkUpdate bulkUpdate)
{
bulkUpdates.Remove(bulkUpdate);
if (bulkUpdates.Count == 0)
{
TryToSave();
}
}
// Registers a bulk update
public void RegisterBulkUpdate(BulkUpdate bulkUpdate)
{
if (bulkUpdates.Contains(bulkUpdate))
{
return;
}
bulkUpdates.Add(bulkUpdate);
}
public void Should_serialize_false()
{
var source = new BulkUpdate
{
DoNotScript = false
};
var serialized = source.ToJson();
Assert.Contains("\"doNotScript\": false", serialized);
}
private void SaveButton_Click(object sender, EventArgs e)
{
Program app = Program.GetInstance();
ChairService chairManager = app.GetService <ChairService>("chairs");
RoomService roomManager = app.GetService <RoomService>("rooms");
// Create bulk update
BulkUpdate bulkUpdate = new BulkUpdate();
bulkUpdate.Begin();
// Save room
Room room = new Room((int)numberInput.Value);
if (!roomManager.SaveRoom(room))
{
GuiHelper.ShowError(ValidationHelper.GetErrorList(room));
return;
}
// Disable save button
saveButton.Enabled = false;
// Create chairs
int rows = (int)rowInput.Value;
int columns = (int)columnInput.Value;
double price = (double)priceInput.Value;
for (int i = 1; i <= rows; i++)
{
for (int j = 1; j <= columns; j++)
{
Chair chair = new Chair(room.id, i, j, price, "default");
if (!chairManager.SaveChair(chair))
{
GuiHelper.ShowError(ValidationHelper.GetErrorList(room));
}
}
}
// End bulk update
bulkUpdate.End();
// Enable save button
saveButton.Enabled = true;
// Redirect to screen
RoomDetail roomDetail = app.GetScreen <RoomDetail>("roomDetail");
roomDetail.SetRoom(room);
app.ShowScreen(roomDetail);
GuiHelper.ShowInfo("Zaal succesvol aangemaakt");
}
/// <summary>
/// Constructor for bulk updates.
/// </summary>
public AtomEntryWriter(BulkUpdate bulkUpdate)
{
if (bulkUpdate == null)
{
throw new CmisInvalidArgumentException("Bulk update data must not be null!");
}
this.bulkUpdate = bulkUpdate;
this.typeDef = null;
this.cmisVersion = CmisVersion.Cmis_1_1;
this.objectData = null;
this.contentStream = null;
this.stream = null;
}
/// <summary>
/// Constructor for types.
/// </summary>
public AtomEntryWriter(ITypeDefinition type, CmisVersion cmisVersion)
{
if (type == null)
{
throw new CmisInvalidArgumentException("Type must not be null!");
}
this.typeDef = type;
this.cmisVersion = cmisVersion;
this.objectData = null;
this.contentStream = null;
this.stream = null;
this.bulkUpdate = null;
}
public async Task UpsertAsync(ISession session, ILogger log, ReferenceCache cache)
{
var client = session.Contents(SchemaName);
var request = new BulkUpdate
{
Jobs = Contents.Select(x => new BulkUpdateJob {
Query = new { filter = x.Filter }, Data = x.Data
}).ToList()
};
var results = await client.BulkUpdateAsync(request);
var i = 0;
foreach (var result in results)
{
var content = Contents[i];
log.StepStart($"{content.File} / {content.Ref}");
if (result.ContentId != null)
{
log.StepSuccess();
}
else if (result.Error != null)
{
log.StepFailed(result.Error.ToString());
}
else
{
log.StepSkipped("Unknown Reason");
}
i++;
}
}
public JobStatusResponse BulkUpdate(IEnumerable <long> ids, BulkUpdate info)
{
return(GenericPut <JobStatusResponse>($"{_tickets}/update_many.json?ids={ids.ToCsv()}", new { ticket = info }));
}
public async Task ImportAsync(ISyncService sync, SyncOptions options, ISession session)
{
var models =
GetFiles(sync.FileSystem)
.Select(x => (x, sync.Read <ContentsModel>(x, log)));
var schemas = await session.Schemas.GetSchemasAsync(session.App);
var schemaMap = schemas.Items.ToDictionary(x => x.Name, x => x.Id);
foreach (var(file, model) in models)
{
if (model?.Contents?.Count > 0)
{
model.Clear(options.Languages);
foreach (var content in model.Contents)
{
content.MapComponents(schemaMap);
}
var client = session.Contents(model.Contents[0].Schema);
var request = new BulkUpdate
{
OptimizeValidation = true,
DoNotScript = true,
DoNotValidate = false,
DoNotValidateWorkflow = true,
Jobs = model.Contents.Select(x => x.ToUpsert(schemas)).ToList()
};
var contentIdAssigned = false;
var contentIndex = 0;
var results = await client.BulkUpdateAsync(request);
foreach (var content in model.Contents)
{
var result = results.Find(x => x.JobIndex == contentIndex);
log.StepStart($"Upserting #{contentIndex}");
if (result?.Error != null)
{
log.StepFailed(result.Error.ToString());
}
else if (result?.ContentId != null)
{
if (string.IsNullOrWhiteSpace(content.Id))
{
content.Id = result.ContentId;
contentIdAssigned = true;
}
log.StepSuccess();
}
else
{
log.StepSkipped("Unknown Reason");
}
contentIndex++;
}
if (contentIdAssigned)
{
await log.DoSafeAsync($"Saving {file.Name}", async() =>
{
await sync.WriteWithSchema(file, model, Ref);
});
}
}
}
}
public static async Task ImportAsync(this ISession session, IImportSettings setting, ILogger log, IEnumerable <DynamicData> datas)
{
var contents = session.Contents(setting.Schema);
var totalWritten = 0;
using (var logLine = log.WriteSameLine())
{
var keyField = setting.KeyField;
var update = new BulkUpdate
{
Jobs = new List <BulkUpdateJob>(),
DoNotScript = false,
DoNotValidate = false,
Publish = !setting.Unpublished
};
const string op = "eq";
foreach (var batch in datas.Batch(50))
{
update.Jobs.Clear();
foreach (var data in batch)
{
var job = new BulkUpdateJob
{
Data = data,
};
if (!string.IsNullOrWhiteSpace(keyField))
{
if (!data.TryGetValue(keyField, out var temp) || temp is not JObject obj || !obj.TryGetValue("iv", StringComparison.Ordinal, out var value))
{
throw new InvalidOperationException($"Cannot find key '{keyField}' in data.");
}
job.Query = new
{
filter = new
{
path = $"data.{keyField}.iv",
op,
value,
}
};
job.Type = BulkUpdateType.Upsert;
}
else
{
job.Type = BulkUpdateType.Create;
}
update.Jobs.Add(job);
}
var result = await contents.BulkUpdateAsync(update);
var error = result.Find(x => x.Error != null)?.Error;
if (error != null)
{
throw new SquidexManagementException <ErrorDto>(error.Message, error.StatusCode, null, null, error, null);
}
totalWritten += update.Jobs.Count;
logLine.WriteLine("> Imported: {0}.", totalWritten);
}
}
log.WriteLine("> Imported: {0}. Completed.", totalWritten);
}
public async Task ImportAsync(Stream stream)
{
var deserializeBlock = new TransformBlock <CsvRecord, AuthorRecord>(x =>
{
var data = new AuthorData();
var json = JObject.Parse(x.Json);
if (json.TryGetValue("name", out var name))
{
data.Name = GetString(name);
}
if (json.TryGetValue("birth_date", out var birthdate))
{
data.Birthdate = GetString(birthdate);
}
if (json.TryGetValue("bio", out var bio))
{
data.Bio = GetString(bio);
}
if (json.TryGetValue("personal_name", out var personalName))
{
data.PersonalName = GetString(personalName);
}
if (json.TryGetValue("wikipedia", out var wikipedia))
{
data.Wikipedia = GetString(wikipedia);
}
return(new AuthorRecord(x.Id, data));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
MaxMessagesPerTask = 1,
BoundedCapacity = 100
});
var batchBlock = new BatchBlock <AuthorRecord>(100, new GroupingDataflowBlockOptions
{
BoundedCapacity = 100
});
var totalFailed = 0;
var totalSuccess = 0;
Console.Write("Importing (success/failed)...");
var y = Console.CursorTop;
var x = Console.CursorLeft;
var lockObject = new object();
var importBlock = new ActionBlock <AuthorRecord[]>(async authors =>
{
var request = new BulkUpdate
{
OptimizeValidation = true,
DoNotScript = true,
DoNotValidate = false,
DoNotValidateWorkflow = true,
Jobs = authors.Select(x =>
{
return(new BulkUpdateJob
{
Id = x.Id,
Data = x.Author,
Type = BulkUpdateType.Upsert
});
}).ToList()
};
var response = await client.BulkUpdateAsync(request);
lock (lockObject)
{
totalFailed += response.Count(x => x.Error != null);
totalSuccess += response.Count(x => x.Error == null);
Console.SetCursorPosition(x, y);
Console.Write("{0}/{1}", totalSuccess, totalFailed);
}
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
MaxMessagesPerTask = 1,
BoundedCapacity = Environment.ProcessorCount * 2
});
deserializeBlock.LinkTo(batchBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
batchBlock.LinkTo(importBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
using (var streamReader = new StreamReader(stream))
{
var configuration = new CsvConfiguration(CultureInfo.InvariantCulture)
{
BadDataFound = null,
Delimiter = "\t"
};
using (var csvReader = new CsvReader(streamReader, configuration))
{
while (await csvReader.ReadAsync())
{
var record = new CsvRecord(
csvReader.GetField(1),
csvReader.GetField(4));
await deserializeBlock.SendAsync(record);
}
}
}
deserializeBlock.Complete();
await importBlock.Completion;
Console.WriteLine();
}
public JobStatusResponse BulkUpdate(List <long> ids, BulkUpdate info)
{
var body = new { ticket = info };
return(GenericPut <JobStatusResponse>(string.Format("{0}/update_many.json?ids={1}", _tickets, ids.ToCsv()), body));
}
public async Task <JobStatusResponse> BulkUpdateAsync(IEnumerable <long> ids, BulkUpdate info)
{
return(await GenericPutAsync <JobStatusResponse>($"{_tickets}/update_many.json?ids={ids.ToCsv()}", new { ticket = info }));
}
public async void Update()
{
if (_isUpdating)
{
return;
}
lock (_sync)
{
if (_isUpdating)
{
return;
}
_isUpdating = true;
}
BulkUpdate bulkUpdate = new BulkUpdate();
try
{
ulong lastUpdatedSyncHeight = 0;
AsyncServerStreamingCall <SyncBlockDescriptor> serverStreamingCall = _syncManagerClient.GetDeltaSyncBlocks(new ByHeightRequest {
Height = _lastUpdatedSyncHeight
});
while (await serverStreamingCall.ResponseStream.MoveNext())
{
SyncBlockDescriptor syncBlockDescriptor = serverStreamingCall.ResponseStream.Current;
if (syncBlockDescriptor.Height > _lastUpdatedSyncHeight)
{
bulkUpdate.SyncBlockInfos.Add(new Models.SyncBlockInfo {
SyncBlockHeight = syncBlockDescriptor.Height
});
if (lastUpdatedSyncHeight < syncBlockDescriptor.Height)
{
lastUpdatedSyncHeight = syncBlockDescriptor.Height;
}
}
}
if (_lastUpdatedSyncHeight < lastUpdatedSyncHeight)
{
_lastUpdatedSyncHeight = lastUpdatedSyncHeight;
}
ulong lastUpdatedCombinedBlockHeight = 0;
AsyncServerStreamingCall <CombinedRegistryBlockInfo> serverStreamingCall2 = _syncManagerClient.GetCombinedRegistryBlocksInfoSinceHeight(new ByHeightRequest {
Height = _lastUpdatedCombinedBlockHeight
});
while (await serverStreamingCall2.ResponseStream.MoveNext())
{
CombinedRegistryBlockInfo combinedRegistryBlockInfo = serverStreamingCall2.ResponseStream.Current;
if (combinedRegistryBlockInfo.Height > _lastUpdatedCombinedBlockHeight)
{
bulkUpdate.CombinedBlockInfos.Add(
new CombinedBlockInfo
{
SyncBlockHeight = combinedRegistryBlockInfo.SyncBlockHeight,
BlockHeight = combinedRegistryBlockInfo.Height,
CombinedRegistryBlocksCount = combinedRegistryBlockInfo.CombinedRegistryBlocksCount,
RegistryFullBlockInfos = combinedRegistryBlockInfo.BlockDescriptors.Select(
b => new RegistryFullBlockInfo
{
SyncBlockHeight = b.SyncBlockHeight,
Round = b.Round,
TransactionsCount = b.TransactionsCount
}).ToList()
});
if (lastUpdatedCombinedBlockHeight < combinedRegistryBlockInfo.Height)
{
lastUpdatedCombinedBlockHeight = combinedRegistryBlockInfo.Height;
}
}
}
if (_lastUpdatedCombinedBlockHeight < lastUpdatedCombinedBlockHeight)
{
_lastUpdatedCombinedBlockHeight = lastUpdatedCombinedBlockHeight;
}
}
finally
{
_isUpdating = false;
}
foreach (IObserver <BulkUpdate> observer in _observers)
{
observer.OnNext(bulkUpdate);
}
}
public async Task <JobStatusResponse> BulkUpdateAsync(IEnumerable <long> ids, BulkUpdate info)
{
var body = new { ticket = info };
return(await GenericPutAsync <JobStatusResponse>(string.Format("{0}/update_many.json?ids={1}", _tickets, ids.ToCsv()), body));
}
public async Task ImportAsync(DirectoryInfo directoryInfo, JsonHelper jsonHelper, SyncOptions options, ISession session)
{
var models =
GetFiles(directoryInfo)
.Select(x => (x, jsonHelper.Read <ContentsModel>(x, log)));
foreach (var(file, model) in models)
{
if (model?.Contents?.Count > 0)
{
model.Clear(options.Languages);
var client = session.Contents(model.Contents.First().Schema);
var request = new BulkUpdate
{
OptimizeValidation = true,
DoNotScript = true,
DoNotValidate = false,
DoNotValidateWorkflow = true,
Jobs = model.Contents.Select(x => x.ToJob()).ToList()
};
var contentIdAssigned = false;
var contentIndex = 0;
var results = await client.BulkUpdateAsync(request);
foreach (var content in model.Contents)
{
var result = results.FirstOrDefault(x => x.JobIndex == contentIndex);
log.StepStart($"Upserting #{contentIndex}");
if (result?.Error != null)
{
log.StepFailed(result.Error.ToString());
}
else if (result?.ContentId != null)
{
if (string.IsNullOrWhiteSpace(content.Id))
{
content.Id = result.ContentId;
contentIdAssigned = true;
}
log.StepSuccess();
}
else
{
log.StepSkipped("Unknown Reason");
}
contentIndex++;
}
if (contentIdAssigned)
{
await log.DoSafeAsync($"Saving {file.Name}", async() =>
{
await jsonHelper.WriteWithSchema(directoryInfo, file.FullName, model, Ref);
});
}
}
}
}
