public IActionResult Post([FromBody] DocumentBulkSettings settings)
{
var contentLength = contextAccessor.HttpContext.Request.ContentLength;
if (contentLength == null)
{
return(HttpErrorResult(StatusCodes.Status411LengthRequired, GlobalResources.ContentLengthIsNotSet));
}
var results = documentService.Bulk(
dataSetName,
settings.Documents,
contentLength.Value,
parallelService.ParallelLimit);
return(HttpObjectResult(StatusCodes.Status200OK, results));
}
public async Task <ClientResponseWithObject <BulkResults> > BulkDocumentsAsync(DocumentBulkSettings settings)
{
var client = new ApiClient(_configuration, BulkEndpoint);
return(await client.SendAsync <BulkResults>(System.Net.Http.HttpMethod.Post, settings, null, null, Headers, true));
}
private async Task Import <T>(List <JToken> tokens)
{
var cancellationToken = new CancellationTokenSource();
var status = new StatusDialogViewModel {
Title = typeof(T) == typeof(Tag) ? "Importing Tags" : "Importing documents", CancellationTokenSource = cancellationToken
};
await _dialogHandler.Show(new StatusDialog { DataContext = status }, "RootDialog", async (object sender, DialogOpenedEventArgs oa) =>
{
try
{
var all = tokens.Count;
var done = 0;
await Task.Run(() =>
{
try
{
if (typeof(T) == typeof(Tag))
{
var response = new ClientResponseWithObject <BulkResults>();
try
{
var settings = new TagBulkSettings();
settings.Tags = tokens.Select(t => t.ToObject <Tag>()).ToList();
response = new TagManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkTagsAsync(settings).Result;
ResponseValidator.Validate(response, false);
}
catch (Exception ex)
{
Messenger.Default.Send(ex);
status.ErrorCount += tokens.Count;
}
finally
{
var bulkErrors = response.ResponseObject?.Results.Where(br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
if (bulkErrors != null && bulkErrors.Any())
{
foreach (var error in bulkErrors)
{
Messenger.Default.Send(new Exception(string.Format("Id: {0}, error: {1}", error.Id, error.Error)));
status.ErrorCount++;
}
}
status.DoneCount += tokens.Count;
status.ProgressValue = 100;
}
}
else
{
var remaining = tokens.Count;
while ((remaining - GlobalStore.SelectedEndpoint.BulkSize) > 0 && !cancellationToken.IsCancellationRequested)
{
var response = new ClientResponseWithObject <BulkResults>();
try
{
var settings = new DocumentBulkSettings();
settings.Documents = tokens.Take(GlobalStore.SelectedEndpoint.BulkSize).Select(t => t.ToObject <object>()).ToList();
response = new DocumentManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkDocumentsAsync(settings).Result;
ResponseValidator.Validate(response, false);
}
catch (Exception ex)
{
Messenger.Default.Send(new Exception(string.Format("Error during bulk process at range [{0}-{1}]", done, done + remaining), ex));
status.ErrorCount += GlobalStore.SelectedEndpoint.BulkSize;
}
finally
{
var bulkErrors = response.ResponseObject.Results.Where(br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
if (bulkErrors.Any())
{
foreach (var error in bulkErrors)
{
Messenger.Default.Send(new Exception(string.Format("Id: {0}, error: {1}", error.Id, error.Error)));
status.ErrorCount++;
}
}
done += GlobalStore.SelectedEndpoint.BulkSize;
status.DoneCount = done;
status.ProgressValue = (done / (double)all) * 100;
remaining -= GlobalStore.SelectedEndpoint.BulkSize;
tokens = tokens.Skip(GlobalStore.SelectedEndpoint.BulkSize).ToList();
}
}
if (remaining > 0 && !cancellationToken.IsCancellationRequested)
{
var response = new ClientResponseWithObject <BulkResults>();
try
{
var settings = new DocumentBulkSettings();
settings.Documents = tokens.Take(remaining).Select(t => t.ToObject <object>()).ToList();
response = new DocumentManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkDocumentsAsync(settings).Result;
ResponseValidator.Validate(response, false);
}
catch (Exception ex)
{
Messenger.Default.Send(new Exception(string.Format("Error during bulk process at range [{0}-{1}]", done, done + remaining), ex));
status.ErrorCount += remaining;
}
finally
{
var bulkErrors = response.ResponseObject?.Results.Where(br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
if (bulkErrors != null && bulkErrors.Any())
{
foreach (var error in bulkErrors)
{
Messenger.Default.Send(new Exception(string.Format("Id: {0}, error: {1}", error.Id, error.Error)));
status.ErrorCount++;
}
}
done += remaining;
status.DoneCount = done;
status.ProgressValue = (done / (int)all) * 100;
remaining -= remaining;
}
}
}
}
catch (OperationCanceledException)
{
Log.Info(LogMessages.OperationCancelled);
}
});
}
catch (Exception exception)
{
DispatcherHelper.CheckBeginInvokeOnUI(() => Messenger.Default.Send(exception));
}
finally
{
status.OperationIsFinished = true;
}
});
}
private async void ImportCsv <T>()
{
var ofd = new OpenFileDialog
{
Multiselect = false,
Filter = "CSV|*.csv"
};
if (ofd.ShowDialog() == true)
{
var importSettings = new CsvImportSettings {
Delimiter = ",", Force = true
};
var dialogResult = await _dialogHandler.Show(
new CommonDialog
{
DataContext =
new CommonDialogViewModel
{
Header = "Csv Import Settings",
Content = importSettings,
Buttons = ButtonsEnum.OkCancel
}
}, "RootDialog");
if ((CommonDialogResult)dialogResult == CommonDialogResult.Cancel)
{
return;
}
var invalidRows = new List <int>();
var errors = new ConcurrentBag <string>();
var cancellationToken = new CancellationTokenSource();
var status = new StatusDialogViewModel {
ProgressValue = 0, Title = typeof(T) == typeof(Tag) ? "Importing Tags" : "Importing documents", CancellationTokenSource = cancellationToken
};
await _dialogHandler.Show(new StatusDialog { DataContext = status }, "RootDialog",
async (object sender, DialogOpenedEventArgs oa) =>
{
FileStream stream = null;
try
{
stream = new FileStream(ofd.FileName, FileMode.Open);
stream.Position = 0;
var importResult = new CsvImportResult();
importSettings.CsvReader = CsvProcesser.GetCsvReader(stream, importSettings);
while (importResult.CanContinue && !cancellationToken.IsCancellationRequested)
{
importResult = await CsvProcesser.GetTokens(importSettings, typeof(T) == typeof(Tag));
invalidRows.AddRange(importResult.InvalidRows);
status.ErrorCount += importResult.InvalidRows.Count;
if (typeof(T) == typeof(Tag))
{
var response = new ClientResponseWithObject <BulkResults>();
try
{
var settings = new TagBulkSettings();
settings.Tags = importResult.Tokens.Select(t => t.ToObject <Tag>()).ToList();
response = await new TagManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkTagsAsync(settings);
ResponseValidator.Validate(response, false);
}
catch (Exception ex)
{
errors.Add(string.Format("Error during bulk process:{0}{1}: {2}", Environment.NewLine, ex.Message, string.Join(", ", response.Errors)));
status.ErrorCount += importResult.Tokens.Count;
}
finally
{
if (response.IsSuccessful)
{
var bulkErrors =
response.ResponseObject.Results.Where(
br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
if (bulkErrors.Any())
{
foreach (var error in bulkErrors)
{
errors.Add(string.Format("Id: {0}, error: {1}", error.Id,
error.Error));
status.ErrorCount++;
}
}
}
status.DoneCount += (importResult.Tokens.Count + importResult.InvalidRows.Count);
status.ProgressValue = (stream.Position / (double)stream.Length) * 100;
importResult.Tokens = null;
importResult.InvalidRows = null;
}
}
else
{
var response = new ClientResponseWithObject <BulkResults>();
try
{
var settings = new DocumentBulkSettings();
var idFieldIsArray = false;
if (SelectedDataSet.SampleDocument != null)
{
var docObject = JObject.FromObject(SelectedDataSet.SampleDocument);
idFieldIsArray = docObject?[SelectedDataSet.TagField] is JArray;
}
else
{
var docObject = JObject.FromObject(SelectedDataSet.Schema);
idFieldIsArray = docObject?["properties"][SelectedDataSet.TagField]["type"].ToString().ToLower() == "array";
}
if (idFieldIsArray)
{
settings.Documents = importResult.Tokens.Select(t =>
{
t[SelectedDataSet.TagField] = new JArray(t[SelectedDataSet.TagField]);
return(t);
}).Select(t => t.ToObject <object>()).ToList();
}
else
{
settings.Documents = importResult.Tokens.Select(t => t.ToObject <object>()).ToList();
}
response = await new DocumentManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkDocumentsAsync(settings);
ResponseValidator.Validate(response, false);
}
catch (Exception ex)
{
Messenger.Default.Send(ex);
status.ErrorCount += GlobalStore.SelectedEndpoint.BulkSize;
}
finally
{
if (response.IsSuccessful)
{
var bulkErrors =
response.ResponseObject.Results.Where(
br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
if (bulkErrors.Any())
{
foreach (var error in bulkErrors)
{
errors.Add(string.Format("Id: {0}, error: {1}", error.Id,
error.Error));
status.ErrorCount++;
}
}
}
status.DoneCount += (importResult.Tokens.Count + importResult.InvalidRows.Count);
status.ProgressValue = (stream.Position / (double)stream.Length) * 100;
importResult.Tokens = null;
importResult.InvalidRows = null;
}
}
}
}
catch (Exception exception)
{
DispatcherHelper.CheckBeginInvokeOnUI(() => Messenger.Default.Send(exception));
}
finally
{
stream?.Close();
status.OperationIsFinished = true;
}
});
if (invalidRows.Any())
{
Messenger.Default.Send(new Exception($"Invalid rows:{Environment.NewLine}{string.Join(Environment.NewLine, invalidRows)}"));
}
if (errors.Any())
{
Messenger.Default.Send(new Exception($"Errors:{Environment.NewLine}{string.Join(Environment.NewLine, errors)}"));
}
}
}