Ejemplo n.º 1
0
        public IActionResult Post([FromBody] DocumentBulkSettings settings)
        {
            var contentLength = contextAccessor.HttpContext.Request.ContentLength;

            if (contentLength == null)
            {
                return(HttpErrorResult(StatusCodes.Status411LengthRequired, GlobalResources.ContentLengthIsNotSet));
            }

            var results = documentService.Bulk(
                dataSetName,
                settings.Documents,
                contentLength.Value,
                parallelService.ParallelLimit);

            return(HttpObjectResult(StatusCodes.Status200OK, results));
        }
Ejemplo n.º 2
0
        public async Task <ClientResponseWithObject <BulkResults> > BulkDocumentsAsync(DocumentBulkSettings settings)
        {
            var client = new ApiClient(_configuration, BulkEndpoint);

            return(await client.SendAsync <BulkResults>(System.Net.Http.HttpMethod.Post, settings, null, null, Headers, true));
        }
 private async Task Import <T>(List <JToken> tokens)
 {
     var cancellationToken = new CancellationTokenSource();
     var status            = new StatusDialogViewModel {
         Title = typeof(T) == typeof(Tag) ? "Importing Tags" : "Importing documents", CancellationTokenSource = cancellationToken
     };
     await _dialogHandler.Show(new StatusDialog { DataContext = status }, "RootDialog", async (object sender, DialogOpenedEventArgs oa) =>
     {
         try
         {
             var all  = tokens.Count;
             var done = 0;
             await Task.Run(() =>
             {
                 try
                 {
                     if (typeof(T) == typeof(Tag))
                     {
                         var response = new ClientResponseWithObject <BulkResults>();
                         try
                         {
                             var settings  = new TagBulkSettings();
                             settings.Tags = tokens.Select(t => t.ToObject <Tag>()).ToList();
                             response      = new TagManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkTagsAsync(settings).Result;
                             ResponseValidator.Validate(response, false);
                         }
                         catch (Exception ex)
                         {
                             Messenger.Default.Send(ex);
                             status.ErrorCount += tokens.Count;
                         }
                         finally
                         {
                             var bulkErrors = response.ResponseObject?.Results.Where(br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
                             if (bulkErrors != null && bulkErrors.Any())
                             {
                                 foreach (var error in bulkErrors)
                                 {
                                     Messenger.Default.Send(new Exception(string.Format("Id: {0}, error: {1}", error.Id, error.Error)));
                                     status.ErrorCount++;
                                 }
                             }
                             status.DoneCount    += tokens.Count;
                             status.ProgressValue = 100;
                         }
                     }
                     else
                     {
                         var remaining = tokens.Count;
                         while ((remaining - GlobalStore.SelectedEndpoint.BulkSize) > 0 && !cancellationToken.IsCancellationRequested)
                         {
                             var response = new ClientResponseWithObject <BulkResults>();
                             try
                             {
                                 var settings       = new DocumentBulkSettings();
                                 settings.Documents = tokens.Take(GlobalStore.SelectedEndpoint.BulkSize).Select(t => t.ToObject <object>()).ToList();
                                 response           = new DocumentManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkDocumentsAsync(settings).Result;
                                 ResponseValidator.Validate(response, false);
                             }
                             catch (Exception ex)
                             {
                                 Messenger.Default.Send(new Exception(string.Format("Error during bulk process at range [{0}-{1}]", done, done + remaining), ex));
                                 status.ErrorCount += GlobalStore.SelectedEndpoint.BulkSize;
                             }
                             finally
                             {
                                 var bulkErrors = response.ResponseObject.Results.Where(br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
                                 if (bulkErrors.Any())
                                 {
                                     foreach (var error in bulkErrors)
                                     {
                                         Messenger.Default.Send(new Exception(string.Format("Id: {0}, error: {1}", error.Id, error.Error)));
                                         status.ErrorCount++;
                                     }
                                 }
                                 done                += GlobalStore.SelectedEndpoint.BulkSize;
                                 status.DoneCount     = done;
                                 status.ProgressValue = (done / (double)all) * 100;
                                 remaining           -= GlobalStore.SelectedEndpoint.BulkSize;
                                 tokens               = tokens.Skip(GlobalStore.SelectedEndpoint.BulkSize).ToList();
                             }
                         }
                         if (remaining > 0 && !cancellationToken.IsCancellationRequested)
                         {
                             var response = new ClientResponseWithObject <BulkResults>();
                             try
                             {
                                 var settings       = new DocumentBulkSettings();
                                 settings.Documents = tokens.Take(remaining).Select(t => t.ToObject <object>()).ToList();
                                 response           = new DocumentManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkDocumentsAsync(settings).Result;
                                 ResponseValidator.Validate(response, false);
                             }
                             catch (Exception ex)
                             {
                                 Messenger.Default.Send(new Exception(string.Format("Error during bulk process at range [{0}-{1}]", done, done + remaining), ex));
                                 status.ErrorCount += remaining;
                             }
                             finally
                             {
                                 var bulkErrors = response.ResponseObject?.Results.Where(br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
                                 if (bulkErrors != null && bulkErrors.Any())
                                 {
                                     foreach (var error in bulkErrors)
                                     {
                                         Messenger.Default.Send(new Exception(string.Format("Id: {0}, error: {1}", error.Id, error.Error)));
                                         status.ErrorCount++;
                                     }
                                 }
                                 done                += remaining;
                                 status.DoneCount     = done;
                                 status.ProgressValue = (done / (int)all) * 100;
                                 remaining           -= remaining;
                             }
                         }
                     }
                 }
                 catch (OperationCanceledException)
                 {
                     Log.Info(LogMessages.OperationCancelled);
                 }
             });
         }
         catch (Exception exception)
         {
             DispatcherHelper.CheckBeginInvokeOnUI(() => Messenger.Default.Send(exception));
         }
         finally
         {
             status.OperationIsFinished = true;
         }
     });
 }
        private async void ImportCsv <T>()
        {
            var ofd = new OpenFileDialog
            {
                Multiselect = false,
                Filter      = "CSV|*.csv"
            };

            if (ofd.ShowDialog() == true)
            {
                var importSettings = new CsvImportSettings {
                    Delimiter = ",", Force = true
                };
                var dialogResult = await _dialogHandler.Show(
                    new CommonDialog
                {
                    DataContext =
                        new CommonDialogViewModel
                    {
                        Header  = "Csv Import Settings",
                        Content = importSettings,
                        Buttons = ButtonsEnum.OkCancel
                    }
                }, "RootDialog");

                if ((CommonDialogResult)dialogResult == CommonDialogResult.Cancel)
                {
                    return;
                }
                var invalidRows       = new List <int>();
                var errors            = new ConcurrentBag <string>();
                var cancellationToken = new CancellationTokenSource();
                var status            = new StatusDialogViewModel {
                    ProgressValue = 0, Title = typeof(T) == typeof(Tag) ? "Importing Tags" : "Importing documents", CancellationTokenSource = cancellationToken
                };
                await _dialogHandler.Show(new StatusDialog { DataContext = status }, "RootDialog",
                                          async (object sender, DialogOpenedEventArgs oa) =>
                {
                    FileStream stream = null;
                    try
                    {
                        stream                   = new FileStream(ofd.FileName, FileMode.Open);
                        stream.Position          = 0;
                        var importResult         = new CsvImportResult();
                        importSettings.CsvReader = CsvProcesser.GetCsvReader(stream, importSettings);
                        while (importResult.CanContinue && !cancellationToken.IsCancellationRequested)
                        {
                            importResult = await CsvProcesser.GetTokens(importSettings, typeof(T) == typeof(Tag));
                            invalidRows.AddRange(importResult.InvalidRows);
                            status.ErrorCount += importResult.InvalidRows.Count;

                            if (typeof(T) == typeof(Tag))
                            {
                                var response = new ClientResponseWithObject <BulkResults>();
                                try
                                {
                                    var settings  = new TagBulkSettings();
                                    settings.Tags = importResult.Tokens.Select(t => t.ToObject <Tag>()).ToList();
                                    response      = await new TagManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkTagsAsync(settings);
                                    ResponseValidator.Validate(response, false);
                                }
                                catch (Exception ex)
                                {
                                    errors.Add(string.Format("Error during bulk process:{0}{1}: {2}", Environment.NewLine, ex.Message, string.Join(", ", response.Errors)));
                                    status.ErrorCount += importResult.Tokens.Count;
                                }
                                finally
                                {
                                    if (response.IsSuccessful)
                                    {
                                        var bulkErrors =
                                            response.ResponseObject.Results.Where(
                                                br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
                                        if (bulkErrors.Any())
                                        {
                                            foreach (var error in bulkErrors)
                                            {
                                                errors.Add(string.Format("Id: {0}, error: {1}", error.Id,
                                                                         error.Error));
                                                status.ErrorCount++;
                                            }
                                        }
                                    }
                                    status.DoneCount        += (importResult.Tokens.Count + importResult.InvalidRows.Count);
                                    status.ProgressValue     = (stream.Position / (double)stream.Length) * 100;
                                    importResult.Tokens      = null;
                                    importResult.InvalidRows = null;
                                }
                            }
                            else
                            {
                                var response = new ClientResponseWithObject <BulkResults>();
                                try
                                {
                                    var settings       = new DocumentBulkSettings();
                                    var idFieldIsArray = false;
                                    if (SelectedDataSet.SampleDocument != null)
                                    {
                                        var docObject  = JObject.FromObject(SelectedDataSet.SampleDocument);
                                        idFieldIsArray = docObject?[SelectedDataSet.TagField] is JArray;
                                    }
                                    else
                                    {
                                        var docObject  = JObject.FromObject(SelectedDataSet.Schema);
                                        idFieldIsArray = docObject?["properties"][SelectedDataSet.TagField]["type"].ToString().ToLower() == "array";
                                    }
                                    if (idFieldIsArray)
                                    {
                                        settings.Documents = importResult.Tokens.Select(t =>
                                        {
                                            t[SelectedDataSet.TagField] = new JArray(t[SelectedDataSet.TagField]);
                                            return(t);
                                        }).Select(t => t.ToObject <object>()).ToList();
                                    }
                                    else
                                    {
                                        settings.Documents = importResult.Tokens.Select(t => t.ToObject <object>()).ToList();
                                    }
                                    response = await new DocumentManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkDocumentsAsync(settings);
                                    ResponseValidator.Validate(response, false);
                                }
                                catch (Exception ex)
                                {
                                    Messenger.Default.Send(ex);
                                    status.ErrorCount += GlobalStore.SelectedEndpoint.BulkSize;
                                }
                                finally
                                {
                                    if (response.IsSuccessful)
                                    {
                                        var bulkErrors =
                                            response.ResponseObject.Results.Where(
                                                br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
                                        if (bulkErrors.Any())
                                        {
                                            foreach (var error in bulkErrors)
                                            {
                                                errors.Add(string.Format("Id: {0}, error: {1}", error.Id,
                                                                         error.Error));
                                                status.ErrorCount++;
                                            }
                                        }
                                    }
                                    status.DoneCount        += (importResult.Tokens.Count + importResult.InvalidRows.Count);
                                    status.ProgressValue     = (stream.Position / (double)stream.Length) * 100;
                                    importResult.Tokens      = null;
                                    importResult.InvalidRows = null;
                                }
                            }
                        }
                    }
                    catch (Exception exception)
                    {
                        DispatcherHelper.CheckBeginInvokeOnUI(() => Messenger.Default.Send(exception));
                    }
                    finally
                    {
                        stream?.Close();
                        status.OperationIsFinished = true;
                    }
                });

                if (invalidRows.Any())
                {
                    Messenger.Default.Send(new Exception($"Invalid rows:{Environment.NewLine}{string.Join(Environment.NewLine, invalidRows)}"));
                }
                if (errors.Any())
                {
                    Messenger.Default.Send(new Exception($"Errors:{Environment.NewLine}{string.Join(Environment.NewLine, errors)}"));
                }
            }
        }