public static async Task <CsvImportResult> GetTokens(CsvImportSettings importSettings, bool unlimied = false)
        {
            return(await Task.Run(() =>
            {
                var result = new CsvImportResult();

                while (result.CanContinue && (unlimied || result.Tokens.Count + result.InvalidRows.Count < GlobalStore.SelectedEndpoint.BulkSize))
                {
                    try
                    {
                        result.CanContinue = importSettings.CsvReader.Read();
                    }
                    catch (Exception exception)
                    {
                        result.InvalidRows.Add(((CsvParser)importSettings.CsvReader.Parser).RawRow);
                        if (!importSettings.Force)
                        {
                            result.CanContinue = false;
                        }
                        continue;
                    }
                    if (!result.CanContinue)
                    {
                        continue;
                    }

                    var tokenString = "";
                    try
                    {
                        tokenString = "{" + string.Join(",", importSettings.CsvReader.FieldHeaders.Select(h => $"\"{h}\":\"{(importSettings.CsvReader[h] == "NULL" ? "" : importSettings.CsvReader[h].Replace("\"","\\\""))}\"")) + "}";
                        result.Tokens.Add(JToken.Parse(tokenString));
                    }
                    catch (Exception exception)
                    {
                        result.InvalidRows.Add(((CsvParser)importSettings.CsvReader.Parser).RawRow);
                        if (!importSettings.Force)
                        {
                            result.CanContinue = false;
                        }
                    }
                }
                return result;
            }));
        }
示例#2
0
 /// <summary>
 /// ログ出力
 /// </summary>
 /// <param name="result"></param>
 static void WriteLog(CsvImportResult result)
 {
     var log = new StringBuilder();
     if (result.Successed == 0)
     {
         log.AppendLine("全件失敗");
     }
     else
     {
         log.Append("成功:").Append(result.Successed);
         log.AppendLine();
     }
     if (result.FailedInfo.Length > 0)
     {
         log.Append("失敗:").Append(result.FailedInfo.Length);
         log.AppendLine();
         foreach (var failed in result.FailedInfo)
         {
             log.Append(failed.LineNo).Append("行目:").Append(failed.Message);
             log.AppendLine();
         }
     }
     Console.WriteLine(log);
 }
        private async void ImportCsv <T>()
        {
            var ofd = new OpenFileDialog
            {
                Multiselect = false,
                Filter      = "CSV|*.csv"
            };

            if (ofd.ShowDialog() == true)
            {
                var importSettings = new CsvImportSettings {
                    Delimiter = ",", Force = true
                };
                var dialogResult = await _dialogHandler.Show(
                    new CommonDialog
                {
                    DataContext =
                        new CommonDialogViewModel
                    {
                        Header  = "Csv Import Settings",
                        Content = importSettings,
                        Buttons = ButtonsEnum.OkCancel
                    }
                }, "RootDialog");

                if ((CommonDialogResult)dialogResult == CommonDialogResult.Cancel)
                {
                    return;
                }
                var invalidRows       = new List <int>();
                var errors            = new ConcurrentBag <string>();
                var cancellationToken = new CancellationTokenSource();
                var status            = new StatusDialogViewModel {
                    ProgressValue = 0, Title = typeof(T) == typeof(Tag) ? "Importing Tags" : "Importing documents", CancellationTokenSource = cancellationToken
                };
                await _dialogHandler.Show(new StatusDialog { DataContext = status }, "RootDialog",
                                          async (object sender, DialogOpenedEventArgs oa) =>
                {
                    FileStream stream = null;
                    try
                    {
                        stream                   = new FileStream(ofd.FileName, FileMode.Open);
                        stream.Position          = 0;
                        var importResult         = new CsvImportResult();
                        importSettings.CsvReader = CsvProcesser.GetCsvReader(stream, importSettings);
                        while (importResult.CanContinue && !cancellationToken.IsCancellationRequested)
                        {
                            importResult = await CsvProcesser.GetTokens(importSettings, typeof(T) == typeof(Tag));
                            invalidRows.AddRange(importResult.InvalidRows);
                            status.ErrorCount += importResult.InvalidRows.Count;

                            if (typeof(T) == typeof(Tag))
                            {
                                var response = new ClientResponseWithObject <BulkResults>();
                                try
                                {
                                    var settings  = new TagBulkSettings();
                                    settings.Tags = importResult.Tokens.Select(t => t.ToObject <Tag>()).ToList();
                                    response      = await new TagManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkTagsAsync(settings);
                                    ResponseValidator.Validate(response, false);
                                }
                                catch (Exception ex)
                                {
                                    errors.Add(string.Format("Error during bulk process:{0}{1}: {2}", Environment.NewLine, ex.Message, string.Join(", ", response.Errors)));
                                    status.ErrorCount += importResult.Tokens.Count;
                                }
                                finally
                                {
                                    if (response.IsSuccessful)
                                    {
                                        var bulkErrors =
                                            response.ResponseObject.Results.Where(
                                                br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
                                        if (bulkErrors.Any())
                                        {
                                            foreach (var error in bulkErrors)
                                            {
                                                errors.Add(string.Format("Id: {0}, error: {1}", error.Id,
                                                                         error.Error));
                                                status.ErrorCount++;
                                            }
                                        }
                                    }
                                    status.DoneCount        += (importResult.Tokens.Count + importResult.InvalidRows.Count);
                                    status.ProgressValue     = (stream.Position / (double)stream.Length) * 100;
                                    importResult.Tokens      = null;
                                    importResult.InvalidRows = null;
                                }
                            }
                            else
                            {
                                var response = new ClientResponseWithObject <BulkResults>();
                                try
                                {
                                    var settings       = new DocumentBulkSettings();
                                    var idFieldIsArray = false;
                                    if (SelectedDataSet.SampleDocument != null)
                                    {
                                        var docObject  = JObject.FromObject(SelectedDataSet.SampleDocument);
                                        idFieldIsArray = docObject?[SelectedDataSet.TagField] is JArray;
                                    }
                                    else
                                    {
                                        var docObject  = JObject.FromObject(SelectedDataSet.Schema);
                                        idFieldIsArray = docObject?["properties"][SelectedDataSet.TagField]["type"].ToString().ToLower() == "array";
                                    }
                                    if (idFieldIsArray)
                                    {
                                        settings.Documents = importResult.Tokens.Select(t =>
                                        {
                                            t[SelectedDataSet.TagField] = new JArray(t[SelectedDataSet.TagField]);
                                            return(t);
                                        }).Select(t => t.ToObject <object>()).ToList();
                                    }
                                    else
                                    {
                                        settings.Documents = importResult.Tokens.Select(t => t.ToObject <object>()).ToList();
                                    }
                                    response = await new DocumentManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkDocumentsAsync(settings);
                                    ResponseValidator.Validate(response, false);
                                }
                                catch (Exception ex)
                                {
                                    Messenger.Default.Send(ex);
                                    status.ErrorCount += GlobalStore.SelectedEndpoint.BulkSize;
                                }
                                finally
                                {
                                    if (response.IsSuccessful)
                                    {
                                        var bulkErrors =
                                            response.ResponseObject.Results.Where(
                                                br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
                                        if (bulkErrors.Any())
                                        {
                                            foreach (var error in bulkErrors)
                                            {
                                                errors.Add(string.Format("Id: {0}, error: {1}", error.Id,
                                                                         error.Error));
                                                status.ErrorCount++;
                                            }
                                        }
                                    }
                                    status.DoneCount        += (importResult.Tokens.Count + importResult.InvalidRows.Count);
                                    status.ProgressValue     = (stream.Position / (double)stream.Length) * 100;
                                    importResult.Tokens      = null;
                                    importResult.InvalidRows = null;
                                }
                            }
                        }
                    }
                    catch (Exception exception)
                    {
                        DispatcherHelper.CheckBeginInvokeOnUI(() => Messenger.Default.Send(exception));
                    }
                    finally
                    {
                        stream?.Close();
                        status.OperationIsFinished = true;
                    }
                });

                if (invalidRows.Any())
                {
                    Messenger.Default.Send(new Exception($"Invalid rows:{Environment.NewLine}{string.Join(Environment.NewLine, invalidRows)}"));
                }
                if (errors.Any())
                {
                    Messenger.Default.Send(new Exception($"Errors:{Environment.NewLine}{string.Join(Environment.NewLine, errors)}"));
                }
            }
        }
        static void Main(string[] args)
        {
            var stopWatch = Stopwatch.StartNew();

            Console.WriteLine("ESFA RoATP Data Importer");

            if (args.Length < 1 || args.Length > 2)
            {
                Console.WriteLine("Usage: dotnet RoATPImporter.dll CSV file path [log file path]");
                Environment.Exit(-1);
            }

            string csvFilePath = args[0];

            if (args.Length == 2)
            {
                string logFilePath = args[1];
                RegisterImportLogger.Instance.LogFileName = logFilePath;
            }

            Configuration = new ConfigurationBuilder()
                            .AddJsonFile("appsettings.json", true, true)
                            .Build();

            ApplicationConfiguration = ConfigurationService.GetConfig(Configuration["EnvironmentName"], Configuration["ConfigurationStorageConnectionString"], Version, ServiceName).Result;

            using (var reader = new StreamReader(csvFilePath))
            {
                CsvParser       parser  = new CsvParser(new Logger <CsvParser>(NullLoggerFactory.Instance));
                CsvImportResult results = parser.ParseCsvFile(reader);

                if (results.ErrorLog.Count == 0)
                {
                    RegisterImporter importer = new RegisterImporter(new Logger <RegisterImporter>(NullLoggerFactory.Instance));

                    try
                    {
                        importer.ImportRegisterEntries(ApplicationConfiguration.SqlConnectionString, results.Entries).GetAwaiter().GetResult();
                    }
                    catch (RegisterImportException importException)
                    {
                        Console.WriteLine("Unexpected error when importing organisation " + importException.UKPRN);
                        Console.WriteLine(importException.ImportErrorMessage);
                        Environment.Exit(-1);
                    }

                    Console.WriteLine(results.Entries.Count + " register entries imported successfully");
                }
                else
                {
                    foreach (var errorMessage in results.ErrorLog)
                    {
                        Console.WriteLine(errorMessage);
                    }
                }
                Console.WriteLine("Completed in " + stopWatch.ElapsedMilliseconds + "ms");

                if (RegisterImportLogger.Instance.LogEnabled)
                {
                    RegisterImportLogger.Instance.Close();
                    Console.WriteLine("SQL output logged to " + RegisterImportLogger.Instance.LogFileName);
                }

                Console.ReadLine();
            }
        }