public static async Task <CsvImportResult> GetTokens(CsvImportSettings importSettings, bool unlimied = false)
{
return(await Task.Run(() =>
{
var result = new CsvImportResult();
while (result.CanContinue && (unlimied || result.Tokens.Count + result.InvalidRows.Count < GlobalStore.SelectedEndpoint.BulkSize))
{
try
{
result.CanContinue = importSettings.CsvReader.Read();
}
catch (Exception exception)
{
result.InvalidRows.Add(((CsvParser)importSettings.CsvReader.Parser).RawRow);
if (!importSettings.Force)
{
result.CanContinue = false;
}
continue;
}
if (!result.CanContinue)
{
continue;
}
var tokenString = "";
try
{
tokenString = "{" + string.Join(",", importSettings.CsvReader.FieldHeaders.Select(h => $"\"{h}\":\"{(importSettings.CsvReader[h] == "NULL" ? "" : importSettings.CsvReader[h].Replace("\"","\\\""))}\"")) + "}";
result.Tokens.Add(JToken.Parse(tokenString));
}
catch (Exception exception)
{
result.InvalidRows.Add(((CsvParser)importSettings.CsvReader.Parser).RawRow);
if (!importSettings.Force)
{
result.CanContinue = false;
}
}
}
return result;
}));
}
/// <summary>
/// ログ出力
/// </summary>
/// <param name="result"></param>
static void WriteLog(CsvImportResult result)
{
var log = new StringBuilder();
if (result.Successed == 0)
{
log.AppendLine("全件失敗");
}
else
{
log.Append("成功：").Append(result.Successed);
log.AppendLine();
}
if (result.FailedInfo.Length > 0)
{
log.Append("失敗：").Append(result.FailedInfo.Length);
log.AppendLine();
foreach (var failed in result.FailedInfo)
{
log.Append(failed.LineNo).Append("行目：").Append(failed.Message);
log.AppendLine();
}
}
Console.WriteLine(log);
}
private async void ImportCsv <T>()
{
var ofd = new OpenFileDialog
{
Multiselect = false,
Filter = "CSV|*.csv"
};
if (ofd.ShowDialog() == true)
{
var importSettings = new CsvImportSettings {
Delimiter = ",", Force = true
};
var dialogResult = await _dialogHandler.Show(
new CommonDialog
{
DataContext =
new CommonDialogViewModel
{
Header = "Csv Import Settings",
Content = importSettings,
Buttons = ButtonsEnum.OkCancel
}
}, "RootDialog");
if ((CommonDialogResult)dialogResult == CommonDialogResult.Cancel)
{
return;
}
var invalidRows = new List <int>();
var errors = new ConcurrentBag <string>();
var cancellationToken = new CancellationTokenSource();
var status = new StatusDialogViewModel {
ProgressValue = 0, Title = typeof(T) == typeof(Tag) ? "Importing Tags" : "Importing documents", CancellationTokenSource = cancellationToken
};
await _dialogHandler.Show(new StatusDialog { DataContext = status }, "RootDialog",
async (object sender, DialogOpenedEventArgs oa) =>
{
FileStream stream = null;
try
{
stream = new FileStream(ofd.FileName, FileMode.Open);
stream.Position = 0;
var importResult = new CsvImportResult();
importSettings.CsvReader = CsvProcesser.GetCsvReader(stream, importSettings);
while (importResult.CanContinue && !cancellationToken.IsCancellationRequested)
{
importResult = await CsvProcesser.GetTokens(importSettings, typeof(T) == typeof(Tag));
invalidRows.AddRange(importResult.InvalidRows);
status.ErrorCount += importResult.InvalidRows.Count;
if (typeof(T) == typeof(Tag))
{
var response = new ClientResponseWithObject <BulkResults>();
try
{
var settings = new TagBulkSettings();
settings.Tags = importResult.Tokens.Select(t => t.ToObject <Tag>()).ToList();
response = await new TagManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkTagsAsync(settings);
ResponseValidator.Validate(response, false);
}
catch (Exception ex)
{
errors.Add(string.Format("Error during bulk process:{0}{1}: {2}", Environment.NewLine, ex.Message, string.Join(", ", response.Errors)));
status.ErrorCount += importResult.Tokens.Count;
}
finally
{
if (response.IsSuccessful)
{
var bulkErrors =
response.ResponseObject.Results.Where(
br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
if (bulkErrors.Any())
{
foreach (var error in bulkErrors)
{
errors.Add(string.Format("Id: {0}, error: {1}", error.Id,
error.Error));
status.ErrorCount++;
}
}
}
status.DoneCount += (importResult.Tokens.Count + importResult.InvalidRows.Count);
status.ProgressValue = (stream.Position / (double)stream.Length) * 100;
importResult.Tokens = null;
importResult.InvalidRows = null;
}
}
else
{
var response = new ClientResponseWithObject <BulkResults>();
try
{
var settings = new DocumentBulkSettings();
var idFieldIsArray = false;
if (SelectedDataSet.SampleDocument != null)
{
var docObject = JObject.FromObject(SelectedDataSet.SampleDocument);
idFieldIsArray = docObject?[SelectedDataSet.TagField] is JArray;
}
else
{
var docObject = JObject.FromObject(SelectedDataSet.Schema);
idFieldIsArray = docObject?["properties"][SelectedDataSet.TagField]["type"].ToString().ToLower() == "array";
}
if (idFieldIsArray)
{
settings.Documents = importResult.Tokens.Select(t =>
{
t[SelectedDataSet.TagField] = new JArray(t[SelectedDataSet.TagField]);
return(t);
}).Select(t => t.ToObject <object>()).ToList();
}
else
{
settings.Documents = importResult.Tokens.Select(t => t.ToObject <object>()).ToList();
}
response = await new DocumentManager(GlobalStore.SelectedEndpoint, SelectedDataSet.Name).BulkDocumentsAsync(settings);
ResponseValidator.Validate(response, false);
}
catch (Exception ex)
{
Messenger.Default.Send(ex);
status.ErrorCount += GlobalStore.SelectedEndpoint.BulkSize;
}
finally
{
if (response.IsSuccessful)
{
var bulkErrors =
response.ResponseObject.Results.Where(
br => br.StatusCode != (int)HttpStatusCode.OK).ToList();
if (bulkErrors.Any())
{
foreach (var error in bulkErrors)
{
errors.Add(string.Format("Id: {0}, error: {1}", error.Id,
error.Error));
status.ErrorCount++;
}
}
}
status.DoneCount += (importResult.Tokens.Count + importResult.InvalidRows.Count);
status.ProgressValue = (stream.Position / (double)stream.Length) * 100;
importResult.Tokens = null;
importResult.InvalidRows = null;
}
}
}
}
catch (Exception exception)
{
DispatcherHelper.CheckBeginInvokeOnUI(() => Messenger.Default.Send(exception));
}
finally
{
stream?.Close();
status.OperationIsFinished = true;
}
});
if (invalidRows.Any())
{
Messenger.Default.Send(new Exception($"Invalid rows:{Environment.NewLine}{string.Join(Environment.NewLine, invalidRows)}"));
}
if (errors.Any())
{
Messenger.Default.Send(new Exception($"Errors:{Environment.NewLine}{string.Join(Environment.NewLine, errors)}"));
}
}
}
static void Main(string[] args)
{
var stopWatch = Stopwatch.StartNew();
Console.WriteLine("ESFA RoATP Data Importer");
if (args.Length < 1 || args.Length > 2)
{
Console.WriteLine("Usage: dotnet RoATPImporter.dll CSV file path [log file path]");
Environment.Exit(-1);
}
string csvFilePath = args[0];
if (args.Length == 2)
{
string logFilePath = args[1];
RegisterImportLogger.Instance.LogFileName = logFilePath;
}
Configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", true, true)
.Build();
ApplicationConfiguration = ConfigurationService.GetConfig(Configuration["EnvironmentName"], Configuration["ConfigurationStorageConnectionString"], Version, ServiceName).Result;
using (var reader = new StreamReader(csvFilePath))
{
CsvParser parser = new CsvParser(new Logger <CsvParser>(NullLoggerFactory.Instance));
CsvImportResult results = parser.ParseCsvFile(reader);
if (results.ErrorLog.Count == 0)
{
RegisterImporter importer = new RegisterImporter(new Logger <RegisterImporter>(NullLoggerFactory.Instance));
try
{
importer.ImportRegisterEntries(ApplicationConfiguration.SqlConnectionString, results.Entries).GetAwaiter().GetResult();
}
catch (RegisterImportException importException)
{
Console.WriteLine("Unexpected error when importing organisation " + importException.UKPRN);
Console.WriteLine(importException.ImportErrorMessage);
Environment.Exit(-1);
}
Console.WriteLine(results.Entries.Count + " register entries imported successfully");
}
else
{
foreach (var errorMessage in results.ErrorLog)
{
Console.WriteLine(errorMessage);
}
}
Console.WriteLine("Completed in " + stopWatch.ElapsedMilliseconds + "ms");
if (RegisterImportLogger.Instance.LogEnabled)
{
RegisterImportLogger.Instance.Close();
Console.WriteLine("SQL output logged to " + RegisterImportLogger.Instance.LogFileName);
}
Console.ReadLine();
}
}
