public static DataLoaderBase CreateLoader(ImportMode mode, SqlConnection connection,
BatchParams batchParams, string targetTable, IEnumerable<string> columns)
{
switch (mode)
{
case ImportMode.BulkInsert:
return new BulkDataLoader(connection, batchParams, targetTable, columns, false);
case ImportMode.BulkInsertWithSkipped:
return new BulkDataLoader(connection, batchParams, targetTable, columns, true);
case ImportMode.BulkDelete:
return new DeleteDataLoader(connection, batchParams, targetTable, columns);
case ImportMode.Smart:
return new SmartDataLoader(connection, batchParams, targetTable, columns);
default:
throw new InvalidOperationException("Unknown ImportMode - '" + mode.ToString() + "'");
}
}
public async Task ValidateAsync_ProductMissed_WillFail(ImportMode importMode)
{
// Arrange
var validator = GetValidator();
// Act
var validationResult = await validator.ValidateAsync(Prices, options => options.IncludeRuleSets(importMode.ToString()));
// Assert
var errors = validationResult.Errors.Where(validationError => validationError.ErrorCode == ModuleConstants.ValidationErrors.ProductMissingError).ToArray();
Assert.NotNull(errors);
Assert.Equal(4, errors.Length);
}
public override void Execute()
{
try
{
Dictionary <string, SyncEntity> completedImports = ReadCompletedImports();
var allFiles = ReadFiles(FilesFilter);
Program.Console($"Found {allFiles.Length} files to import");
foreach (var f in allFiles.OrderBy(f => f.Name))
{
if (completedImports.ContainsKey(f.Name))
{
Program.Console($"Skipping {f.Name}, already imported on {completedImports[f.Name].Imported.ToString("u")}");
continue;
}
var fileNameSplit = f.Name.Split('.');
if (fileNameSplit.Length != 4)
{
throw new InvalidOperationException($"File {f.Name} do not have 4 components: [Order].[Collection].[Operation].json");
}
string collectionName = fileNameSplit[1];
ImportMode importMode = Enum.Parse <ImportMode>(fileNameSplit[2], true);
Program.Console($"Importing {f.Name} to collection {collectionName}, mode {importMode}");
SyncEntity importResult = new SyncEntity();
importResult.FileName = f.Name;
try
{
switch (importMode)
{
case ImportMode.Drop:
dropCollection(collectionName);
break;
case ImportMode.Delete:
deleteFromCollection(collectionName, f);
break;
case ImportMode.Insert:
case ImportMode.Upsert:
case ImportMode.Merge:
var resultCode = Program.Exec(IMPORT_COMMAND, $"{ConnectionString.ToCommandLine()}{AuthenticationDatabaseToCommandLine()} --collection {collectionName} --type json --mode {importMode.ToString().ToLower()} --stopOnError --file {f.Name}");
if (resultCode != 0)
{
throw new InvalidOperationException($"mongoimport result code {resultCode}, interrupting");
}
break;
case ImportMode.Eval:
var evalResultCode = Program.Exec(MONGO_COMMAND, $"{Connection} {f.Name}");
if (evalResultCode != 0)
{
throw new InvalidOperationException($"mongo result code {evalResultCode}, interrupting");
}
break;
case ImportMode.CreateIndex:
var text = File.ReadAllText(f.FullName);
Program.Console("Index params:");
Program.Console(text);
var command = string.Format(CREATE_INDEX, collectionName, text);
var fileName = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".js");
File.WriteAllText(fileName, command);
try
{
var createIndexResultCode = Program.Exec(MONGO_COMMAND, $"{Connection} {fileName}");
switch (createIndexResultCode)
{
// no error
case 0:
break;
case 11000:
throw new InvalidOperationException($"CreateIndex failed with error 'duplicate key error', interrupting");
default:
// all error codes with explanation
// https://github.com/mongodb/mongo/blob/master/src/mongo/base/error_codes.err
throw new InvalidOperationException($"CreateIndex result code {createIndexResultCode}, interrupting");
}
}
finally
{
File.Delete(fileName);
}
break;
default: throw new InvalidOperationException($"Import mode {importMode} not implemented yet");
}
importResult.Success = true;
}
finally
{
SyncCollection.InsertOne(importResult);
}
}
Program.Console($"Import completed successfully");
}
catch (Exception ex)
{
Program.Console($"Error during import: {ex.Message}");
Environment.Exit(1);
}
}