public override async Task RemoveEntitiesAsync(IEnumerable entities, ImportFileResult result)
{
var store = _provider.GetRequiredService <IAdminStore <T> >();
foreach (var entity in entities)
{
await RemoveEntityAsync(entity as T, store, result).ConfigureAwait(false);
}
}
public void AddImportedFile_AddsFileResult()
{
var result = new ImportResult();
var file = new ImportFileResult("1.txt", 123);
result.AddImportedFile(file);
Assert.AreEqual(1, result.Files.Count(), "Expected 1 file in Files" );
Assert.That(result.Files.Any(f => f == file), "Expected file not found");
}
public override async Task <ImportFileResult> ImportAsync(string content)
{
var result = new ImportFileResult();
var entityList = JsonConvert.DeserializeObject <PageResponse <T> >(content);
var store = _provider.GetRequiredService <IAdminStore <T> >();
foreach (var entity in entityList.Items)
{
await AddOrUpdateEntityAsync(entity, store, result).ConfigureAwait(false);
}
return(result);
}
private async Task AddOrUpdateEntityAsync(T entity, IAdminStore <T> store, ImportFileResult result)
{
var subEntities = GetSubEntities(entity);
var existing = await store.GetAsync(entity.Id, null).ConfigureAwait(false);
if (existing != null)
{
entity = await store.UpdateAsync(entity).ConfigureAwait(false);
result.Updated.Add(entity.Id);
}
else
{
entity = await store.CreateAsync(entity).ConfigureAwait(false);
result.Created.Add(entity.Id);
}
var subResults = new ImportFileResult();
result.SubEntitiesResults.Add(entity.Id, subResults);
await ImportSubEntitiesAsync(entity, subEntities, store, subResults).ConfigureAwait(false);
}
private async Task ImportSubEntitiesAsync(T entity, Dictionary <string, IEnumerable> subEntities, IAdminStore <T> store, ImportFileResult result)
{
if (!subEntities.Any())
{
return;
}
var expand = string.Join(",", subEntities.Keys);
entity = await store.GetAsync(entity.Id, new GetRequest { Expand = expand }).ConfigureAwait(false);
foreach (var key in subEntities.Keys)
{
if (subEntities[key] == null)
{
continue;
}
var property = typeof(T).GetProperty(key);
var subEntityList = property.GetValue(entity) as IEnumerable;
var entityType = property.PropertyType.GetGenericArguments()[0];
var importerType = typeof(Importer <>).MakeGenericType(entityType);
var importer = Activator.CreateInstance(importerType, _provider) as Importer;
await importer.RemoveEntitiesAsync(subEntityList, result).ConfigureAwait(false);
}
foreach (var value in subEntities.Select(e => e.Value))
{
var enumerator = value.GetEnumerator();
if (!enumerator.MoveNext())
{
continue;
}
var entityType = enumerator.Current.GetType();
var importerType = typeof(Importer <>).MakeGenericType(entityType);
var importer = Activator.CreateInstance(importerType, _provider) as Importer;
await importer.AddOrUpdateSubEntitiesAsync(value, result).ConfigureAwait(false);
}
}
public static ImportFileResult ImportFile(string _filename, TaxonSearch _searchTool, bool _logFoundTaxon = true)
{
ImportFileResult result = new ImportFileResult();
List <string> notFound = new List <string>();
List <Tuple <string, TaxonTreeNode> > found = new List <Tuple <string, TaxonTreeNode> >();
using (StreamReader file = new StreamReader(_filename))
{
string line;
while ((line = file.ReadLine()) != null)
{
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
TaxonTreeNode node = _searchTool.FindOne(line);
if (node == null)
{
notFound.Add(line);
}
else
{
found.Add(new Tuple <string, TaxonTreeNode>(line, node));
}
}
}
result.LogFilename = _filename.Replace(".txt", "") + "_import.log";
using (StreamWriter log = new StreamWriter(result.LogFilename))
{
log.WriteLine("Import " + _filename + " results");
log.WriteLine("");
log.WriteLine(found.Count + " taxons found");
log.WriteLine(notFound.Count + " taxons not found");
log.WriteLine("");
log.WriteLine("Not found:");
foreach (string name in notFound)
{
log.WriteLine(" " + name);
}
log.WriteLine("");
if (_logFoundTaxon)
{
log.WriteLine("Found:");
foreach (Tuple <string, TaxonTreeNode> tuple in found)
{
log.WriteLine(" " + tuple.Item1 + " ==> " + tuple.Item2.GetHierarchicalName());
}
}
log.WriteLine("");
}
result.TaxonsFound = found.Count;
result.TaxonNotFound = notFound.Count;
Dictionary <TaxonTreeNode, bool> dico = new Dictionary <TaxonTreeNode, bool>();
foreach (Tuple <string, TaxonTreeNode> tuple in found)
{
if (dico.ContainsKey(tuple.Item2))
{
continue;
}
dico[tuple.Item2] = true;
}
result.List = dico.Keys.ToList();
return(result);
}
private static async Task RemoveEntityAsync(T entity, IAdminStore <T> store, ImportFileResult result)
{
await store.DeleteAsync(entity.Id).ConfigureAwait(false);
result.Deleted.Add(entity.Id);
}
public void TestFileImportServer()
{
// start the server, connect a client1, and shutdown
using (ILogger logger = new TraceLogger(true))
{
const EnvId env = EnvId.Utt_UnitTest;
var logRef = Reference <ILogger> .Create(logger);
var random = new Random(Environment.TickCount);
int port = random.Next(8000, 8099);
using (var server = new CoreServer(logRef, env.ToString(), NodeType.Router, port, WcfConst.NetTcp))
{
// start server
server.Start();
// connect client
using (ICoreClient client = new CoreClientFactory(logRef).SetEnv(env.ToString()).Create())
{
// create test import rule
var rule = new FileImportRule
{
RuleName = "UnitTest",
SourceLocation = @"C:\windows\system32",
TargetLocation = @"c:\temp",
CopyFilePatterns = "notep*.exe"
};
var name = rule.RuleName;
// start the file import server
//DateTimeOffset waitCompleted;
var target = new FileImportServer {
Client = Reference <ICoreClient> .Create(client)
};
client.SaveObject(rule, "UnitTest", null, true, TimeSpan.MaxValue);
using (target)//logger, env
{
target.Start();
Thread.Sleep(TimeSpan.FromSeconds(5));
//waitCompleted = DateTimeOffset.Now;
target.Stop();
}
{
var results1 = client.LoadItem <FileImportRule>(name);
Assert.IsNotNull(results1);
List <ImportRuleResult> results = client.LoadObjects <ImportRuleResult>(Expr.ALL);
Assert.AreEqual(1, results.Count);
ImportRuleResult result = results[0];
Assert.AreEqual("UnitTest", result.RuleName);
Assert.AreEqual("Completed", result.ImportResult);
Assert.AreEqual(1, result.FileNames.Length);
Assert.AreEqual("notepad.exe", result.FileNames[0].ToLower());
}
{
List <ImportFileResult> results = client.LoadObjects <ImportFileResult>(Expr.ALL);
Assert.AreEqual(1, results.Count);
ImportFileResult result = results[0];
Assert.AreEqual("UnitTest", result.RuleName);
Assert.AreEqual("Completed", result.ImportResult);
Assert.AreEqual("notepad.exe", result.FileName.ToLower());
}
}
// explicit shutdown
// - not necessary in a "using" block but run it anyway
server.Stop();
}
}
}
private void ProcessRule(ICoreClient client, InternalRule rule, DateTimeOffset currentTime)
{
RuleStatusEnum ruleStatus;
ILogger logger = new FilterLogger(
Logger, $"Rule {rule.RuleName}: ",
rule.DebugEnabled ? LogSeverity.Debug : LogSeverity.Info);
using (var settings = new SettingsTracker(client, "FileImporter." + rule.RuleName))
{
bool lastCheckFailed = false;
string lastCheckException = "(null)";
DateTimeOffset lastCheckDateTime = currentTime;
//logger.LogDebug("Processing...");
try
{
ruleStatus = RuleStatusEnum.Disabled;
if (!rule.Disabled)
{
// evaluate rule constraint and condition
var properties = new NamedValueSet(settings.GetAllValues(true));
properties.Add(rule.Properties);
// last import date/time (default to 4 days ago)
var lastImportDateTime = settings.GetSetValue(RuleConst.LastImportDateTime, DateTimeOffset.Now.AddDays(-4));
properties.Set(RuleConst.LastImportDateTime, lastImportDateTime);
// calculate effective "as at" date
var thisImportDateTime = Expr.CastTo(rule.EffectiveDateExpr.Evaluate(properties, currentTime), currentTime);
properties.Set(RuleConst.EffectiveDateTime, thisImportDateTime);
// add useful date/time tokens
//foreach (string token in new string[] { "dd", "ddd", "MM", "MMM", "yyyy" })
//{
// properties.Set(token, thisImportDateTime.ToString(token));
//}
// calculate import delay
var thisImportDelay = Expr.CastTo(rule.ImportDelayExpr.Evaluate(properties, currentTime), TimeSpan.Zero);
properties.Set(RuleConst.ImportDelay, thisImportDelay);
// evaluate rule check constraint and import condition
logger.LogDebug("Evaluation Params :");
properties.LogValues(text => logger.LogDebug(" " + text));
logger.LogDebug("Check Constraint : {0}", rule.CheckConstraint);
ruleStatus = RuleStatusEnum.Inactive;
if (Expr.CastTo(rule.CheckConstraint.Evaluate(properties, currentTime), false))
{
logger.LogDebug("Import Condition : {0}", rule.ImportCondition);
ruleStatus = RuleStatusEnum.NotReady;
if (Expr.CastTo(rule.ImportCondition.Evaluate(properties, currentTime), false))
{
ruleStatus = RuleStatusEnum.Failed;
// import condition is true
// process date/time tokens
string targetLocation = StringHelper.ReplaceDateTimeTokens(rule.TargetLocation, thisImportDateTime);
string sourceLocation = StringHelper.ReplaceDateTimeTokens(rule.SourceLocation, thisImportDateTime);
var importedFiles = new List <string>();
logger.LogInfo("Source Location : {0}", sourceLocation);
logger.LogInfo("Target Location : {0}", targetLocation);
logger.LogInfo("Filenames to copy: {0}", rule.CopyFilePatterns);
string thisImportException = "(null)";
try
{
// import the file
// - optionally clean up old files aged more than 7 days
if (rule.RemoveOldTargetFiles)
{
try
{
string[] oldTargetFiles = Directory.GetFiles(targetLocation, "*.*", SearchOption.TopDirectoryOnly);
foreach (string oldTargetFile in oldTargetFiles)
{
var targetFileInfo = new FileInfo(oldTargetFile);
if ((currentTime - targetFileInfo.LastWriteTime) > TimeSpan.FromDays(2))
{
File.Delete(oldTargetFile);
}
}
}
catch (IOException removeExcp)
{
logger.LogWarning("Error removing old files: {0}", removeExcp.GetType().Name);
// ignored
}
}
// - create target directory if required
if (!Directory.Exists(targetLocation))
{
Directory.CreateDirectory(targetLocation);
}
// - copy file(s) from source to target
foreach (string ruleFilePattern in rule.CopyFilePatterns.Split(';'))
{
string filePattern = StringHelper.ReplaceDateTimeTokens(ruleFilePattern, thisImportDateTime);
string[] sourceFiles = Directory.GetFiles(sourceLocation, filePattern, SearchOption.TopDirectoryOnly);
logger.LogInfo("Copying file(s): {0} ({1} found)", filePattern, sourceFiles.Length);
int copiedCount = 0;
int skippedCount = 0;
foreach (string sourceFileFullname in sourceFiles)
{
string sourceFileBaseName = Path.GetFileName(sourceFileFullname);
string targetFileFullname = $@"{targetLocation}\{sourceFileBaseName}";
bool copyRequired = true;
if (File.Exists(targetFileFullname) && rule.OnlyCopyUpdatedFiles)
{
var sourceFileInfo = new FileInfo(sourceFileFullname);
var targetFileInfo = new FileInfo(targetFileFullname);
copyRequired = (sourceFileInfo.LastWriteTime > targetFileInfo.LastWriteTime);
}
if (copyRequired)
{
logger.LogInfo("Copying file : {0}", sourceFileBaseName);
logger.LogInfo(" From source: {0}", sourceLocation);
logger.LogInfo(" To target: {0}", targetLocation);
DateTime copyCommenced = DateTime.Now;
File.Copy(sourceFileFullname, targetFileFullname, true);
TimeSpan copyDuration = DateTime.Now - copyCommenced;
importedFiles.Add(sourceFileBaseName);
var targetFileInfo = new FileInfo(targetFileFullname);
copiedCount++;
logger.LogInfo(" Copied {0}MB in {1}s ({2}KB/sec)",
(targetFileInfo.Length / 1000000.0).ToString("N"),
copyDuration.TotalSeconds.ToString("N"),
(targetFileInfo.Length / (1000.0 * copyDuration.TotalSeconds)).ToString("N"));
// publish rule import status
var importFileResult = new ImportFileResult
{
hostEnvName = EnvHelper.EnvName(IntClient.Target.ClientInfo.ConfigEnv),
hostComputer = IntClient.Target.ClientInfo.HostName,
hostInstance = null,
hostUserName = client.ClientInfo.UserName,
RuleName = rule.RuleName,
FileName = sourceFileBaseName,
DebugEnabled = rule.DebugEnabled,
DebugProperties = rule.DebugProperties.Serialise(),
FileContentType = rule.FileContentType,
ImportResult = RuleStatusEnum.Completed.ToString(),
ImportException = null,
ImportDateTime = currentTime.ToString("o"),
SourceSystem = rule.SourceSystem,
SourceLocation = sourceLocation,
TargetLocation = targetLocation
};
IntClient.Target.SaveObject(importFileResult, true, TimeSpan.FromDays(30));
}
else
{
skippedCount++;
logger.LogDebug("Skipping file : {0}", sourceFileBaseName);
}
} // foreach file
logger.LogInfo("Copied {0} file(s), skipped {1} file(s).", copiedCount, skippedCount);
}
// - optionally decompress target
// todo
// done
ruleStatus = RuleStatusEnum.Completed;
lastImportDateTime = Expr.CastTo(rule.DateUpdateExpr.Evaluate(properties, currentTime), currentTime);
}
catch (Exception e2)
{
logger.Log(e2);
thisImportException = e2.ToString();
ruleStatus = RuleStatusEnum.Failed;
}
finally
{
settings.SetNewValue(RuleConst.LastImportResult, ruleStatus.ToString());
settings.SetNewValue(RuleConst.LastImportException, thisImportException);
settings.SetNewValue(RuleConst.LastImportDateTime, lastImportDateTime);
}
// publish rule import status
var importRuleResult = new ImportRuleResult
{
hostEnvName = EnvHelper.EnvName(IntClient.Target.ClientInfo.ConfigEnv),
hostComputer = IntClient.Target.ClientInfo.HostName,
hostInstance = null,
hostUserName = client.ClientInfo.UserName,
RuleName = rule.RuleName,
ImportResult = ruleStatus.ToString(),
ImportException = thisImportException,
ImportDateTime = currentTime.ToString("o"),
SourceSystem = rule.SourceSystem,
SourceLocation = sourceLocation,
TargetLocation = targetLocation,
FileNames = importedFiles.ToArray()
};
IntClient.Target.SaveObject(importRuleResult, true, TimeSpan.FromDays(30));
}
}
}
}
catch (Exception e)
{
logger.Log(e);
lastCheckFailed = true;
lastCheckException = e.ToString();
ruleStatus = RuleStatusEnum.Failed;
}
settings.SetNewValue(RuleConst.LastCheckFailed, lastCheckFailed);
settings.SetNewValue(RuleConst.LastCheckException, lastCheckException);
settings.SetNewValue(RuleConst.LastCheckDateTime, lastCheckDateTime);
} // commit unsaved settings
logger.LogDebug("Status={0}", ruleStatus);
}
public void Constructor_Success_SetsSuccessFlagToTrue()
{
var file = new ImportFileResult(TestFileName, TestNumberOfTweets);
Assert.IsTrue(file.Success);
}
public void Constructor_Success_SetsNumberOfTweets()
{
var file = new ImportFileResult(TestFileName, TestNumberOfTweets);
Assert.AreEqual(TestNumberOfTweets, file.NumberOfTweets);
}
public void Constructor_Success_DefaultsErrorMessageToEmpty()
{
var file = new ImportFileResult(TestFileName, TestNumberOfTweets);
Assert.AreEqual(String.Empty, file.ErrorMessage);
}
public void Constructor_Failure_SetsSuccessFlagToFalse()
{
var file = new ImportFileResult(TestFileName, TestErrorMessage);
Assert.IsFalse(file.Success);
}
public void Constructor_Failure_SetsFileName()
{
var file = new ImportFileResult(TestFileName, TestErrorMessage);
Assert.AreEqual(TestFileName, file.FileName);
}
