public void CanHandleOrgProcessingAndCancel()
{
OrgsHandler.CheckAvailableOrgs(null);
Thread.Sleep(TimeSpan.FromSeconds(2));
// OrgsHandler.OrgProcessingTasks.ForEach(t => t.Item3.Cancel());
}
public void CanAddMoreOrgsToExisting()
{
OrgsHandler.CheckAvailableOrgs(null);
//Add here more orgs here
//And make sure that they are merged
OrgsHandler.CheckAvailableOrgs(null);
Assert.AreEqual(MockFileRepository.OrgIncrement * 2, OrgsHandler.OrgProcessingTasks.Count);
}
public void Initialize()
{
unityContainer = new UnityContainer();
unityContainer.RegisterInstance <IFileRepository>(respositoryInstance);
unityContainer.RegisterType <IHarvesterTasks, MockTaskScheduler>();
unityContainer.RegisterInstance <ILog>(log);
OrgsHandler.Clean();
respositoryInstance.Clean();
OrgsHandler.Initialize(unityContainer);
}
public void Initialize()
{
unityContainer = new UnityContainer();
unityContainer.RegisterInstance <IFileRepository>(respositoryInstance)
.RegisterInstance <IHarvesterTasks>(new LimitedConcurrencyHarvesterTasks())
.RegisterType <ITAGProcessorClient, MockRaptor>();
unityContainer.RegisterInstance <ILog>(log);
OrgsHandler.Clean();
OrgsHandler.TagFileSubmitterTasksTimeout = TimeSpan.FromMinutes(5);
respositoryInstance.Clean();
OrgsHandler.Initialize(unityContainer);
}
public void CanRemoveAbsentOrgs()
{
OrgsHandler.CheckAvailableOrgs(null);
//Save here two items from the list of created tasks
var org1 = OrgsHandler.OrgProcessingTasks.First().Value;
var org2 = OrgsHandler.OrgProcessingTasks.Last().Value;
//Remove these orgs from the list of orgs
respositoryInstance.DeleteOrgs(new Organization[] { org1.Item1, org2.Item1 });
OrgsHandler.CheckAvailableOrgs(null);
Assert.AreEqual(MockFileRepository.OrgIncrement * 2 - 2, OrgsHandler.OrgProcessingTasks.Count);
}
public static void Start()
{
log.LogDebug("TagFileHarvester.Start: Entered Start()");
//register dependencies here
OrgsHandler.Initialize(new UnityContainer().RegisterType <IFileRepository, FileRepository>()
.RegisterInstance <IHarvesterTasks>(new LimitedConcurrencyHarvesterTasks())
.RegisterInstance(log));
//here we need to sync filespaces and tasks
SyncTimer = new Timer(OrgsHandler.CheckAvailableOrgs);
SyncTimer.Change(TimeSpan.FromMinutes(3), TimeSpan.FromMilliseconds(0));
log.LogDebug("TagFileHarvester.Start: Running on multiple threads");
}
public Task StartAsync(CancellationToken cancellationToken)
{
OrgsHandler.MaxThreadsToProcessTagFiles = config.GetValueInt("MaxThreadsToProcessTagFiles");
OrgsHandler.tccSynchFilespaceShortName = config.GetValueString("TCCSynchFilespaceShortName");
OrgsHandler.tccSynchMachineFolder = config.GetValueString("TCCSynchMachineControlFolder");
OrgsHandler.TCCSynchProductionDataFolder = config.GetValueString("TCCSynchProductionDataFolder");
OrgsHandler.TCCSynchProductionDataArchivedFolder =
config.GetValueString("TCCSynchProductionDataArchivedFolder");
OrgsHandler.TCCSynchProjectBoundaryIssueFolder =
config.GetValueString("TCCSynchProjectBoundaryIssueFolder");
OrgsHandler.TCCSynchSubscriptionIssueFolder =
config.GetValueString("TCCSynchSubscriptionIssueFolder");
OrgsHandler.TCCSynchOtherIssueFolder = config.GetValueString("TCCSynchOtherIssueFolder");
OrgsHandler.TagFilesFolderLifeSpanInDays = (byte)config.GetValueInt("TagFilesFolderLifeSpanInDays");
OrgsHandler.TagFileSubmitterTasksTimeout = (TimeSpan)config.GetValueTimeSpan("TagFileSubmitterTasksTimeout");
OrgsHandler.TCCRequestTimeout = (TimeSpan)config.GetValueTimeSpan("TCCRequestTimeout");
OrgsHandler.NumberOfFilesInPackage = config.GetValueInt("NumberOfFilesInPackage");
OrgsHandler.OrgProcessingDelay = (TimeSpan)config.GetValueTimeSpan("OrgProcessingDelay");
OrgsHandler.CacheEnabled = (bool)config.GetValueBool("CacheEnabled");
OrgsHandler.FilenameDumpEnabled = (bool)config.GetValueBool("FilenameDumpEnabled");
OrgsHandler.ShortOrgName = config.GetValueString("ShortOrgName");
OrgsHandler.TagFileEndpoint = config.GetValueString("TagFileEndpoint");
OrgsHandler.newrelic = config.GetValueString("newrelic");
ServicePointManager.DefaultConnectionLimit = 8;
Log.LogDebug("TagFileHarvester.Start: Entered Start()");
var container = new UnityContainer().RegisterType <IFileRepository, FileRepository>()
.RegisterInstance <IHarvesterTasks>(new LimitedConcurrencyHarvesterTasks())
.RegisterInstance(Log);
//register dependencies here
OrgsHandler.Initialize(container);
FileRepository.Log = Log;
//here we need to sync filespaces and tasks
SyncTimer = new Timer(OrgsHandler.CheckAvailableOrgs);
SyncTimer.Change(TimeSpan.FromSeconds(5), (TimeSpan)config.GetValueTimeSpan("RefreshOrgsDelay"));
Log.LogInformation("TagFileHarvester.Started.");
return(Task.CompletedTask);
}
public OrgProcessingResult ProcessOrg(bool SingleCycle = false, Action <OrgProcessorTask> onOrgProcessed = null)
{
var fileTasks = new List <Task>();
var filenames = new List <FileRepository.TagFile>();
var failuredFiles = new List <FileRepository.TagFile>();
var processedFiles = new List <FileRepository.TagFile>();
var repositoryError = false;
if (cancellationToken.IsCancellationRequested)
{
return(Result);
}
if (OrgsHandler.newrelic == "true")
{
var eventAttributes = new Dictionary <string, object>
{
{ "Org", org.shortName }
};
NewRelic.Api.Agent.NewRelic.RecordCustomEvent("TagFileHarvester_Process", eventAttributes);
}
//Resolve all dependencies here
var fileRepository = Container.Resolve <IFileRepository>();
var harvesterTasks = Container.Resolve <IHarvesterTasks>();
var filetasksCancel = new CancellationTokenSource();
var sleepFlag = false;
try
{
//Clear previous results
filenames.Clear();
fileTasks.Clear();
Result.Reset();
failuredFiles.Clear();
//We need to get list of folder recursevly here
try
{
var folders = fileRepository.ListFolders(org, out _).ToList();
//this could be a long time to get files, so check if we are requested to stop
if (cancellationToken.IsCancellationRequested)
{
return(Result);
}
log.LogDebug("Found {0} folders for org {1}", folders.Count, org.shortName);
var files = folders.SelectMany(f => fileRepository.ListFiles(org, f)).ToList();
log.LogDebug("Found {0} files for org {1}", files.Count, org.shortName);
//this could be a long time to get files, so check if we are requested to stop
if (cancellationToken.IsCancellationRequested)
{
return(Result);
}
files.OrderBy(t => t.createdUTC).Take(OrgsHandler.NumberOfFilesInPackage).ForEach(filenames.Add);
//this could be a long time to get files, so check if we are requested to stop
if (cancellationToken.IsCancellationRequested)
{
return(Result);
}
log.LogDebug("Got {0} files for org {1}", filenames.Count, org.shortName);
}
catch (Exception ex)
{
repositoryError = true;
log.LogWarning("Repository error occured for org {0}, could not get files or folders from TCC Exception: {1}",
org.shortName, ex.Message);
}
var filelistlock = new object();
//If we are good with the repository proceed with files
if (!repositoryError && filenames.Count > 0)
{
//this could be a long time to get files, so check if we are requested to stop
if (cancellationToken.IsCancellationRequested)
{
return(Result);
}
//foreach filenames here - build chain of tasks and track execution
filenames.ForEach(f => fileTasks.Add(harvesterTasks
.StartNewLimitedConcurrency(
() =>
{
var localresult = new WebApiTagFileProcessTask(Container,
filetasksCancel.Token)
.ProcessTagfile(f.fullName, org);
lock (filelistlock)
{
Result.AggregateOrgResult(localresult);
if (localresult == null)
{
repositoryError = true;
failuredFiles.Add(f);
}
else
{
processedFiles.Add(f);
}
// raise flag that we have at least one failured file
log.LogDebug(
"TagFile {0} processed with result {1}",
f.fullName, JsonConvert.SerializeObject(localresult));
return(localresult);
}
}, filetasksCancel.Token)));
//And schedule processing of found tagfiles
if (!Task.WaitAll(fileTasks.ToArray(), (int)OrgsHandler.TagFileSubmitterTasksTimeout.TotalMilliseconds,
cancellationToken.Token))
{
log.LogWarning("Filetasks ran out of time for completion for org {0}", org.shortName);
repositoryError = true;
}
//cleanup tasks
fileTasks.Clear();
}
log.LogInformation("Org {0} cycle completed. Submitted files {1} Refused files {2} Errors {3}", org.shortName,
Result.ProcessedFiles, Result.RefusedFiles, Result.ErroneousFiles);
//Run callback action
try
{
if (onOrgProcessed != null)
{
onOrgProcessed.Invoke(this);
}
}
catch (Exception ex)
{
log.LogError("Failed while calling back", ex);
}
if (SingleCycle)
{
return(Result);
}
}
catch (Exception ex)
{
log.LogError("Exception while processing org {0} occured {1}", org.shortName, ex.Message);
return(Result);
}
if (!cancellationToken.IsCancellationRequested)
{
log.LogInformation("Rescheduling processing of org {0}", org.shortName);
//delete current task from org tacker and add a new one
//sleep only if there is nothing to process. Otherwise process everything we could have
var delayExecution = false;
log.LogDebug("Trying to Sleep for the org {0} {1} {2}", org.shortName, Result.ProcessedFiles,
Result.RefusedFiles);
if (!fileRepository.IsAnythingInCahe(org) && Result.ProcessedFiles == 0 && Result.RefusedFiles == 0 ||
sleepFlag)
{
log.LogDebug("Sleeping for the org {0}", org.shortName);
/*Task.Delay(OrgsHandler.OrgProcessingDelay, cancellationToken.Token).Wait();*/
delayExecution = true;
}
lock (OrgsHandler.OrgListLocker)
{
orgsTracker.Remove(orgsTracker.First(t => t.Value.Item1.shortName == org.shortName).Key);
orgsTracker.Add(harvesterTasks.StartNewLimitedConcurrency2(() =>
{
ProcessOrg(false,
t => log
.LogInformation("Tasks status is {0} in Queue1 and {1} in Queue2 on {2} Threads",
harvesterTasks.Status().Item1,
harvesterTasks.Status().Item2, OrgsHandler.GetUsedThreads()));
}, cancellationToken.Token, delayExecution),
new Tuple <Organization, CancellationTokenSource>(org, cancellationToken));
}
}
return(Result);
}
public void CanListOrgs()
{
Assert.AreEqual(MockFileRepository.OrgIncrement, OrgsHandler.GetOrgs().Count);
}
public void CanAddNewlyFoundOrgs()
{
OrgsHandler.CheckAvailableOrgs(null);
Assert.AreEqual(MockFileRepository.OrgIncrement, OrgsHandler.OrgProcessingTasks.Count);
}
