protected void ParallelExecute(IEnumerable <Action> Actions)
{
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
int TotalCount = 0;
foreach (Action Action in Actions)
{
Queue.Enqueue(Action);
TotalCount++;
}
for (; ;)
{
Queue.Wait(TimeSpan.FromSeconds(10.0));
int NumRemaining = Queue.NumRemaining;
Log.TraceInformation("Processed {0:n0} ({1}%)", TotalCount - NumRemaining, ((TotalCount - NumRemaining) * 100) / TotalCount);
if (NumRemaining == 0)
{
break;
}
}
}
}
/// <summary>
/// Executes the command
/// </summary>
public override void ExecuteBuild()
{
// Build a list of all the allowed tags
HashSet <string> AllowTags = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
foreach (string AllowTag in ParseParamValues("Allow"))
{
AllowTags.Add(AllowTag);
}
if (P4Enabled)
{
AllowTags.Add(P4Env.Branch);
AllowTags.Add(P4Env.Branch.Trim('/'));
AllowTags.UnionWith(P4Env.Branch.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries));
}
// Enumerate all the files in the workspace
Log("Finding files in workspace...");
List <FileInfo> FilesToCheck = new List <FileInfo>();
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
DirectoryInfo BaseDir = new DirectoryInfo(RootDirectory.FullName);
Queue.Enqueue(() => FindAllFiles(Queue, BaseDir, FilesToCheck));
Queue.Wait();
}
// Scan all of the files for invalid comments
Log("Scanning files...", FilesToCheck.Count);
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
foreach (FileInfo File in FilesToCheck)
{
FileInfo FileCapture = File;
Queue.Enqueue(() => ScanSourceFile(FileCapture, AllowTags));
}
while (!Queue.Wait(5 * 1000))
{
lock (this)
{
Log("Scanning files... [{0}/{1}]", FilesToCheck.Count - Queue.NumRemaining, FilesToCheck.Count);
}
}
}
}
/// <summary>
/// Enumerates all files under the given directory
/// </summary>
/// <param name="Queue">Queue to add additional subfolders to search to</param>
/// <param name="BaseDir">Directory to search</param>
/// <param name="FilesToCheck">Output list of files to check. Will be locked before adding items.</param>
void FindAllFiles(ThreadPoolWorkQueue Queue, DirectoryInfo BaseDir, List <FileInfo> FilesToCheck)
{
foreach (DirectoryInfo SubDir in BaseDir.EnumerateDirectories())
{
DirectoryInfo SubDirCapture = SubDir;
Queue.Enqueue(() => FindAllFiles(Queue, SubDirCapture, FilesToCheck));
}
foreach (FileInfo FileToCheck in BaseDir.EnumerateFiles())
{
if (TextExtensions.Contains(FileToCheck.Extension))
{
lock (FilesToCheck)
{
FilesToCheck.Add(FileToCheck);
}
}
}
}
/// <summary>
/// Finds all the source files under a given directory
/// </summary>
/// <param name="BaseDir">Directory to search</param>
/// <param name="SourceFiles">List to receive the files found. A lock will be taken on this object to ensure multiple threads do not add to it simultaneously.</param>
/// <param name="Queue">Queue for additional tasks to be added to</param>
void FindSourceFiles(DirectoryInfo BaseDir, List <FileReference> SourceFiles, ThreadPoolWorkQueue Queue)
{
foreach (DirectoryInfo SubDir in BaseDir.EnumerateDirectories())
{
if (!SubDir.Name.Equals("Intermediate", StringComparison.OrdinalIgnoreCase))
{
Queue.Enqueue(() => FindSourceFiles(SubDir, SourceFiles, Queue));
}
}
foreach (FileInfo File in BaseDir.EnumerateFiles())
{
if (File.Name.EndsWith(".h", StringComparison.OrdinalIgnoreCase) || File.Name.EndsWith(".cpp", StringComparison.OrdinalIgnoreCase))
{
if (!IgnoreFileNames.Contains(File.Name))
{
lock (SourceFiles)
{
SourceFiles.Add(new FileReference(File));
}
}
}
}
}
/// <summary>
/// Main entry point for the command
/// </summary>
public override void ExecuteBuild()
{
// Build a lookup of flags to set and clear for each identifier
Dictionary <string, int> IdentifierToIndex = new Dictionary <string, int>();
for (int Idx = 0; Idx < MacroPairs.GetLength(0); Idx++)
{
IdentifierToIndex[MacroPairs[Idx, 0]] = Idx;
IdentifierToIndex[MacroPairs[Idx, 1]] = ~Idx;
}
// Check if we want to just parse a single file
string FileParam = ParseParamValue("File");
if (FileParam != null)
{
// Check the file exists
FileReference File = new FileReference(FileParam);
if (!FileReference.Exists(File))
{
throw new AutomationException("File '{0}' does not exist", File);
}
CheckSourceFile(File, IdentifierToIndex, new object());
}
else
{
// Add the additional files to be ignored
foreach (string IgnoreFileName in ParseParamValues("Ignore"))
{
IgnoreFileNames.Add(IgnoreFileName);
}
// Create a list of all the root directories
HashSet <DirectoryReference> RootDirs = new HashSet <DirectoryReference>();
RootDirs.Add(EngineDirectory);
// Add the enterprise directory
DirectoryReference EnterpriseDirectory = DirectoryReference.Combine(RootDirectory, "Enterprise");
if (DirectoryReference.Exists(EnterpriseDirectory))
{
RootDirs.Add(EnterpriseDirectory);
}
// Add the project directories
string[] ProjectParams = ParseParamValues("Project");
foreach (string ProjectParam in ProjectParams)
{
FileReference ProjectFile = new FileReference(ProjectParam);
if (!FileReference.Exists(ProjectFile))
{
throw new AutomationException("Unable to find project '{0}'", ProjectFile);
}
RootDirs.Add(ProjectFile.Directory);
}
// Recurse through the tree
LogInformation("Finding source files...");
List <FileReference> SourceFiles = new List <FileReference>();
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
foreach (DirectoryReference RootDir in RootDirs)
{
DirectoryInfo PluginsDir = new DirectoryInfo(Path.Combine(RootDir.FullName, "Plugins"));
if (PluginsDir.Exists)
{
Queue.Enqueue(() => FindSourceFiles(PluginsDir, SourceFiles, Queue));
}
DirectoryInfo SourceDir = new DirectoryInfo(Path.Combine(RootDir.FullName, "Source"));
if (SourceDir.Exists)
{
Queue.Enqueue(() => FindSourceFiles(SourceDir, SourceFiles, Queue));
}
}
Queue.Wait();
}
// Loop through all the source files
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
object LogLock = new object();
foreach (FileReference SourceFile in SourceFiles)
{
Queue.Enqueue(() => CheckSourceFile(SourceFile, IdentifierToIndex, LogLock));
}
using (LogStatusScope Scope = new LogStatusScope("Checking source files..."))
{
while (!Queue.Wait(10 * 1000))
{
Scope.SetProgress("{0}/{1}", SourceFiles.Count - Queue.NumRemaining, SourceFiles.Count);
}
}
}
}
}
/// <summary>
/// Execute a command for each item in the given list, printing progress messages and queued up error strings
/// </summary>
/// <typeparam name="T">Type of the item to process</typeparam>
/// <param name="Items">List of items</param>
/// <param name="CreateAction">Delegate which will create an action to execute for an item</param>
/// <param name="Message">Prefix to add to progress messages</param>
static void ForEach <T>(IList <T> Items, Func <T, ConcurrentQueue <string>, Action> CreateAction, string Message, CancellationToken?CancellationToken = null)
{
using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue())
{
ConcurrentQueue <string> Warnings = new ConcurrentQueue <string>();
foreach (T Item in Items)
{
Action ThisAction = CreateAction(Item, Warnings);
if (CancellationToken.HasValue)
{
Action OriginalAction = ThisAction;
CancellationToken Token = CancellationToken.Value;
ThisAction = () => { if (!Token.IsCancellationRequested)
{
OriginalAction();
}
};
}
Queue.Enqueue(ThisAction);
}
DateTime StartTime = DateTime.UtcNow;
DateTime NextUpdateTime = DateTime.UtcNow + TimeSpan.FromSeconds(0.5);
for (;;)
{
bool bResult = Queue.Wait(2000);
if (!CancellationToken.HasValue || !CancellationToken.Value.IsCancellationRequested)
{
DateTime CurrentTime = DateTime.UtcNow;
if (CurrentTime >= NextUpdateTime || bResult)
{
int NumRemaining = Queue.NumRemaining;
Console.WriteLine("{0} ({1}/{2}; {3}%)", Message, Items.Count - NumRemaining, Items.Count, (int)((Items.Count - NumRemaining) * 100.0f / Items.Count));
NextUpdateTime = CurrentTime + TimeSpan.FromSeconds(10.0);
}
}
if (bResult)
{
break;
}
}
List <string> WarningsList = new List <string>(Warnings);
if (WarningsList.Count > 0)
{
const int MaxWarnings = 50;
if (WarningsList.Count > MaxWarnings)
{
Console.WriteLine("{0} warnings, showing first {1}:", WarningsList.Count, MaxWarnings);
}
else
{
Console.WriteLine("{0} {1}:", WarningsList.Count, (WarningsList.Count == 1) ? "warning" : "warnings");
}
for (int Idx = 0; Idx < WarningsList.Count && Idx < MaxWarnings; Idx++)
{
Console.WriteLine(" {0}", WarningsList[Idx]);
}
}
}
}
