public override void ExecuteBuild() { Log("************************* Reworking stats code"); var Wildcards = new List <string>() { "*.h", "*.cpp", "*.inl" }; // @todo: Add support for STATCAT_ var DeclareStrings = new List <string>() { "DECLARE_CYCLE_STAT", "DECLARE_FLOAT_COUNTER_STAT", "DECLARE_DWORD_COUNTER_STAT", "DECLARE_FLOAT_ACCUMULATOR_STAT", "DECLARE_DWORD_ACCUMULATOR_STAT", "DECLARE_MEMORY_STAT", "DECLARE_MEMORY_STAT_POOL", "DECLARE_STATS_GROUP" }; DirectoryInfo DirInfo = new DirectoryInfo(CmdEnv.LocalRoot); var TopLevelDirs = DirInfo.GetDirectories(); var Dirs = new List <string>(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); } } var AllFiles = new List <string>(); foreach (var Dir in Dirs) { foreach (var Wildcard in Wildcards) { foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(Wildcard, true, Dir)) { if ( !ThisFile.Contains(@"Runtime\Core\Public\Stats\Stats2") && !ThisFile.Contains(@"\ThirdParty\") ) { Log("Source File: {0}", ThisFile); AllFiles.Add(ThisFile); } } } } var DeclareFiles = new Dictionary <string, string>(); var DeclareLines = new Dictionary <string, string>(); var EnumFiles = new Dictionary <string, string>(); var EnumLines = new Dictionary <string, string>(); var Broken = new Dictionary <string, string>(); foreach (var ThisFile in AllFiles) { var FileText = ReadAllText(ThisFile); if (FileText.Contains("STAT_") || FileText.Contains("STATGROUP_")) { var Lines = ReadAllLines(ThisFile); foreach (var LineWithWS in Lines) { var Line = LineWithWS.Trim(); if (Line.Contains("STAT_") || Line.Contains("STATGROUP_")) { string TypeString = "STAT_"; if (!Line.Contains(TypeString) || Line.Contains("DECLARE_STATS_GROUP")) { TypeString = "STATGROUP_"; } bool bDeclareLine = false; foreach (var DeclareString in DeclareStrings) { if (Line.Contains(DeclareString)) { bDeclareLine = true; break; } } var Cut = Line; string Exception = "DECLARE_MEMORY_STAT_POOL"; if (Line.StartsWith(Exception)) { Cut = Line.Substring(Exception.Length); } Cut = Cut.Substring(Cut.IndexOf(TypeString)); int End = Cut.IndexOf(","); if (End < 0) { End = Cut.Length; } int EndEq = Cut.IndexOf("="); int EndParen = Cut.IndexOf(")"); int EndParen2 = Cut.IndexOf("("); if (EndEq > 0 && EndEq < End) { End = EndEq; } if (EndParen > 0 && EndParen < End) { End = EndParen; } if (EndParen2 > 0 && EndParen2 < End) { End = EndParen2; } string StatName = Cut.Substring(0, End).Trim(); bool bEnumLine = false; if (!bDeclareLine) { if ((Line.EndsWith(",") || Line == StatName) && Line.StartsWith(TypeString)) { bEnumLine = true; } } if (bEnumLine || bDeclareLine) { Log("{0} {1} Line: {2} : {3}", bEnumLine ? "Enum" : "Declare", TypeString, StatName, Line); if (bEnumLine) { if (EnumFiles.ContainsKey(StatName)) { if (!Broken.ContainsKey(StatName)) { Broken.Add(StatName, Line); } } else { EnumFiles.Add(StatName, ThisFile); EnumLines.Add(StatName, Line); } } else { if (DeclareFiles.ContainsKey(StatName)) { if (!Broken.ContainsKey(StatName)) { Broken.Add(StatName, Line); } } else { DeclareFiles.Add(StatName, ThisFile); DeclareLines.Add(StatName, Line); } } } } } } } var AllGoodStats = new List <string>(); var AllGoodGroups = new List <string>(); foreach (var DeclareLine in DeclareLines) { if (!Broken.ContainsKey(DeclareLine.Key)) { if (EnumFiles.ContainsKey(DeclareLine.Key)) { if (DeclareLine.Key.StartsWith("STATGROUP_")) { AllGoodGroups.Add(DeclareLine.Key); } else { AllGoodStats.Add(DeclareLine.Key); } } else { Broken.Add(DeclareLine.Key, DeclareLine.Value); } } } var ToCheckOuts = new HashSet <string>(); Log("Stats *************************"); foreach (var AllGoodStat in AllGoodStats) { Log("{0}", AllGoodStat); ToCheckOuts.Add(DeclareFiles[AllGoodStat]); ToCheckOuts.Add(EnumFiles[AllGoodStat]); } Log("Groups *************************"); foreach (var AllGoodGroup in AllGoodGroups) { Log("{0}", AllGoodGroup); ToCheckOuts.Add(DeclareFiles[AllGoodGroup]); ToCheckOuts.Add(EnumFiles[AllGoodGroup]); } Log("Broken *************************"); foreach (var BrokenItem in Broken) { Log("{0}", BrokenItem.Key); } Log("*************************"); int WorkingCL = -1; if (P4Enabled) { WorkingCL = P4.CreateChange(P4Env.Client, "Stat code surgery"); Log("Working in {0}", WorkingCL); } else { throw new AutomationException("this command needs to run with P4."); } var CheckedOuts = new HashSet <string>(); foreach (var ToCheckOut in ToCheckOuts) { if (P4.Edit_NoExceptions(WorkingCL, ToCheckOut)) { CheckedOuts.Add(ToCheckOut); } } Log("Checked Out *************************"); foreach (var CheckedOut in CheckedOuts) { Log("{0}", CheckedOut); } Log("Failed to check out *************************"); foreach (var ToCheckOut in ToCheckOuts) { if (!CheckedOuts.Contains(ToCheckOut)) { Log("{0}", ToCheckOut); } } Log("*************************"); foreach (var AllGoodStat in AllGoodStats) { if (EnumFiles[AllGoodStat].EndsWith(".cpp", StringComparison.InvariantCultureIgnoreCase)) { var DeclareFileText = ReadAllText(DeclareFiles[AllGoodStat]); DeclareFileText = DeclareFileText.Replace(DeclareLines[AllGoodStat], ""); WriteAllText(DeclareFiles[AllGoodStat], DeclareFileText); var EnumFileText = ReadAllText(EnumFiles[AllGoodStat]); EnumFileText = EnumFileText.Replace(EnumLines[AllGoodStat], DeclareLines[AllGoodStat]); WriteAllText(EnumFiles[AllGoodStat], EnumFileText); } else { var DeclareFileText = ReadAllText(DeclareFiles[AllGoodStat]); DeclareFileText = DeclareFileText.Replace(DeclareLines[AllGoodStat], "DEFINE_STAT(" + AllGoodStat + ");"); WriteAllText(DeclareFiles[AllGoodStat], DeclareFileText); var EnumFileText = ReadAllText(EnumFiles[AllGoodStat]); var ExternDeclare = DeclareLines[AllGoodStat]; int Paren = ExternDeclare.IndexOf("("); ExternDeclare = ExternDeclare.Substring(0, Paren) + "_EXTERN" + ExternDeclare.Substring(Paren); Paren = ExternDeclare.LastIndexOf(")"); ExternDeclare = ExternDeclare.Substring(0, Paren) + ", " + ExternDeclare.Substring(Paren); EnumFileText = EnumFileText.Replace(EnumLines[AllGoodStat], ExternDeclare); WriteAllText(EnumFiles[AllGoodStat], EnumFileText); } } foreach (var AllGoodGroup in AllGoodGroups) { var DeclareFileText = ReadAllText(DeclareFiles[AllGoodGroup]); DeclareFileText = DeclareFileText.Replace(DeclareLines[AllGoodGroup], ""); WriteAllText(DeclareFiles[AllGoodGroup], DeclareFileText); var EnumFileText = ReadAllText(EnumFiles[AllGoodGroup]); EnumFileText = EnumFileText.Replace(EnumLines[AllGoodGroup], DeclareLines[AllGoodGroup]); WriteAllText(EnumFiles[AllGoodGroup], EnumFileText); } Log("*************************"); }
public override void ExecuteBuild() { Log("************************* UpdateCopyright"); var Wildcards = new List <string>() { "*.h", "*.cpp", "*.inl", "*.cs" }; var Dirs = new List <string>(); Dirs.Add(CombinePaths(CmdEnv.LocalRoot, "Engine", "Plugins")); { DirectoryInfo DirInfo = new DirectoryInfo(CmdEnv.LocalRoot); var TopLevelDirs = DirInfo.GetDirectories(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts")); } } } { DirectoryInfo DirInfo = new DirectoryInfo(CombinePaths(CmdEnv.LocalRoot, "Samples", "SampleGames")); var TopLevelDirs = DirInfo.GetDirectories(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts")); } } } { DirectoryInfo DirInfo = new DirectoryInfo(CombinePaths(CmdEnv.LocalRoot, "Samples", "Showcases")); var TopLevelDirs = DirInfo.GetDirectories(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts")); } } } { DirectoryInfo DirInfo = new DirectoryInfo(CombinePaths(CmdEnv.LocalRoot, "Templates")); var TopLevelDirs = DirInfo.GetDirectories(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts")); } } } var AllFiles = new List <string>(); foreach (var Dir in Dirs) { foreach (var Wildcard in Wildcards) { foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(Wildcard, true, Dir)) { if (!ThisFile.Contains(@"\ThirdParty\")) { Log("Source File: {0}", ThisFile); AllFiles.Add(ThisFile); } } } } int WorkingCL = -1; if (P4Enabled) { WorkingCL = P4.CreateChange(P4Env.Client, "Stat code surgery"); Log("Working in {0}", WorkingCL); } else { throw new AutomationException("this command needs to run with P4."); } foreach (var ToCheckOut in AllFiles) { if (P4.Edit_NoExceptions(WorkingCL, ToCheckOut)) { Log("Checked out {0}", ToCheckOut); } else if (ToCheckOut.Contains("CodeSurgery")) { Log("Couldn't check out {0}", ToCheckOut); } else { P4.RevertAll(WorkingCL); throw new AutomationException("Couldn't check out {0}", ToCheckOut); } } var Normal = new HashSet <string>(); var Missing = new HashSet <string>(); foreach (var ThisFile in AllFiles) { var FileText = ReadAllText(ThisFile); if (FileText.Contains("2013 Epic Games, Inc. All Rights Reserved.")) { FileText = FileText.Replace("2013 Epic Games, Inc. All Rights Reserved.", "2014 Epic Games, Inc. All Rights Reserved."); WriteAllText(ThisFile, FileText); Normal.Add(ThisFile); } else { var Lines = new List <string>(ReadAllLines(ThisFile)); Lines.Insert(0, "// Copyright 1998-2014 Epic Games, Inc. All Rights Reserved."); WriteAllLines(ThisFile, Lines.ToArray()); Missing.Add(ThisFile); } } Log("Normal Copyright *************************"); foreach (var ThisFile in Normal) { Log(" {0}", ThisFile); } Log("Missing Copyright *************************"); foreach (var ThisFile in Missing) { Log(" {0}", ThisFile); } }
/// <summary> /// Retrieves and saves all artifacts from the provided session role. Artifacts are saved to the destination path /// </summary> /// <param name="InContext"></param> /// <param name="InRunningRole"></param> /// <param name="InDestArtifactPath"></param> /// <returns></returns> public UnrealRoleArtifacts SaveRoleArtifacts(UnrealTestContext InContext, UnrealSessionInstance.RoleInstance InRunningRole, string InDestArtifactPath) { bool IsServer = InRunningRole.Role.RoleType.IsServer(); string RoleName = (InRunningRole.Role.IsDummy() ? "Dummy" : "") + InRunningRole.Role.RoleType.ToString(); UnrealTargetPlatform?Platform = InRunningRole.Role.Platform; string RoleConfig = InRunningRole.Role.Configuration.ToString(); Directory.CreateDirectory(InDestArtifactPath); // Don't archive editor data, there can be a *lot* of stuff in that saved folder! bool IsEditor = InRunningRole.Role.RoleType.UsesEditor(); bool IsDevBuild = InContext.TestParams.ParseParam("dev"); string DestSavedDir = Path.Combine(InDestArtifactPath, "Saved"); string SourceSavedDir = ""; // save the contents of the saved directory SourceSavedDir = InRunningRole.AppInstance.ArtifactPath; // save the output from TTY string ArtifactLogPath = Path.Combine(InDestArtifactPath, RoleName + "Output.log"); // Write a brief Gauntlet header to aid debugging StringBuilder LogOut = new StringBuilder(); LogOut.AppendLine("------ Gauntlet Test ------"); LogOut.AppendFormat("Role: {0}\r\n", InRunningRole.Role); LogOut.AppendFormat("Automation Command: {0}\r\n", Environment.CommandLine); LogOut.AppendLine("---------------------------"); // Write instance stdout stream LogOut.Append(InRunningRole.AppInstance.StdOut); File.WriteAllText(ArtifactLogPath, LogOut.ToString()); Log.Info("Wrote Log to {0}", ArtifactLogPath); if (IsServer == false) { // gif-ify and jpeg-ify any screenshots try { string ScreenshotPath = Path.Combine(SourceSavedDir, "Screenshots", Platform.ToString()).ToLower(); if (Directory.Exists(ScreenshotPath) && Directory.GetFiles(ScreenshotPath).Length > 0) { Log.Info("Downsizing and gifying session images at {0}", ScreenshotPath); // downsize first so gif-step is quicker and takes less resoruces. Utils.Image.ConvertImages(ScreenshotPath, ScreenshotPath, "jpg", true); string GifPath = Path.Combine(InDestArtifactPath, RoleName + "Test.gif"); if (Utils.Image.SaveImagesAsGif(ScreenshotPath, GifPath)) { Log.Info("Saved gif to {0}", GifPath); } } } catch (Exception Ex) { Log.Info("Failed to downsize and gif-ify images! {0}", Ex); } } // don't archive data in dev mode, because peoples saved data could be huuuuuuuge! if (IsEditor == false) { LogLevel OldLevel = Log.Level; Log.Level = LogLevel.Normal; if (Directory.Exists(SourceSavedDir)) { Utils.SystemHelpers.CopyDirectory(SourceSavedDir, DestSavedDir); Log.Info("Archived artifacts to to {0}", DestSavedDir); } else { Log.Info("Archive path '{0}' was not found!", SourceSavedDir); } Log.Level = OldLevel; } else { if (IsEditor) { Log.Info("Skipping archival of assets for editor {0}", RoleName); } else if (IsDevBuild) { Log.Info("Skipping archival of assets for dev build"); } } foreach (EIntendedBaseCopyDirectory ArtifactDir in InRunningRole.Role.AdditionalArtifactDirectories) { if (InRunningRole.AppInstance.Device.GetPlatformDirectoryMappings().ContainsKey(ArtifactDir)) { string SourcePath = InRunningRole.AppInstance.Device.GetPlatformDirectoryMappings()[ArtifactDir]; var DirToCopy = new DirectoryInfo(SourcePath); if (DirToCopy.Exists) { // Grab the final dir name to copy everything into so everything's not just going into root artifact dir. string IntendedCopyLocation = Path.Combine(InDestArtifactPath, DirToCopy.Name); Utils.SystemHelpers.CopyDirectory(SourcePath, IntendedCopyLocation); } } } // TODO REMOVEME- this should go elsewhere, likely a util that can be called or inserted by relevant test nodes. if (IsServer == false) { // Copy over PSOs try { if (InContext.Options.LogPSO) { foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(true, "*.rec.upipelinecache", true, DestSavedDir)) { bool Copied = false; var JustFile = Path.GetFileName(ThisFile); if (JustFile.StartsWith("++")) { var Parts = JustFile.Split(new Char[] { '+', '-' }).Where(A => A != "").ToArray(); if (Parts.Count() >= 2) { string ProjectName = Parts[0].ToString(); string BuildRoot = CommandUtils.CombinePaths(CommandUtils.RootBuildStorageDirectory()); string SrcBuildPath = CommandUtils.CombinePaths(BuildRoot, ProjectName); string SrcBuildPath2 = CommandUtils.CombinePaths(BuildRoot, ProjectName.Replace("Game", "").Replace("game", "")); if (!CommandUtils.DirectoryExists(SrcBuildPath)) { SrcBuildPath = SrcBuildPath2; } if (CommandUtils.DirectoryExists(SrcBuildPath)) { var JustBuildFolder = JustFile.Replace("-" + Parts.Last(), ""); string PlatformStr = Platform.ToString(); string SrcCLMetaPath = CommandUtils.CombinePaths(SrcBuildPath, JustBuildFolder, PlatformStr, "MetaData"); if (CommandUtils.DirectoryExists(SrcCLMetaPath)) { string SrcCLMetaPathCollected = CommandUtils.CombinePaths(SrcCLMetaPath, "CollectedPSOs"); if (!CommandUtils.DirectoryExists(SrcCLMetaPathCollected)) { Log.Info("Creating Directory {0}", SrcCLMetaPathCollected); CommandUtils.CreateDirectory(SrcCLMetaPathCollected); } if (CommandUtils.DirectoryExists(SrcCLMetaPathCollected)) { string DestFile = CommandUtils.CombinePaths(SrcCLMetaPathCollected, JustFile); CommandUtils.CopyFile_NoExceptions(ThisFile, DestFile, true); if (CommandUtils.FileExists(true, DestFile)) { Log.Info("Deleting local file, copied to {0}", DestFile); CommandUtils.DeleteFile_NoExceptions(ThisFile, true); Copied = true; } } } } } } if (!Copied) { Log.Warning("Could not find anywhere to put this file {0}", JustFile); } } } } catch (Exception Ex) { Log.Info("Failed to copy upipelinecaches to the network {0}", Ex); } } // END REMOVEME UnrealLogParser LogParser = new UnrealLogParser(InRunningRole.AppInstance.StdOut); int ExitCode = InRunningRole.AppInstance.ExitCode; LogParser.GetTestExitCode(out ExitCode); UnrealRoleArtifacts Artifacts = new UnrealRoleArtifacts(InRunningRole.Role, InRunningRole.AppInstance, InDestArtifactPath, ArtifactLogPath, LogParser); return(Artifacts); }