private void CreateChangelist(ProjectParams Params) { Log("Running Step:- RebuildLightMaps::CheckOutMaps"); // Setup a P4 Cl we will use to submit the new lightmaps WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} rebuilding lightmaps from changelist {1}\n#rb None\n#tests None", Params.ShortProjectName, P4Env.Changelist)); Log("Working in {0}", WorkingCL); }
public override void ExecuteBuild() { Log("************************** ShooterGame_BasicBuild"); var Params = GetParams(this); int WorkingCL = -1; if (P4Enabled) { WorkingCL = P4.CreateChange(P4Env.Client, String.Format("ShooterGameBuild built from changelist {0}", P4Env.Changelist)); Log("Build from {0} Working in {1}", P4Env.Changelist, WorkingCL); } Project.Build(this, Params, WorkingCL); Project.Cook(Params); Project.CopyBuildToStagingDirectory(Params); if (ParseParam("Package")) { Project.Package(Params, WorkingCL); } Project.Archive(Params); Project.Deploy(Params); PrintRunTime(); Project.Run(Params); if (WorkingCL > 0) { //Check everything in and label it int SubmittedCL; P4.Submit(WorkingCL, out SubmittedCL, true, true); P4.MakeDownstreamLabel(P4Env, "ShooterGameBasicBuild"); } }
private void CreateChangelist(ProjectParams Params) { LogInformation("Running Step:- RebuildHLOD::CheckOutMaps"); // Setup a P4 Cl we will use to submit the new HLOD data WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} rebuilding HLODs from changelist {1}\n#rb None\n#tests None\n#jira none\n{2}", Params.ShortProjectName, P4Env.Changelist, RobomergeCommand)); LogInformation("Working in {0}", WorkingCL); }
protected void DoBuildCookRun(ProjectParams Params) { const ProjectBuildTargets ClientTargets = ProjectBuildTargets.ClientCooked | ProjectBuildTargets.ServerCooked; bool bGenerateNativeScripts = Params.RunAssetNativization; int WorkingCL = -1; if (P4Enabled && GlobalCommandLine.Submit && AllowSubmit) { WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} build from changelist {1}", Params.ShortProjectName, P4Env.Changelist)); } Project.Build(this, Params, WorkingCL, bGenerateNativeScripts ? (ProjectBuildTargets.All & ~ClientTargets) : ProjectBuildTargets.All); Project.Cook(Params); if (bGenerateNativeScripts) { Project.Build(this, Params, WorkingCL, ClientTargets); } Project.CopyBuildToStagingDirectory(Params); Project.Package(Params, WorkingCL); Project.Archive(Params); Project.Deploy(Params); PrintRunTime(); Project.Run(Params); // Check everything in! if (WorkingCL != -1) { int SubmittedCL; P4.Submit(WorkingCL, out SubmittedCL, true, true); } }
protected void DoBuildCookRun(ProjectParams Params) { int WorkingCL = -1; if (P4Enabled && AllowSubmit) { WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} build from changelist {1}", Params.ShortProjectName, P4Env.Changelist)); } Project.Build(this, Params, WorkingCL); Project.Cook(Params); Project.CopyBuildToStagingDirectory(Params); Project.Package(Params, WorkingCL); Project.Archive(Params); Project.Deploy(Params); PrintRunTime(); Project.Run(Params); // Check everything in! if (WorkingCL != -1) { int SubmittedCL; P4.Submit(WorkingCL, out SubmittedCL, true, true); } }
protected void DoBuildCookRun(ProjectParams Params) { const ProjectBuildTargets ClientTargets = ProjectBuildTargets.ClientCooked | ProjectBuildTargets.ServerCooked; bool bGenerateNativeScripts = Params.RunAssetNativization; int WorkingCL = -1; if (P4Enabled && GlobalCommandLine.Submit && AllowSubmit) { WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} build from changelist {1}", Params.ShortProjectName, P4Env.Changelist)); } Project.Build(this, Params, WorkingCL, bGenerateNativeScripts ? (ProjectBuildTargets.All & ~ClientTargets) : ProjectBuildTargets.All); Project.Cook(Params); if (bGenerateNativeScripts) { // crash reporter is built along with client targets, so we need to // include that target flag here as well - note: that its not folded // into ClientTargets because the editor needs its own CrashReporter // as well (which would be built above) Project.Build(this, Params, WorkingCL, ClientTargets | ProjectBuildTargets.CrashReporter); } else { ConfigHierarchy GameIni = ConfigCache.ReadHierarchy(ConfigHierarchyType.Game, Params.RawProjectPath.Directory, HostPlatform.Current.HostEditorPlatform); if (GameIni != null) { List <string> NativizeBlueprintAssets = null; string BlueprintNativizationMethod = "Disabled"; bool bWarnIfPackagedWithoutNativizationFlag = true; GameIni.GetString("/Script/UnrealEd.ProjectPackagingSettings", "BlueprintNativizationMethod", out BlueprintNativizationMethod); GameIni.GetBool("/Script/UnrealEd.ProjectPackagingSettings", "bWarnIfPackagedWithoutNativizationFlag", out bWarnIfPackagedWithoutNativizationFlag); GameIni.GetArray("/Script/UnrealEd.ProjectPackagingSettings", "NativizeBlueprintAssets", out NativizeBlueprintAssets); if (bWarnIfPackagedWithoutNativizationFlag && BlueprintNativizationMethod != "Disabled") { // Warn if we're cooking without the -nativizeAssets flag, when the project settings specify a nativization method. // If the "exclusive" (whitelist) method is set, we only warn if at least one asset has been selected for conversion. if (BlueprintNativizationMethod != "Exclusive" || (NativizeBlueprintAssets != null && NativizeBlueprintAssets.Count > 0)) { LogWarning("Project is configured for Blueprint nativization, but the conversion flag (-nativizeAssets) was omitted from the command line. No nativized assets have been built as a result."); } } } } Project.CopyBuildToStagingDirectory(Params); Project.Package(Params, WorkingCL); Project.Archive(Params); Project.Deploy(Params); PrintRunTime(); Project.Run(Params); // Check everything in! if (WorkingCL != -1) { int SubmittedCL; P4.Submit(WorkingCL, out SubmittedCL, true, true); } }
public override void ExecuteBuild() { LogInformation("P4CLIENT={0}", GetEnvVar("P4CLIENT")); LogInformation("P4PORT={0}", GetEnvVar("P4PORT")); var CLDescription = "AutomationTool TestP4"; LogInformation("Creating new changelist \"{0}\" using client \"{1}\"", CLDescription, GetEnvVar("P4CLIENT")); var ChangelistNumber = P4.CreateChange(Description: CLDescription); LogInformation("Created changelist {0}", ChangelistNumber); }
private void CheckOutMaps(ProjectParams Params) { Log("Running Step:- RebuildLightMaps::CheckOutMaps"); // Setup a P4 Cl we will use to submit the new lightmaps WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} rebuilding lightmaps from changelist {1}", Params.ShortProjectName, P4Env.Changelist)); Log("Working in {0}", WorkingCL); string AllMapsWildcardCmdline = String.Format("{0}\\...\\*.umap", Params.RawProjectPath.Directory); string Output = ""; P4.P4Output(out Output, String.Format("edit -c {0} {1}", WorkingCL, AllMapsWildcardCmdline)); // We need to ensure that any error in the output log is observed. // P4 is still successful if it manages to run the operation. if (FoundCheckOutErrorInP4Output(Output) == true) { LogError("Failed to check out every one of the project maps."); throw new AutomationException("Failed to check out every one of the project maps."); } }
protected void DoBuildCookRun(ProjectParams Params) { const ProjectBuildTargets ClientTargets = ProjectBuildTargets.ClientCooked | ProjectBuildTargets.ServerCooked; bool bGenerateNativeScripts = Params.RunAssetNativization; int WorkingCL = -1; if (P4Enabled && GlobalCommandLine.Submit && AllowSubmit) { WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} build from changelist {1}", Params.ShortProjectName, P4Env.Changelist)); } Project.Build(this, Params, WorkingCL, bGenerateNativeScripts ? (ProjectBuildTargets.All & ~ClientTargets) : ProjectBuildTargets.All); Project.Cook(Params); if (bGenerateNativeScripts) { // crash reporter is built along with client targets, so we need to // include that target flag here as well - note: that its not folded // into ClientTargets because the editor needs its own CrashReporter // as well (which would be built above) Project.Build(this, Params, WorkingCL, ClientTargets | ProjectBuildTargets.CrashReporter); } Project.CopyBuildToStagingDirectory(Params); Project.Package(Params, WorkingCL); Project.Archive(Params); Project.Deploy(Params); PrintRunTime(); Project.Run(Params); Project.GetFile(Params); // Check everything in! if (WorkingCL != -1) { int SubmittedCL; P4.Submit(WorkingCL, out SubmittedCL, true, true); } }
public override void ExecuteBuild() { int WorkingCL = P4.CreateChange(P4Env.Client, String.Format("TestP4_StrandCheckout, head={0}", P4Env.Changelist)); LogInformation("Build from {0} Working in {1}", P4Env.Changelist, WorkingCL); List <string> Sign = new List <string>(); Sign.Add(CombinePaths(CmdEnv.LocalRoot, @"\Engine\Binaries\DotNET\AgentInterface.dll")); LogInformation("Signing and adding {0} build products to changelist {1}...", Sign.Count, WorkingCL); CodeSign.SignMultipleIfEXEOrDLL(this, Sign); foreach (var File in Sign) { P4.Sync("-f -k " + File + "#head"); // sync the file without overwriting local one if (!FileExists(File)) { throw new AutomationException("BUILD FAILED {0} was a build product but no longer exists", File); } P4.ReconcileNoDeletes(WorkingCL, File); } }
public override void ExecuteBuild() { string BucketName = ParseRequiredStringParam("Bucket"); FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile"); string CredentialsKey = ParseRequiredStringParam("CredentialsKey"); DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir"); DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir"); int Days = ParseParamInt("Days", 7); int MaxFileSize = ParseParamInt("MaxFileSize", 0); string RootManifestPath = ParseRequiredStringParam("Manifest"); string KeyPrefix = ParseParamValue("KeyPrefix", ""); bool bReset = ParseParam("Reset"); // The credentials to upload with AWSCredentials Credentials; // Try to get the credentials by the key passed in from the script CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName); if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials)) { throw new AutomationException("Unknown credentials key: {0}", CredentialsKey); } // Create the new client using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region)) { using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4)) { // Read the filters HashSet <string> Paths = new HashSet <string>(); foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt")) { TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc; if (Age < TimeSpan.FromDays(3)) { Log.TraceInformation("Reading {0}", FilterFile.FullName); string[] Lines = File.ReadAllLines(FilterFile.FullName); foreach (string Line in Lines) { string TrimLine = Line.Trim().Replace('\\', '/'); if (TrimLine.Length > 0) { Paths.Add(TrimLine); } } } else if (Age > TimeSpan.FromDays(5)) { try { Log.TraceInformation("Deleting {0}", FilterFile.FullName); FilterFile.Delete(); } catch (Exception Ex) { Log.TraceWarning("Unable to delete: {0}", Ex.Message); Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex)); } } } Log.TraceInformation("Found {0:n0} files", Paths.Count); // Enumerate all the files that are in the network DDC Log.TraceInformation(""); Log.TraceInformation("Filtering files in {0}...", CacheDir); List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag)); // Filter to the maximum size if (MaxFileSize != 0) { int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize); Log.TraceInformation(""); Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize); } // Create the working directory DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC"); DirectoryReference.CreateDirectory(WorkingDir); // Get the path to the manifest FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath); // Read the old root manifest RootManifest OldRootManifest = new RootManifest(); if (FileReference.Exists(RootManifestFile)) { OldRootManifest.Read(JsonObject.Read(RootManifestFile)); } // Read the old bundle manifest BundleManifest OldBundleManifest = new BundleManifest(); if (OldRootManifest.Entries.Count > 0) { FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json"); if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest)) { OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest)); } } // Create the new manifest BundleManifest NewBundleManifest = new BundleManifest(); // Try to download the old manifest, and add all the bundles we want to keep to the new manifest if (!bReset) { foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries) { FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name); if (!FileReference.Exists(BundleFile)) { Log.TraceInformation("Downloading {0}", BundleFile); FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz"); if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile)) { Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey); continue; } FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming"); try { DecompressFile(TempCompressedFile, TempUncompressedFile); } catch (Exception Ex) { Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString()); continue; } FileReference.Move(TempUncompressedFile, BundleFile); } NewBundleManifest.Entries.Add(Bundle); } } // Figure out all the item digests that we already have Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >(); foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries) { HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>(); FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name); using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read)) { BinaryReader Reader = new BinaryReader(Stream); uint Signature = Reader.ReadUInt32(); if (Signature != BundleSignatureV1) { throw new Exception(String.Format("Invalid signature for {0}", BundleFile)); } int NumEntries = Reader.ReadInt32(); for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++) { byte[] Digest = new byte[ContentHash.LengthSHA1]; if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1) { throw new Exception("Unexpected EOF"); } KeyHashes.Add(new ContentHash(Digest)); Stream.Seek(4, SeekOrigin.Current); } } BundleToKeyHashes[Bundle] = KeyHashes; } // Calculate the download size of the manifest long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength); // Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these // files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times. foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes) { long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length); if (ValidBundleSize < MinBundleSize) { NewBundleManifest.Entries.Remove(Pair.Key); } } // Find all the valid digests HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>(); foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries) { ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]); } // Remove all the files which already exist int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash)); if (NumRemovedExist > 0) { Log.TraceInformation(""); Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist); } // Read all the files we want to include List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >(); if (Files.Count > 0) { Log.TraceInformation(""); Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024)); FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y))); } // Generate new data using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider()) { // Flag for whether to update the manifest bool bUpdateManifest = false; // Upload the new bundle Log.TraceInformation(""); if (FilesToInclude.Count == 0) { Log.TraceInformation("No new files to add."); } else { // Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together. Log.TraceInformation("Sorting input files"); List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList(); // Get the target bundle size long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length); int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize); long TargetBundleSize = TotalSize / NumBundles; // Split the input data into bundles List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >(); long BundleSize = 0; for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize) { List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >(); for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++) { BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]); BundleSize += SortedFilesToInclude[FileIdx].Item2.Length; } BundleFilesToIncludeList.Add(BundleFilesToInclude); } // Upload each bundle DateTime NewBundleTime = DateTime.UtcNow; for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++) { List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx]; // Get the new bundle info string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : ""; string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix); // Create a random number for the object key string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto); // Create the bundle header byte[] Header; using (MemoryStream HeaderStream = new MemoryStream()) { BinaryWriter Writer = new BinaryWriter(HeaderStream); Writer.Write(BundleSignatureV1); Writer.Write(BundleFilesToInclude.Count); foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude) { Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1); Writer.Write((int)FileToInclude.Item2.Length); } Header = HeaderStream.ToArray(); } // Create the output file FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz"); Log.TraceInformation("Writing {0}", NewBundleFile); using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read)) { using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true)) { ZipStream.Write(Header, 0, Header.Length); foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude) { ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length); } } } // Upload the file long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length; long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length); Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f)); UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null); // Add the bundle to the new manifest BundleManifest.Entry Bundle = new BundleManifest.Entry(); Bundle.Name = NewBundleName; Bundle.ObjectKey = NewBundleObjectKey; Bundle.Time = NewBundleTime; Bundle.CompressedLength = (int)NewBundleCompressedLength; Bundle.UncompressedLength = (int)NewBundleUncompressedLength; NewBundleManifest.Entries.Add(Bundle); // Mark the manifest as requiring an update bUpdateManifest = true; } } // Update the manifest if (bUpdateManifest) { DateTime UtcNow = DateTime.UtcNow; DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0); // Update the root manifest RootManifest NewRootManifest = new RootManifest(); NewRootManifest.AccessKey = OldRootManifest.AccessKey; NewRootManifest.SecretKey = OldRootManifest.SecretKey; foreach (RootManifest.Entry Entry in OldRootManifest.Entries) { if (Entry.CreateTime >= RemoveBundleManifestsBefore) { NewRootManifest.Entries.Add(Entry); } } // Make sure there's an entry for the last 24h DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0); if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter)) { RootManifest.Entry NewEntry = new RootManifest.Entry(); NewEntry.CreateTime = UtcNow; NewEntry.Key = KeyPrefix + CreateObjectName(Crypto); NewRootManifest.Entries.Add(NewEntry); } // Save out the new bundle manifest FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json"); NewBundleManifest.Save(NewBundleManifestFile); // Update all the bundle manifests still valid foreach (RootManifest.Entry Entry in NewRootManifest.Entries) { Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key); UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null); } // Overwrite all the existing manifests if (AllowSubmit) { List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName)); // Create a changelist containing the new manifest int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest"); if (ExistingFiles.Count > 0) { P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName)); NewRootManifest.Save(RootManifestFile); } else { NewRootManifest.Save(RootManifestFile); P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName)); } // Submit it int SubmittedChangeNumber; P4.Submit(ChangeNumber, out SubmittedChangeNumber, true); if (SubmittedChangeNumber <= 0) { throw new AutomationException("Failed to submit change"); } // Delete any bundles that are no longer referenced HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey)); foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries) { if (!KeepObjectKeys.Contains(OldEntry.ObjectKey)) { Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey); DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null); } } // Delete any bundle manifests which are no longer referenced HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key)); foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries) { if (!KeepManifestKeys.Contains(OldEntry.Key)) { Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key); DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null); } } } else { // Skip submitting Log.TraceWarning("Skipping manifest submit due to missing -Submit argument."); } // Update the new download size DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength); } } // Print some stats about the final manifest Log.TraceInformation(""); Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0)); Log.TraceInformation(""); } } }
public override void ExecuteBuild() { int WorkingCL = -1; if (P4Enabled && AllowSubmit) { string CmdLine = ""; foreach (var Arg in Params) { CmdLine += Arg.ToString() + " "; } WorkingCL = P4.CreateChange(P4Env.Client, String.Format("MegaXGE build from changelist {0} - Params: {1}", P4Env.Changelist, CmdLine)); } LogInformation("************************* MegaXGE"); bool Clean = ParseParam("Clean"); string CleanToolLocation = CombinePaths(CmdEnv.LocalRoot, "Engine", "Build", "Batchfiles", "Clean.bat"); bool ShowProgress = ParseParam("Progress"); var UE4Build = new UE4Build(this); var Agenda = new UE4Build.BuildAgenda(); // we need to always build UHT when we use mega XGE var ProgramTargets = new string[] { "UnrealHeaderTool", }; Agenda.AddTargets(ProgramTargets, UnrealTargetPlatform.Win64, UnrealTargetConfiguration.Development); if (Clean) { LogSetProgress(ShowProgress, "Cleaning previous builds..."); foreach (var CurTarget in ProgramTargets) { string Args = String.Format("{0} {1} {2}", CurTarget, UnrealTargetPlatform.Win64.ToString(), UnrealTargetConfiguration.Development.ToString()); RunAndLog(CmdEnv, CleanToolLocation, Args); } } LogInformation("*************************"); for (int Arg = 1; Arg < 100; Arg++) { string Parm = String.Format("Target{0}", Arg); string Target = ParseParamValue(Parm, ""); if (String.IsNullOrEmpty(Target)) { break; } FileReference ProjectFile = null; string ProjectFileParam = ParseParamValue(String.Format("Project{0}", Arg), null); if (ProjectFileParam != null) { ProjectFile = new FileReference(ProjectFileParam); if (!FileReference.Exists(ProjectFile)) { throw new AutomationException("Project file '{0}' could not be found"); } } var Parts = Target.Split(' '); string JustTarget = Parts[0]; if (String.IsNullOrEmpty(JustTarget)) { throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target); } var Targets = JustTarget.Split('|'); if (Targets.Length < 1) { throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target); } var Platforms = new List <UnrealTargetPlatform>(); var Configurations = new List <UnrealTargetConfiguration>(); for (int Part = 1; Part < Parts.Length; Part++) { if (!String.IsNullOrEmpty(Parts[Part])) { var SubParts = Parts[Part].Split('|'); foreach (var SubPart in SubParts) { UnrealTargetPlatform Platform; if (UnrealTargetPlatform.TryParse(SubPart, out Platform)) { Platforms.Add(Platform); } else { switch (SubPart.ToUpperInvariant()) { case "DEBUG": Configurations.Add(UnrealTargetConfiguration.Debug); break; case "DEBUGGAME": Configurations.Add(UnrealTargetConfiguration.DebugGame); break; case "DEVELOPMENT": Configurations.Add(UnrealTargetConfiguration.Development); break; case "SHIPPING": Configurations.Add(UnrealTargetConfiguration.Shipping); break; case "TEST": Configurations.Add(UnrealTargetConfiguration.Test); break; default: throw new AutomationException("BUILD FAILED target option {0} not recognized.", SubPart); } } } } } if (Platforms.Count < 1) { Platforms.Add(UnrealTargetPlatform.Win64); } if (Configurations.Count < 1) { Configurations.Add(UnrealTargetConfiguration.Development); } foreach (var Platform in Platforms) { foreach (var CurTarget in Targets) { foreach (var Configuration in Configurations) { Agenda.AddTargets(new string[] { CurTarget }, Platform, Configuration, ProjectFile); LogInformation("Target {0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString()); if (Clean) { string Args = String.Format("{0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString()); RunAndLog(CmdEnv, CleanToolLocation, Args); } } } } } LogInformation("*************************"); UE4Build.Build(Agenda, InUpdateVersionFiles: IsBuildMachine, InUseParallelExecutor: ParseParam("useparallelexecutor"), InShowProgress: ShowProgress); // if (WorkingCL > 0) // only move UAT files if we intend to check in some build products // { // UE4Build.CopyUATFilesAndAddToBuildProducts(); // } UE4Build.CheckBuildProducts(UE4Build.BuildProductFiles); if (WorkingCL > 0) { // Sign everything we built CodeSign.SignMultipleIfEXEOrDLL(this, UE4Build.BuildProductFiles); // Open files for add or edit UE4Build.AddBuildProductsToChangelist(WorkingCL, UE4Build.BuildProductFiles); int SubmittedCL; P4.Submit(WorkingCL, out SubmittedCL, true, true); } PrintRunTime(); }
public override void ExecuteBuild() { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); if (P4Enabled) { Log("Sync necessary content to head revision"); P4.Sync(P4Env.BuildRootP4 + "/Engine/Config/..."); P4.Sync(P4Env.BuildRootP4 + "/Engine/Content/..."); P4.Sync(P4Env.BuildRootP4 + "/Engine/Source/..."); Log("Localize from label {0}", P4Env.LabelToSync); } OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find("OneSkyConfig_EpicGames"); var oneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret); var projectGroup = GetProjectGroup(oneSkyService, "Unreal Engine"); // Create changelist for backed up POs from OneSky. if (P4Enabled) { OneSkyDownloadedPOChangeList = P4.CreateChange(P4Env.Client, "OneSky downloaded PO backup."); } // Export all text from OneSky ExportProjectToDirectory(oneSkyService, projectGroup, "Engine"); ExportProjectToDirectory(oneSkyService, projectGroup, "Editor"); ExportProjectToDirectory(oneSkyService, projectGroup, "EditorTutorials"); ExportProjectToDirectory(oneSkyService, projectGroup, "PropertyNames"); ExportProjectToDirectory(oneSkyService, projectGroup, "ToolTips"); ExportProjectToDirectory(oneSkyService, projectGroup, "Category"); ExportProjectToDirectory(oneSkyService, projectGroup, "Keywords"); // Submit changelist for backed up POs from OneSky. if (P4Enabled) { int SubmittedChangeList; P4.Submit(OneSkyDownloadedPOChangeList, out SubmittedChangeList); } // Setup editor arguments for SCC. string EditorArguments = String.Empty; if (P4Enabled) { EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken()); } else { EditorArguments = String.Format("-SCCProvider={0}", "None"); } // Setup commandlet arguments for SCC. string CommandletSCCArguments = String.Empty; if (P4Enabled) { CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC"; } if (!AllowSubmit) { CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit"; } // Setup commandlet arguments with configurations. var CommandletArgumentSets = new string[] { String.Format("-config={0}", @"./Config/Localization/Engine.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/Editor.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/EditorTutorials.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/PropertyNames.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/ToolTips.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/Category.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/Keywords.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), }; // Execute commandlet for each set of arguments. foreach (var CommandletArguments in CommandletArgumentSets) { Log("Localization for {0} {1}", EditorArguments, CommandletArguments); Log("Running UE4Editor to generate localization data"); string Arguments = String.Format("-run=GatherText {0} {1}", EditorArguments, CommandletArguments); var RunResult = Run(EditorExe, Arguments); if (RunResult.ExitCode != 0) { throw new AutomationException("Error while executing localization commandlet '{0}'", Arguments); } } // Upload all text to OneSky UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Engine"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Engine"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Editor"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Editor"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "EditorTutorials"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/EditorTutorials"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "PropertyNames"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/PropertyNames"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "ToolTips"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/ToolTips"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Category"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Category"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Keywords"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Keywords"), "*.po"); // Localisation statistics estimator. if (P4Enabled) { // Available only for P4 var EstimatorExePath = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/TranslatedWordsCountEstimator.exe"); var StatisticsFilePath = @"\\epicgames.net\root\UE3\Localization\WordCounts\udn.csv"; var Arguments = string.Format( "{0} {1} {2} {3} {4}", StatisticsFilePath, P4Env.P4Port, P4Env.User, P4Env.Client, Environment.GetEnvironmentVariable("P4PASSWD")); var RunResult = Run(EstimatorExePath, Arguments); if (RunResult.ExitCode != 0) { throw new AutomationException("Error while executing TranslatedWordsCountEstimator with arguments '{0}'", Arguments); } } }
public override void ExecuteBuild() { var UEProjectRoot = ParseParamValue("UEProjectRoot"); if (UEProjectRoot == null) { UEProjectRoot = CmdEnv.LocalRoot; } var UEProjectDirectory = ParseParamValue("UEProjectDirectory"); if (UEProjectDirectory == null) { throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'"); } var UEProjectName = ParseParamValue("UEProjectName"); if (UEProjectName == null) { UEProjectName = ""; } var LocalizationProjectNames = new List <string>(); { var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames"); if (LocalizationProjectNamesStr != null) { foreach (var ProjectName in LocalizationProjectNamesStr.Split(',')) { LocalizationProjectNames.Add(ProjectName.Trim()); } } } var LocalizationProviderName = ParseParamValue("LocalizationProvider"); if (LocalizationProviderName == null) { LocalizationProviderName = "OneSky"; } var LocalizationSteps = new List <string>(); { var LocalizationStepsStr = ParseParamValue("LocalizationSteps"); if (LocalizationStepsStr == null) { LocalizationSteps.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" }); } else { foreach (var StepName in LocalizationStepsStr.Split(',')) { LocalizationSteps.Add(StepName.Trim()); } } LocalizationSteps.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do } var ShouldGatherPlugins = ParseParam("IncludePlugins"); var IncludePlugins = new List <string>(); var ExcludePlugins = new List <string>(); if (ShouldGatherPlugins) { var IncludePluginsStr = ParseParamValue("IncludePlugins"); if (IncludePluginsStr != null) { foreach (var PluginName in IncludePluginsStr.Split(',')) { IncludePlugins.Add(PluginName.Trim()); } } var ExcludePluginsStr = ParseParamValue("ExcludePlugins"); if (ExcludePluginsStr != null) { foreach (var PluginName in ExcludePluginsStr.Split(',')) { ExcludePlugins.Add(PluginName.Trim()); } } } var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments"); if (AdditionalCommandletArguments == null) { AdditionalCommandletArguments = ""; } var LocalizationBatches = new List <LocalizationBatch>(); // Add the static set of localization projects as a batch if (LocalizationProjectNames.Count > 0) { LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames)); } // Build up any additional batches needed for plugins if (ShouldGatherPlugins) { var PluginsRootDirectory = CombinePaths(UEProjectRoot, UEProjectDirectory, "Plugins"); IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(new DirectoryReference(PluginsRootDirectory), UEProjectName.Length == 0 ? PluginLoadedFrom.Engine : PluginLoadedFrom.GameProject); // Add a batch for each plugin that meets our criteria foreach (var PluginInfo in AllPlugins) { bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name); if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0) { var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 var PluginTargetNames = new List <string>(); foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets) { PluginTargetNames.Add(LocalizationTarget.Name); } LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames)); } } } // Create a single changelist to use for all changes, and hash the current PO files on disk so we can work out whether they actually change int PendingChangeList = 0; Dictionary <string, byte[]> InitalPOFileHashes = null; if (P4Enabled) { PendingChangeList = P4.CreateChange(P4Env.Client, "Localization Automation"); InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); } // Process each localization batch foreach (var LocalizationBatch in LocalizationBatches) { ProcessLocalizationProjects(LocalizationBatch, PendingChangeList, UEProjectRoot, UEProjectName, LocalizationProviderName, LocalizationSteps, AdditionalCommandletArguments); } // Submit that single changelist now if (P4Enabled && AllowSubmit) { // Revert any PO files that haven't changed aside from their header { var POFilesToRevert = new List <string>(); var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); foreach (var CurrentPOFileHashPair in CurrentPOFileHashes) { byte[] InitialPOFileHash; if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value)) { POFilesToRevert.Add(CurrentPOFileHashPair.Key); } } if (POFilesToRevert.Count > 0) { var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString())); using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename)) { foreach (var POFileToRevert in POFilesToRevert) { P4RevertArgsWriter.WriteLine(POFileToRevert); } } P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename)); DeleteFile_NoExceptions(P4RevertArgsFilename); } } // Revert any other unchanged files P4.RevertUnchanged(PendingChangeList); int SubmittedChangeList; P4.Submit(PendingChangeList, out SubmittedChangeList); } }
public override void ExecuteBuild() { var UEProjectRoot = ParseParamValue("UEProjectRoot"); if (UEProjectRoot == null) { UEProjectRoot = CmdEnv.LocalRoot; } var UEProjectDirectory = ParseParamValue("UEProjectDirectory"); if (UEProjectDirectory == null) { throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'"); } var UEProjectName = ParseParamValue("UEProjectName"); if (UEProjectName == null) { UEProjectName = ""; } var LocalizationProjectNames = new List <string>(); { var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames"); if (LocalizationProjectNamesStr != null) { foreach (var ProjectName in LocalizationProjectNamesStr.Split(',')) { LocalizationProjectNames.Add(ProjectName.Trim()); } } } var LocalizationProviderName = ParseParamValue("LocalizationProvider"); if (LocalizationProviderName == null) { LocalizationProviderName = ""; } var LocalizationStepNames = new List <string>(); { var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps"); if (LocalizationStepNamesStr == null) { LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" }); } else { foreach (var StepName in LocalizationStepNamesStr.Split(',')) { LocalizationStepNames.Add(StepName.Trim()); } } LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do } var ShouldGatherPlugins = ParseParam("IncludePlugins"); var IncludePlugins = new List <string>(); var ExcludePlugins = new List <string>(); if (ShouldGatherPlugins) { var IncludePluginsStr = ParseParamValue("IncludePlugins"); if (IncludePluginsStr != null) { foreach (var PluginName in IncludePluginsStr.Split(',')) { IncludePlugins.Add(PluginName.Trim()); } } var ExcludePluginsStr = ParseParamValue("ExcludePlugins"); if (ExcludePluginsStr != null) { foreach (var PluginName in ExcludePluginsStr.Split(',')) { ExcludePlugins.Add(PluginName.Trim()); } } } var ShouldGatherPlatforms = ParseParam("IncludePlatforms"); var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments"); if (AdditionalCommandletArguments == null) { AdditionalCommandletArguments = ""; } var EnableParallelGather = ParseParam("ParallelGather"); var StartTime = DateTime.UtcNow; var LocalizationBatches = new List <LocalizationBatch>(); // Add the static set of localization projects as a batch if (LocalizationProjectNames.Count > 0) { LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames)); } // Build up any additional batches needed for platforms if (ShouldGatherPlatforms) { var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms")); if (DirectoryReference.Exists(PlatformsRootDirectory)) { foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory)) { // Find the localization targets defined for this platform var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization"))); if (PlatformTargetNames.Count > 0) { var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames)); } } } } // Build up any additional batches needed for plugins if (ShouldGatherPlugins) { var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory)); IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project); // Add a batch for each plugin that meets our criteria var AvailablePluginNames = new HashSet <string>(); foreach (var PluginInfo in AllPlugins) { AvailablePluginNames.Add(PluginInfo.Name); bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name); if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0) { var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 var PluginTargetNames = new List <string>(); foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets) { PluginTargetNames.Add(LocalizationTarget.Name); } LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames)); } } // If we had an explicit list of plugins to include, warn if any were missing foreach (string PluginName in IncludePlugins) { if (!AvailablePluginNames.Contains(PluginName)) { LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName); } } } // Create a single changelist to use for all changes int PendingChangeList = 0; if (P4Enabled) { var ChangeListCommitMessage = String.Format("Localization Automation using CL {0}", P4Env.Changelist); if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Restricted/NotForLicensees/Build/EpicInternal.txt"))) { ChangeListCommitMessage += "\n#okforgithub ignore"; } PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage); } // Prepare to process each localization batch var LocalizationTasks = new List <LocalizationTask>(); foreach (var LocalizationBatch in LocalizationBatches) { var LocalizationTask = new LocalizationTask(LocalizationBatch, UEProjectRoot, LocalizationProviderName, PendingChangeList, this); LocalizationTasks.Add(LocalizationTask); // Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on if (P4Enabled) { LogInformation("Sync necessary content to head revision"); P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Config/Localization/..."); P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Content/Localization/..."); } // Generate the info we need to gather for each project foreach (var ProjectName in LocalizationTask.Batch.LocalizationProjectNames) { LocalizationTask.ProjectInfos.Add(GenerateProjectInfo(LocalizationTask.RootLocalizationTargetDirectory, ProjectName, LocalizationStepNames)); } } // Hash the current PO files on disk so we can work out whether they actually change Dictionary <string, byte[]> InitalPOFileHashes = null; if (P4Enabled) { InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); } // Download the latest translations from our localization provider if (LocalizationStepNames.Contains("Download")) { foreach (var LocalizationTask in LocalizationTasks) { if (LocalizationTask.LocProvider != null) { foreach (var ProjectInfo in LocalizationTask.ProjectInfos) { LocalizationTask.LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo); } } } } // Begin the gather command for each task // These can run in parallel when ParallelGather is enabled { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); // Set the common basic editor arguments var EditorArguments = P4Enabled ? String.Format("-SCCProvider=Perforce -P4Port={0} -P4User={1} -P4Client={2} -P4Passwd={3} -P4Changelist={4} -EnableSCC -DisableSCCSubmit", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList) : "-SCCProvider=None"; if (IsBuildMachine) { EditorArguments += " -BuildMachine"; } EditorArguments += " -Unattended -LogLocalizationConflicts"; if (EnableParallelGather) { EditorArguments += " -multiprocess"; } if (!String.IsNullOrEmpty(AdditionalCommandletArguments)) { EditorArguments += " " + AdditionalCommandletArguments; } // Set the common process run options var CommandletRunOptions = ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand; // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves later) if (EnableParallelGather) { CommandletRunOptions |= ERunOptions.NoWaitForExit; } foreach (var LocalizationTask in LocalizationTasks) { var ProjectArgument = String.IsNullOrEmpty(UEProjectName) ? "" : String.Format("\"{0}\"", Path.Combine(LocalizationTask.RootWorkingDirectory, String.Format("{0}.uproject", UEProjectName))); foreach (var ProjectInfo in LocalizationTask.ProjectInfos) { var LocalizationConfigFiles = new List <string>(); foreach (var LocalizationStep in ProjectInfo.LocalizationSteps) { if (LocalizationStepNames.Contains(LocalizationStep.Name)) { LocalizationConfigFiles.Add(LocalizationStep.LocalizationConfigFile); } } if (LocalizationConfigFiles.Count > 0) { var Arguments = String.Format("{0} -run=GatherText -config=\"{1}\" {2}", ProjectArgument, String.Join(";", LocalizationConfigFiles), EditorArguments); LogInformation("Running localization commandlet for '{0}': {1}", ProjectInfo.ProjectName, Arguments); LocalizationTask.GatherProcessResults.Add(Run(EditorExe, Arguments, null, CommandletRunOptions)); } else { LocalizationTask.GatherProcessResults.Add(null); } } } } // Wait for each commandlet process to finish and report the result. // This runs even for non-parallel execution to log the exit state of the process. foreach (var LocalizationTask in LocalizationTasks) { for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex) { var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex]; var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex]; if (RunResult != null) { RunResult.WaitForExit(); RunResult.OnProcessExited(); RunResult.DisposeProcess(); if (RunResult.ExitCode == 0) { LogInformation("The localization commandlet for '{0}' exited with code 0.", ProjectInfo.ProjectName); } else { LogWarning("The localization commandlet for '{0}' exited with code {1} which likely indicates a crash.", ProjectInfo.ProjectName, RunResult.ExitCode); } } } } // Upload the latest sources to our localization provider if (LocalizationStepNames.Contains("Upload")) { foreach (var LocalizationTask in LocalizationTasks) { if (LocalizationTask.LocProvider != null) { // Upload all text to our localization provider for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex) { var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex]; var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex]; if (RunResult != null && RunResult.ExitCode == 0) { // Recalculate the split platform paths before doing the upload, as the export may have changed them ProjectInfo.ExportInfo.CalculateSplitPlatformNames(LocalizationTask.RootLocalizationTargetDirectory); LocalizationTask.LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo); } else { LogWarning("Skipping upload to the localization provider for '{0}' due to an earlier commandlet failure.", ProjectInfo.ProjectName); } } } } } // Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to) if (P4Enabled) { // Revert any PO files that haven't changed aside from their header { var POFilesToRevert = new List <string>(); var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); foreach (var CurrentPOFileHashPair in CurrentPOFileHashes) { byte[] InitialPOFileHash; if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value)) { POFilesToRevert.Add(CurrentPOFileHashPair.Key); } } if (POFilesToRevert.Count > 0) { var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString())); using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename)) { foreach (var POFileToRevert in POFilesToRevert) { P4RevertArgsWriter.WriteLine(POFileToRevert); } } P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename)); DeleteFile_NoExceptions(P4RevertArgsFilename); } } // Revert any other unchanged files P4.RevertUnchanged(PendingChangeList); // Submit that single changelist now if (AllowSubmit) { int SubmittedChangeList; P4.Submit(PendingChangeList, out SubmittedChangeList); } } var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds; LogInformation("Localize command finished in {0} seconds", RunDuration / 1000); }
public override void ExecuteBuild() { SetupStaticBuildEnvironment(); bool bBuildSolutions = true; if (ParseParam("SkipBuildSolutions")) { bBuildSolutions = false; } bool bBuildLibraries = true; if (ParseParam("SkipBuild")) { bBuildLibraries = false; } bool bAutoCreateChangelist = true; if (ParseParam("SkipCreateChangelist")) { bAutoCreateChangelist = false; } bool bAutoSubmit = false; // bAutoCreateChangelist; if (ParseParam("SkipSubmit")) { bAutoSubmit = false; } // if we don't pass anything, we'll just merge by default string RobomergeCommand = ParseParamValue("Robomerge", "").ToLower(); if (!string.IsNullOrEmpty(RobomergeCommand)) { // for merge default action, add flag to make sure buildmachine commit isn't skipped if (RobomergeCommand == "merge") { RobomergeCommand = "#robomerge[all] #DisregardExcludedAuthors"; } // otherwise add hashtags else if (RobomergeCommand == "ignore") { RobomergeCommand = "#robomerge #ignore"; } else if (RobomergeCommand == "null") { RobomergeCommand = "#robomerge #null"; } // otherwise the submit will likely fail. else { throw new AutomationException("Invalid Robomerge param passed in {0}. Must be \"merge\", \"null\", or \"ignore\"", RobomergeCommand); } } // get the platforms we want to build for List <TargetPlatformData> TargetPlatforms = GetTargetPlatforms(); // get the platforms we want to build for List <WindowsCompiler> TargetWindowsCompilers = GetTargetWindowsCompilers(); // get the configurations we want to build for List <string> TargetConfigurations = GetTargetConfigurations(); if (bBuildSolutions) { // build target lib for all platforms foreach (TargetPlatformData TargetData in TargetPlatforms) { if (!PlatformSupportsTargetLib(TargetData)) { continue; } SetupBuildForTargetLibAndPlatform(TargetData, TargetConfigurations, TargetWindowsCompilers, false); } } HashSet <FileReference> FilesToReconcile = new HashSet <FileReference>(); if (bBuildLibraries) { // build target lib for all platforms foreach (TargetPlatformData TargetData in TargetPlatforms) { if (!PlatformSupportsTargetLib(TargetData)) { continue; } HashSet <FileReference> FilesToDelete = new HashSet <FileReference>(); foreach (string TargetConfiguration in TargetConfigurations) { // Delete output files before building them if (TargetData.Platform == UnrealTargetPlatform.Win64) { foreach (WindowsCompiler TargetCompiler in TargetWindowsCompilers) { FindOutputFiles(FilesToDelete, TargetData, TargetConfiguration, TargetCompiler); } } else { FindOutputFiles(FilesToDelete, TargetData, TargetConfiguration); } } foreach (FileReference FileToDelete in FilesToDelete) { FilesToReconcile.Add(FileToDelete); InternalUtils.SafeDeleteFile(FileToDelete.ToString()); } BuildTargetLibForPlatform(TargetData, TargetConfigurations, TargetWindowsCompilers); if (DoesPlatformUseMSBuild(TargetData)) { foreach (WindowsCompiler TargetWindowsCompiler in TargetWindowsCompilers) { CopyLibsToFinalDestination(TargetData, TargetConfigurations, TargetWindowsCompiler); } } else { CopyLibsToFinalDestination(TargetData, TargetConfigurations); } } } int P4ChangeList = InvalidChangeList; if (bAutoCreateChangelist) { string RobomergeLine = string.Empty; if (!string.IsNullOrEmpty(RobomergeCommand)) { RobomergeLine = Environment.NewLine + RobomergeCommand; } P4ChangeList = P4.CreateChange(P4Env.Client, "BuildHlslcc.Automation: Deploying hlslcc libs." + Environment.NewLine + "#rb none" + Environment.NewLine + "#lockdown Nick.Penwarden" + Environment.NewLine + "#tests none" + Environment.NewLine + "#jira none" + Environment.NewLine + "#okforgithub ignore" + RobomergeLine); } if (P4ChangeList != InvalidChangeList) { foreach (string TargetConfiguration in TargetConfigurations) { //Add any new files that p4 is not yet tracking. foreach (TargetPlatformData TargetData in TargetPlatforms) { if (!PlatformSupportsTargetLib(TargetData)) { continue; } if (TargetData.Platform == UnrealTargetPlatform.Win64) { foreach (WindowsCompiler TargetCompiler in TargetWindowsCompilers) { FindOutputFiles(FilesToReconcile, TargetData, TargetConfiguration, TargetCompiler); } } else { FindOutputFiles(FilesToReconcile, TargetData, TargetConfiguration); } } } foreach (FileReference FileToReconcile in FilesToReconcile) { P4.Reconcile(P4ChangeList, FileToReconcile.ToString()); } } if (bAutoSubmit && (P4ChangeList != InvalidChangeList)) { if (!P4.TryDeleteEmptyChange(P4ChangeList)) { LogInformation("Submitting changelist " + P4ChangeList.ToString()); int SubmittedChangeList = InvalidChangeList; P4.Submit(P4ChangeList, out SubmittedChangeList); } else { LogInformation("Nothing to submit!"); } } }
public override void ExecuteBuild() { string[] ProjectParams = ParseParamValues("Project"); string UpdateDirParam = ParseParamValue("UpdateDir", null); if (UpdateDirParam == null) { throw new AutomationException("Missing -UpdateDir=... parameter"); } DirectoryReference UpdateDir = new DirectoryReference(UpdateDirParam); bool bWrite = ParseParam("Write"); // Get all the root dirs HashSet <DirectoryReference> RootDirs = new HashSet <DirectoryReference>(); RootDirs.Add(EngineDirectory); // Add the enterprise edirectory DirectoryReference EnterpriseDirectory = DirectoryReference.Combine(RootDirectory, "Enterprise"); if (DirectoryReference.Exists(EnterpriseDirectory)) { RootDirs.Add(EnterpriseDirectory); } // Add the project directories foreach (string ProjectParam in ProjectParams) { FileReference ProjectLocation = new FileReference(ProjectParam); if (!FileReference.Exists(ProjectLocation)) { throw new AutomationException("Unable to find project '{0}'", ProjectLocation); } RootDirs.Add(ProjectLocation.Directory); } // Find all the modules HashSet <DirectoryReference> ModuleDirs = new HashSet <DirectoryReference>(); foreach (DirectoryReference RootDir in RootDirs) { // Find all the modules from the source folder DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Source"); if (DirectoryReference.Exists(SourceDir)) { foreach (FileReference ModuleFile in DirectoryReference.EnumerateFiles(SourceDir, "*.Build.cs", SearchOption.AllDirectories)) { ModuleDirs.Add(ModuleFile.Directory); } } // Find all the modules under the plugins folder DirectoryReference PluginsDir = DirectoryReference.Combine(RootDir, "Plugins"); foreach (FileReference PluginFile in DirectoryReference.EnumerateFiles(PluginsDir, "*.uplugin", SearchOption.AllDirectories)) { DirectoryReference PluginSourceDir = DirectoryReference.Combine(PluginFile.Directory, "Source"); if (DirectoryReference.Exists(PluginSourceDir)) { foreach (FileReference PluginModuleFile in DirectoryReference.EnumerateFiles(PluginSourceDir, "*.Build.cs", SearchOption.AllDirectories)) { ModuleDirs.Add(PluginModuleFile.Directory); } } } } // Find a mapping from old to new include paths Dictionary <string, Tuple <string, FileReference> > RemapIncludePaths = new Dictionary <string, Tuple <string, FileReference> >(StringComparer.InvariantCultureIgnoreCase); foreach (DirectoryReference ModuleDir in ModuleDirs) { DirectoryReference ModulePublicDir = DirectoryReference.Combine(ModuleDir, "Public"); if (DirectoryReference.Exists(ModulePublicDir)) { foreach (FileReference HeaderFile in DirectoryReference.EnumerateFiles(ModulePublicDir, "*.h", SearchOption.AllDirectories)) { string BaseIncludeFile = HeaderFile.GetFileName(); Tuple <string, FileReference> ExistingIncludeName; if (RemapIncludePaths.TryGetValue(BaseIncludeFile, out ExistingIncludeName)) { LogWarning("Multiple include paths for {0}: {1}, {2}", BaseIncludeFile, ExistingIncludeName.Item2, HeaderFile); } else { RemapIncludePaths.Add(BaseIncludeFile, Tuple.Create(HeaderFile.MakeRelativeTo(ModulePublicDir).Replace('\\', '/'), HeaderFile)); } } } } // List of folders to exclude from updates string[] ExcludeFoldersFromUpdate = { "Intermediate", "ThirdParty" }; // Enumerate all the files to update HashSet <FileReference> UpdateFiles = new HashSet <FileReference>(); foreach (FileReference UpdateFile in DirectoryReference.EnumerateFiles(UpdateDir, "*", SearchOption.AllDirectories)) { if (!UpdateFile.ContainsAnyNames(ExcludeFoldersFromUpdate, UpdateDir)) { if (UpdateFile.HasExtension(".cpp") | UpdateFile.HasExtension(".h") || UpdateFile.HasExtension(".inl")) { UpdateFiles.Add(UpdateFile); } } } // Process all the source files Dictionary <FileReference, string[]> ModifiedFiles = new Dictionary <FileReference, string[]>(); foreach (FileReference UpdateFile in UpdateFiles) { bool bModifiedFile = false; string[] Lines = FileReference.ReadAllLines(UpdateFile); for (int Idx = 0; Idx < Lines.Length; Idx++) { Match Match = Regex.Match(Lines[Idx], "^(\\s*#\\s*include\\s+\\\")([^\"]+)(\\\".*)$"); if (Match.Success) { string IncludePath = Match.Groups[2].Value; Tuple <string, FileReference> NewIncludePath; if (RemapIncludePaths.TryGetValue(IncludePath, out NewIncludePath)) { if (IncludePath != NewIncludePath.Item1) { // Log("{0}: Changing {1} -> {2}", UpdateFile, IncludePath, NewIncludePath.Item1); Lines[Idx] = String.Format("{0}{1}{2}", Match.Groups[1].Value, NewIncludePath.Item1, Match.Groups[3].Value); bModifiedFile = true; } } } } if (bModifiedFile) { ModifiedFiles.Add(UpdateFile, Lines); } } // Output them all to disk if (bWrite && ModifiedFiles.Count > 0) { LogInformation("Updating {0} files...", ModifiedFiles.Count); List <FileReference> FilesToCheckOut = new List <FileReference>(); foreach (FileReference ModifiedFile in ModifiedFiles.Keys) { if ((FileReference.GetAttributes(ModifiedFile) & FileAttributes.ReadOnly) != 0) { FilesToCheckOut.Add(ModifiedFile); } } if (FilesToCheckOut.Count > 0) { if (!P4Enabled) { throw new AutomationException("{0} files have been modified, but are read only. Run with -P4 to enable Perforce checkout.\n{1}", FilesToCheckOut.Count, String.Join("\n", FilesToCheckOut.Select(x => " " + x))); } LogInformation("Checking out files from Perforce"); int ChangeNumber = P4.CreateChange(Description: "Updating source files"); P4.Edit(ChangeNumber, FilesToCheckOut.Select(x => x.FullName).ToList(), false); } foreach (KeyValuePair <FileReference, string[]> FileToWrite in ModifiedFiles) { LogInformation("Writing {0}", FileToWrite.Key); FileReference.WriteAllLines(FileToWrite.Key, FileToWrite.Value); } } }
/// <summary> /// Main entry point for the command /// </summary> public override void ExecuteBuild() { string FileSpec = ParseRequiredStringParam("Files"); // Make sure the patterns are a valid syntax if (!FileSpec.StartsWith("//")) { throw new AutomationException("Files must be specified as full depot paths"); } // Pick out the source and target prefixes string Prefix; if (FileSpec.EndsWith("*")) { Prefix = FileSpec.Substring(0, FileSpec.Length - 1); } else if (FileSpec.EndsWith("...")) { Prefix = FileSpec.Substring(0, FileSpec.Length - 3); } else { Prefix = FileSpec; } // Make sure there aren't any other wildcards in the pattern if (Prefix.Contains("?") || Prefix.Contains("*") || Prefix.Contains("...")) { throw new AutomationException("Wildcards are only permitted at the end of filespecs"); } // Find all the source files List <string> SourceFiles = P4.Files(String.Format("-e {0}", FileSpec)); if (SourceFiles.Count == 0) { throw new AutomationException("No files found matching {0}", FileSpec); } SourceFiles.RemoveAll(x => x.StartsWith(Prefix, StringComparison.Ordinal)); // Error if we didn't find anything if (SourceFiles.Count == 0) { throw new AutomationException("No files found matching spec"); } // Find all the target files List <string> TargetFiles = new List <string>(SourceFiles.Count); foreach (string SourceFile in SourceFiles) { if (SourceFile.StartsWith(Prefix, StringComparison.OrdinalIgnoreCase)) { TargetFiles.Add(Prefix + SourceFile.Substring(Prefix.Length)); } else { throw new AutomationException("Source file '{0}' does not start with '{1}'", SourceFile, Prefix); } } // Print what we're going to do LogInformation("Ready to rename {0} files:", SourceFiles.Count); for (int Idx = 0; Idx < SourceFiles.Count; Idx++) { LogInformation("{0,3}: {1}", Idx, SourceFiles[Idx]); LogInformation("{0,3} {1}", "", TargetFiles[Idx]); } // If we're not going through with it, print the renames if (!AllowSubmit) { LogWarning("Skipping due to no -Submit option"); return; } // Force sync all the old files foreach (string OldFile in SourceFiles) { P4.LogP4(String.Format("sync -f {0}", OldFile)); } // Delete all the old files int DeleteChangeNumber = P4.CreateChange(Description: String.Format("Fixing case of {0} (1/2){1}", FileSpec, BoilerplateText)); foreach (string OldFile in SourceFiles) { P4.LogP4(String.Format("delete -k -c {0} {1}", DeleteChangeNumber, OldFile)); } P4.Submit(DeleteChangeNumber); // Re-add all the files in the new location int AddChangeNumber = P4.CreateChange(Description: String.Format("Fixing case of {0} (2/2){1}", FileSpec, BoilerplateText)); foreach (string NewFile in TargetFiles) { P4.LogP4(String.Format("add -c {0} {1}", AddChangeNumber, NewFile)); } P4.Submit(AddChangeNumber); }
public override void ExecuteBuild() { LogInformation("************************* Reworking stats code"); var Wildcards = new List <string>() { "*.h", "*.cpp", "*.inl" }; // @todo: Add support for STATCAT_ var DeclareStrings = new List <string>() { "DECLARE_CYCLE_STAT", "DECLARE_FLOAT_COUNTER_STAT", "DECLARE_DWORD_COUNTER_STAT", "DECLARE_FLOAT_ACCUMULATOR_STAT", "DECLARE_DWORD_ACCUMULATOR_STAT", "DECLARE_MEMORY_STAT", "DECLARE_MEMORY_STAT_POOL", "DECLARE_STATS_GROUP" }; DirectoryInfo DirInfo = new DirectoryInfo(CmdEnv.LocalRoot); var TopLevelDirs = DirInfo.GetDirectories(); var Dirs = new List <string>(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); } } var AllFiles = new List <string>(); foreach (var Dir in Dirs) { foreach (var Wildcard in Wildcards) { foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(Wildcard, true, Dir)) { if ( !ThisFile.Contains(@"Runtime\Core\Public\Stats\Stats2") && !ThisFile.Contains(@"\ThirdParty\") ) { LogInformation("Source File: {0}", ThisFile); AllFiles.Add(ThisFile); } } } } var DeclareFiles = new Dictionary <string, string>(); var DeclareLines = new Dictionary <string, string>(); var EnumFiles = new Dictionary <string, string>(); var EnumLines = new Dictionary <string, string>(); var Broken = new Dictionary <string, string>(); foreach (var ThisFile in AllFiles) { var FileText = ReadAllText(ThisFile); if (FileText.Contains("STAT_") || FileText.Contains("STATGROUP_")) { var Lines = ReadAllLines(ThisFile); foreach (var LineWithWS in Lines) { var Line = LineWithWS.Trim(); if (Line.Contains("STAT_") || Line.Contains("STATGROUP_")) { string TypeString = "STAT_"; if (!Line.Contains(TypeString) || Line.Contains("DECLARE_STATS_GROUP")) { TypeString = "STATGROUP_"; } bool bDeclareLine = false; foreach (var DeclareString in DeclareStrings) { if (Line.Contains(DeclareString)) { bDeclareLine = true; break; } } var Cut = Line; string Exception = "DECLARE_MEMORY_STAT_POOL"; if (Line.StartsWith(Exception)) { Cut = Line.Substring(Exception.Length); } Cut = Cut.Substring(Cut.IndexOf(TypeString)); int End = Cut.IndexOf(","); if (End < 0) { End = Cut.Length; } int EndEq = Cut.IndexOf("="); int EndParen = Cut.IndexOf(")"); int EndParen2 = Cut.IndexOf("("); if (EndEq > 0 && EndEq < End) { End = EndEq; } if (EndParen > 0 && EndParen < End) { End = EndParen; } if (EndParen2 > 0 && EndParen2 < End) { End = EndParen2; } string StatName = Cut.Substring(0, End).Trim(); bool bEnumLine = false; if (!bDeclareLine) { if ((Line.EndsWith(",") || Line == StatName) && Line.StartsWith(TypeString)) { bEnumLine = true; } } if (bEnumLine || bDeclareLine) { LogInformation("{0} {1} Line: {2} : {3}", bEnumLine ? "Enum" : "Declare", TypeString, StatName, Line); if (bEnumLine) { if (EnumFiles.ContainsKey(StatName)) { if (!Broken.ContainsKey(StatName)) { Broken.Add(StatName, Line); } } else { EnumFiles.Add(StatName, ThisFile); EnumLines.Add(StatName, Line); } } else { if (DeclareFiles.ContainsKey(StatName)) { if (!Broken.ContainsKey(StatName)) { Broken.Add(StatName, Line); } } else { DeclareFiles.Add(StatName, ThisFile); DeclareLines.Add(StatName, Line); } } } } } } } var AllGoodStats = new List <string>(); var AllGoodGroups = new List <string>(); foreach (var DeclareLine in DeclareLines) { if (!Broken.ContainsKey(DeclareLine.Key)) { if (EnumFiles.ContainsKey(DeclareLine.Key)) { if (DeclareLine.Key.StartsWith("STATGROUP_")) { AllGoodGroups.Add(DeclareLine.Key); } else { AllGoodStats.Add(DeclareLine.Key); } } else { Broken.Add(DeclareLine.Key, DeclareLine.Value); } } } var ToCheckOuts = new HashSet <string>(); LogInformation("Stats *************************"); foreach (var AllGoodStat in AllGoodStats) { LogInformation("{0}", AllGoodStat); ToCheckOuts.Add(DeclareFiles[AllGoodStat]); ToCheckOuts.Add(EnumFiles[AllGoodStat]); } LogInformation("Groups *************************"); foreach (var AllGoodGroup in AllGoodGroups) { LogInformation("{0}", AllGoodGroup); ToCheckOuts.Add(DeclareFiles[AllGoodGroup]); ToCheckOuts.Add(EnumFiles[AllGoodGroup]); } LogInformation("Broken *************************"); foreach (var BrokenItem in Broken) { LogInformation("{0}", BrokenItem.Key); } LogInformation("*************************"); int WorkingCL = -1; if (P4Enabled) { WorkingCL = P4.CreateChange(P4Env.Client, "Stat code surgery"); LogInformation("Working in {0}", WorkingCL); } else { throw new AutomationException("this command needs to run with P4."); } var CheckedOuts = new HashSet <string>(); foreach (var ToCheckOut in ToCheckOuts) { if (P4.Edit_NoExceptions(WorkingCL, ToCheckOut)) { CheckedOuts.Add(ToCheckOut); } } LogInformation("Checked Out *************************"); foreach (var CheckedOut in CheckedOuts) { LogInformation("{0}", CheckedOut); } LogInformation("Failed to check out *************************"); foreach (var ToCheckOut in ToCheckOuts) { if (!CheckedOuts.Contains(ToCheckOut)) { LogInformation("{0}", ToCheckOut); } } LogInformation("*************************"); foreach (var AllGoodStat in AllGoodStats) { if (EnumFiles[AllGoodStat].EndsWith(".cpp", StringComparison.InvariantCultureIgnoreCase)) { var DeclareFileText = ReadAllText(DeclareFiles[AllGoodStat]); DeclareFileText = DeclareFileText.Replace(DeclareLines[AllGoodStat], ""); WriteAllText(DeclareFiles[AllGoodStat], DeclareFileText); var EnumFileText = ReadAllText(EnumFiles[AllGoodStat]); EnumFileText = EnumFileText.Replace(EnumLines[AllGoodStat], DeclareLines[AllGoodStat]); WriteAllText(EnumFiles[AllGoodStat], EnumFileText); } else { var DeclareFileText = ReadAllText(DeclareFiles[AllGoodStat]); DeclareFileText = DeclareFileText.Replace(DeclareLines[AllGoodStat], "DEFINE_STAT(" + AllGoodStat + ");"); WriteAllText(DeclareFiles[AllGoodStat], DeclareFileText); var EnumFileText = ReadAllText(EnumFiles[AllGoodStat]); var ExternDeclare = DeclareLines[AllGoodStat]; int Paren = ExternDeclare.IndexOf("("); ExternDeclare = ExternDeclare.Substring(0, Paren) + "_EXTERN" + ExternDeclare.Substring(Paren); Paren = ExternDeclare.LastIndexOf(")"); ExternDeclare = ExternDeclare.Substring(0, Paren) + ", " + ExternDeclare.Substring(Paren); EnumFileText = EnumFileText.Replace(EnumLines[AllGoodStat], ExternDeclare); WriteAllText(EnumFiles[AllGoodStat], EnumFileText); } } foreach (var AllGoodGroup in AllGoodGroups) { var DeclareFileText = ReadAllText(DeclareFiles[AllGoodGroup]); DeclareFileText = DeclareFileText.Replace(DeclareLines[AllGoodGroup], ""); WriteAllText(DeclareFiles[AllGoodGroup], DeclareFileText); var EnumFileText = ReadAllText(EnumFiles[AllGoodGroup]); EnumFileText = EnumFileText.Replace(EnumLines[AllGoodGroup], DeclareLines[AllGoodGroup]); WriteAllText(EnumFiles[AllGoodGroup], EnumFileText); } LogInformation("*************************"); }
public override void ExecuteBuild() { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); // Parse out the required command line arguments var UEProjectDirectory = ParseParamValue("UEProjectDirectory"); if (UEProjectDirectory == null) { throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'"); } var UEProjectName = ParseParamValue("UEProjectName"); if (UEProjectName == null) { UEProjectName = ""; } var OneSkyConfigName = ParseParamValue("OneSkyConfigName"); if (OneSkyConfigName == null) { throw new AutomationException("Missing required command line argument: 'OneSkyConfigName'"); } var OneSkyProjectGroupName = ParseParamValue("OneSkyProjectGroupName"); if (OneSkyProjectGroupName == null) { throw new AutomationException("Missing required command line argument: 'OneSkyProjectGroupName'"); } var OneSkyProjectNames = new List <string>(); { var OneSkyProjectNamesStr = ParseParamValue("OneSkyProjectNames"); if (OneSkyProjectNamesStr == null) { throw new AutomationException("Missing required command line argument: 'OneSkyProjectNames'"); } foreach (var ProjectName in OneSkyProjectNamesStr.Split(',')) { OneSkyProjectNames.Add(ProjectName.Trim()); } } var OneSkyBranchSuffix = ParseParamValue("OneSkyBranchSuffix"); var RootWorkingDirectory = CombinePaths(CmdEnv.LocalRoot, UEProjectDirectory); // Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on if (P4Enabled) { Log("Sync necessary content to head revision"); P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Config/Localization/..."); P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Content/Localization/..."); } // Generate the info we need to gather for each project var ProjectInfos = new List <ProjectInfo>(); foreach (var ProjectName in OneSkyProjectNames) { ProjectInfos.Add(GenerateProjectInfo(RootWorkingDirectory, ProjectName)); } OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find(OneSkyConfigName); var OneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret); var OneSkyProjectGroup = GetOneSkyProjectGroup(OneSkyService, OneSkyProjectGroupName); // Create changelist for backed up POs from OneSky. if (P4Enabled) { OneSkyDownloadedPOChangeList = P4.CreateChange(P4Env.Client, "OneSky downloaded PO backup."); } // Export all text from OneSky foreach (var ProjectInfo in ProjectInfos) { ExportOneSkyProjectToDirectory(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo); } // Submit changelist for backed up POs from OneSky. if (P4Enabled) { int SubmittedChangeList; P4.Submit(OneSkyDownloadedPOChangeList, out SubmittedChangeList); } // Setup editor arguments for SCC. string EditorArguments = String.Empty; if (P4Enabled) { EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken()); } else { EditorArguments = String.Format("-SCCProvider={0}", "None"); } // Setup commandlet arguments for SCC. string CommandletSCCArguments = String.Empty; if (P4Enabled) { CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC"; } if (!AllowSubmit) { CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit"; } // Execute commandlet for each config in each project. foreach (var ProjectInfo in ProjectInfos) { foreach (var LocalizationConfigFile in ProjectInfo.LocalizationConfigFiles) { var CommandletArguments = String.Format("-config={0}", LocalizationConfigFile) + (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments); Log("Localization for {0} {1}", EditorArguments, CommandletArguments); Log("Running UE4Editor to generate localization data"); string Arguments = String.Format("{0} -run=GatherText {1} {2}", UEProjectName, EditorArguments, CommandletArguments); var RunResult = Run(EditorExe, Arguments); if (RunResult.ExitCode != 0) { Console.WriteLine("[ERROR] Error while executing localization commandlet '{0}'", Arguments); } } } // Upload all text to OneSky foreach (var ProjectInfo in ProjectInfos) { UploadProjectToOneSky(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo); } }
public override void ExecuteBuild() { Log("************************* Build Third Party Libs"); // figure out what batch/script to run string CompileScriptName; switch (UnrealBuildTool.BuildHostPlatform.Current.Platform) { case UnrealTargetPlatform.Win64: CompileScriptName = WindowsCompileScript; break; case UnrealTargetPlatform.Mac: CompileScriptName = MacCompileScript; break; case UnrealTargetPlatform.Linux: CompileScriptName = LinuxCompileScript; break; default: throw new AutomationException("Unknown runtime platform!"); } // look for changelist on the command line int WorkingCL = Int32.Parse(ParseParamValue("Changelist", "-1")); // if not specified, make one if (WorkingCL == -1) { WorkingCL = P4.CreateChange(P4Env.Client, String.Format("Third party libs built from changelist {0}", P4Env.Changelist)); } Log("Build from {0} Working in {1}", P4Env.Changelist, WorkingCL); // go to the third party lib dir string SearchLibraryDir = ParseParamValue("SearchDir", DefaultLibraryDir); CommandUtils.PushDir(SearchLibraryDir); // figure out what libraries to compile string LibsToCompileString = ParseParamValue("Libs"); // hunt down build batch files if the caller didn't specify a list to compile List <string> LibsToCompile = new List <string>(); if (string.IsNullOrEmpty(LibsToCompileString)) { // loop over third party directories looking for the right batch files foreach (string Dir in Directory.EnumerateDirectories(".")) { if (File.Exists(Path.Combine(Dir, CompileScriptName))) { LibsToCompile.Add(Path.GetFileName(Dir)); } } } else { // just split up the param and make sure the batch file exists string[] Libs = LibsToCompileString.Split('+'); bool bHadError = false; foreach (string Dir in Libs) { if (File.Exists(Path.Combine(Dir, CompileScriptName))) { LibsToCompile.Add(Path.GetFileName(Dir)); } else { LogError("Error: Requested lib {0} does not have a {1}", Dir, CompileScriptName); bHadError = true; } } if (bHadError) { // error out so that we don't fail to build some and have it lost in the noise throw new AutomationException("One or more libs were not set up to compile."); } } // set an envvar so that the inner batch files can check out files into a shared changelist Environment.SetEnvironmentVariable("THIRD_PARTY_CHANGELIST", string.Format("-c {0}", WorkingCL)); // now go through and run each batch file, foreach (string Lib in LibsToCompile) { Log("Building {0}", Lib); // go into the lib dir CommandUtils.PushDir(Lib); // run the builder batch file CommandUtils.RunAndLog(CmdEnv, CompileScriptName, "", "ThirdPartyLib_" + Lib); // go back to ThirdParty dir CommandUtils.PopDir(); } // undo the SearchLibraryDir push CommandUtils.PopDir(); PrintRunTime(); // revert any unchanged files P4.RevertUnchanged(WorkingCL); if (AllowSubmit) { int SubmittedCL; P4.Submit(WorkingCL, out SubmittedCL, true, true); Log("Submitted changelist {0}", SubmittedCL); } }
public override void ExecuteBuild() { bool bAutoCreateChangelist = true; if (ParseParam("SkipCreateChangelist")) { bAutoCreateChangelist = false; } bool bAutoSubmit = bAutoCreateChangelist; if (ParseParam("SkipSubmit")) { bAutoSubmit = false; } // if we don't pass anything, we'll just merge by default string RobomergeCommand = ParseParamValue("Robomerge", "").ToLower(); if (!string.IsNullOrEmpty(RobomergeCommand)) { // for merge default action, add flag to make sure buildmachine commit isn't skipped if (RobomergeCommand == "merge") { RobomergeCommand = "#robomerge[all] #DisregardExcludedAuthors"; } // otherwise add hashtags else if (RobomergeCommand == "ignore") { RobomergeCommand = "#robomerge #ignore"; } else if (RobomergeCommand == "null") { RobomergeCommand = "#robomerge #null"; } // otherwise the submit will likely fail. else { throw new AutomationException("Invalid Robomerge param passed in {0}. Must be \"merge\", \"null\", or \"ignore\"", RobomergeCommand); } } SetupBuildEnvironment(); TargetPlatform Platform = GetTargetPlatform(); TargetLib TargetLib = GetTargetLib(); List <string> TargetConfigurations = GetTargetConfigurations(); if (Platform.SeparateProjectPerConfig) { foreach (string TargetConfiguration in TargetConfigurations) { Platform.SetupTargetLib(TargetLib, TargetConfiguration); } } else { Platform.SetupTargetLib(TargetLib, null); } HashSet <FileReference> FilesToReconcile = new HashSet <FileReference>(); foreach (string TargetConfiguration in TargetConfigurations) { foreach (FileReference FileToDelete in Platform.EnumerateOutputFiles(TargetLib, TargetConfiguration).Distinct()) { FilesToReconcile.Add(FileToDelete); // Also clean the output files InternalUtils.SafeDeleteFile(FileToDelete.FullName); } } foreach (string TargetConfiguration in TargetConfigurations) { Platform.BuildTargetLib(TargetLib, TargetConfiguration); } Platform.CleanupTargetLib(TargetLib, null); const int InvalidChangeList = -1; int P4ChangeList = InvalidChangeList; if (bAutoCreateChangelist) { string LibDeploymentDesc = TargetLib.Name + " " + Platform.FriendlyName; var Builder = new StringBuilder(); Builder.AppendFormat("BuildCMakeLib.Automation: Deploying {0} libs.{1}", LibDeploymentDesc, Environment.NewLine); Builder.AppendLine("#rb none"); Builder.AppendLine("#lockdown Nick.Penwarden"); Builder.AppendLine("#tests none"); Builder.AppendLine("#jira none"); Builder.AppendLine("#okforgithub ignore"); if (!string.IsNullOrEmpty(RobomergeCommand)) { Builder.AppendLine(RobomergeCommand); } P4ChangeList = P4.CreateChange(P4Env.Client, Builder.ToString()); } if (P4ChangeList != InvalidChangeList) { foreach (FileReference FileToReconcile in FilesToReconcile) { P4.Reconcile(P4ChangeList, FileToReconcile.FullName); } if (bAutoSubmit) { if (!P4.TryDeleteEmptyChange(P4ChangeList)) { LogInformation("Submitting changelist " + P4ChangeList.ToString()); int SubmittedChangeList = InvalidChangeList; P4.Submit(P4ChangeList, out SubmittedChangeList); } else { LogInformation("Nothing to submit!"); } } } }
public override void ExecuteBuild() { Log("************************* UpdateCopyright"); var Wildcards = new List <string>() { "*.h", "*.cpp", "*.inl", "*.cs" }; var Dirs = new List <string>(); Dirs.Add(CombinePaths(CmdEnv.LocalRoot, "Engine", "Plugins")); { DirectoryInfo DirInfo = new DirectoryInfo(CmdEnv.LocalRoot); var TopLevelDirs = DirInfo.GetDirectories(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts")); } } } { DirectoryInfo DirInfo = new DirectoryInfo(CombinePaths(CmdEnv.LocalRoot, "Samples", "SampleGames")); var TopLevelDirs = DirInfo.GetDirectories(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts")); } } } { DirectoryInfo DirInfo = new DirectoryInfo(CombinePaths(CmdEnv.LocalRoot, "Samples", "Showcases")); var TopLevelDirs = DirInfo.GetDirectories(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts")); } } } { DirectoryInfo DirInfo = new DirectoryInfo(CombinePaths(CmdEnv.LocalRoot, "Templates")); var TopLevelDirs = DirInfo.GetDirectories(); foreach (var TopLevelDir in TopLevelDirs) { if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source"))) { Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source")); Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts")); } } } var AllFiles = new List <string>(); foreach (var Dir in Dirs) { foreach (var Wildcard in Wildcards) { foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(Wildcard, true, Dir)) { if (!ThisFile.Contains(@"\ThirdParty\")) { Log("Source File: {0}", ThisFile); AllFiles.Add(ThisFile); } } } } int WorkingCL = -1; if (P4Enabled) { WorkingCL = P4.CreateChange(P4Env.Client, "Stat code surgery"); Log("Working in {0}", WorkingCL); } else { throw new AutomationException("this command needs to run with P4."); } foreach (var ToCheckOut in AllFiles) { if (P4.Edit_NoExceptions(WorkingCL, ToCheckOut)) { Log("Checked out {0}", ToCheckOut); } else if (ToCheckOut.Contains("CodeSurgery")) { Log("Couldn't check out {0}", ToCheckOut); } else { P4.RevertAll(WorkingCL); throw new AutomationException("Couldn't check out {0}", ToCheckOut); } } var Normal = new HashSet <string>(); var Missing = new HashSet <string>(); foreach (var ThisFile in AllFiles) { var FileText = ReadAllText(ThisFile); if (FileText.Contains("2013 Epic Games, Inc. All Rights Reserved.")) { FileText = FileText.Replace("2013 Epic Games, Inc. All Rights Reserved.", "2014 Epic Games, Inc. All Rights Reserved."); WriteAllText(ThisFile, FileText); Normal.Add(ThisFile); } else { var Lines = new List <string>(ReadAllLines(ThisFile)); Lines.Insert(0, "// Copyright 1998-2014 Epic Games, Inc. All Rights Reserved."); WriteAllLines(ThisFile, Lines.ToArray()); Missing.Add(ThisFile); } } Log("Normal Copyright *************************"); foreach (var ThisFile in Normal) { Log(" {0}", ThisFile); } Log("Missing Copyright *************************"); foreach (var ThisFile in Missing) { Log(" {0}", ThisFile); } }
public override void ExecuteBuild() { var UEProjectRoot = ParseParamValue("UEProjectRoot"); if (UEProjectRoot == null) { UEProjectRoot = CmdEnv.LocalRoot; } var UEProjectDirectory = ParseParamValue("UEProjectDirectory"); if (UEProjectDirectory == null) { throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'"); } var UEProjectName = ParseParamValue("UEProjectName"); if (UEProjectName == null) { UEProjectName = ""; } var LocalizationProjectNames = new List <string>(); { var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames"); if (LocalizationProjectNamesStr != null) { foreach (var ProjectName in LocalizationProjectNamesStr.Split(',')) { LocalizationProjectNames.Add(ProjectName.Trim()); } } } var LocalizationProviderName = ParseParamValue("LocalizationProvider"); if (LocalizationProviderName == null) { LocalizationProviderName = ""; } var LocalizationStepNames = new List <string>(); { var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps"); if (LocalizationStepNamesStr == null) { LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" }); } else { foreach (var StepName in LocalizationStepNamesStr.Split(',')) { LocalizationStepNames.Add(StepName.Trim()); } } LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do } var ShouldGatherPlugins = ParseParam("IncludePlugins"); var IncludePlugins = new List <string>(); var ExcludePlugins = new List <string>(); if (ShouldGatherPlugins) { var IncludePluginsStr = ParseParamValue("IncludePlugins"); if (IncludePluginsStr != null) { foreach (var PluginName in IncludePluginsStr.Split(',')) { IncludePlugins.Add(PluginName.Trim()); } } var ExcludePluginsStr = ParseParamValue("ExcludePlugins"); if (ExcludePluginsStr != null) { foreach (var PluginName in ExcludePluginsStr.Split(',')) { ExcludePlugins.Add(PluginName.Trim()); } } } var ShouldGatherPlatforms = ParseParam("IncludePlatforms"); var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments"); if (AdditionalCommandletArguments == null) { AdditionalCommandletArguments = ""; } var LocalizationBatches = new List <LocalizationBatch>(); // Add the static set of localization projects as a batch if (LocalizationProjectNames.Count > 0) { LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames)); } // Build up any additional batches needed for platforms if (ShouldGatherPlatforms) { var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms")); if (DirectoryReference.Exists(PlatformsRootDirectory)) { foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory)) { // Find the localization targets defined for this platform var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization"))); if (PlatformTargetNames.Count > 0) { var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames)); } } } } // Build up any additional batches needed for plugins if (ShouldGatherPlugins) { var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory)); IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project); // Add a batch for each plugin that meets our criteria var AvailablePluginNames = new HashSet <string>(); foreach (var PluginInfo in AllPlugins) { AvailablePluginNames.Add(PluginInfo.Name); bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name); if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0) { var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 var PluginTargetNames = new List <string>(); foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets) { PluginTargetNames.Add(LocalizationTarget.Name); } LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames)); } } // If we had an explicit list of plugins to include, warn if any were missing foreach (string PluginName in IncludePlugins) { if (!AvailablePluginNames.Contains(PluginName)) { LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName); } } } // Create a single changelist to use for all changes, and hash the current PO files on disk so we can work out whether they actually change int PendingChangeList = 0; Dictionary <string, byte[]> InitalPOFileHashes = null; if (P4Enabled) { var ChangeListCommitMessage = "Localization Automation"; if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Build/NotForLicensees/EpicInternal.txt"))) { ChangeListCommitMessage += "\n#okforgithub ignore"; } PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage); InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); } // Process each localization batch foreach (var LocalizationBatch in LocalizationBatches) { ProcessLocalizationProjects(LocalizationBatch, PendingChangeList, UEProjectRoot, UEProjectName, LocalizationProviderName, LocalizationStepNames, AdditionalCommandletArguments); } // Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to) if (P4Enabled) { // Revert any PO files that haven't changed aside from their header { var POFilesToRevert = new List <string>(); var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); foreach (var CurrentPOFileHashPair in CurrentPOFileHashes) { byte[] InitialPOFileHash; if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value)) { POFilesToRevert.Add(CurrentPOFileHashPair.Key); } } if (POFilesToRevert.Count > 0) { var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString())); using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename)) { foreach (var POFileToRevert in POFilesToRevert) { P4RevertArgsWriter.WriteLine(POFileToRevert); } } P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename)); DeleteFile_NoExceptions(P4RevertArgsFilename); } } // Revert any other unchanged files P4.RevertUnchanged(PendingChangeList); // Submit that single changelist now if (AllowSubmit) { int SubmittedChangeList; P4.Submit(PendingChangeList, out SubmittedChangeList); } } }