private void CreateChangelist(ProjectParams Params)
{
Log("Running Step:- RebuildLightMaps::CheckOutMaps");
// Setup a P4 Cl we will use to submit the new lightmaps
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} rebuilding lightmaps from changelist {1}\n#rb None\n#tests None", Params.ShortProjectName, P4Env.Changelist));
Log("Working in {0}", WorkingCL);
}
public override void ExecuteBuild()
{
Log("************************** ShooterGame_BasicBuild");
var Params = GetParams(this);
int WorkingCL = -1;
if (P4Enabled)
{
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("ShooterGameBuild built from changelist {0}", P4Env.Changelist));
Log("Build from {0} Working in {1}", P4Env.Changelist, WorkingCL);
}
Project.Build(this, Params, WorkingCL);
Project.Cook(Params);
Project.CopyBuildToStagingDirectory(Params);
if (ParseParam("Package"))
{
Project.Package(Params, WorkingCL);
}
Project.Archive(Params);
Project.Deploy(Params);
PrintRunTime();
Project.Run(Params);
if (WorkingCL > 0)
{
//Check everything in and label it
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
P4.MakeDownstreamLabel(P4Env, "ShooterGameBasicBuild");
}
}
private void CreateChangelist(ProjectParams Params)
{
LogInformation("Running Step:- RebuildHLOD::CheckOutMaps");
// Setup a P4 Cl we will use to submit the new HLOD data
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} rebuilding HLODs from changelist {1}\n#rb None\n#tests None\n#jira none\n{2}", Params.ShortProjectName, P4Env.Changelist, RobomergeCommand));
LogInformation("Working in {0}", WorkingCL);
}
protected void DoBuildCookRun(ProjectParams Params)
{
const ProjectBuildTargets ClientTargets = ProjectBuildTargets.ClientCooked | ProjectBuildTargets.ServerCooked;
bool bGenerateNativeScripts = Params.RunAssetNativization;
int WorkingCL = -1;
if (P4Enabled && GlobalCommandLine.Submit && AllowSubmit)
{
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} build from changelist {1}", Params.ShortProjectName, P4Env.Changelist));
}
Project.Build(this, Params, WorkingCL, bGenerateNativeScripts ? (ProjectBuildTargets.All & ~ClientTargets) : ProjectBuildTargets.All);
Project.Cook(Params);
if (bGenerateNativeScripts)
{
Project.Build(this, Params, WorkingCL, ClientTargets);
}
Project.CopyBuildToStagingDirectory(Params);
Project.Package(Params, WorkingCL);
Project.Archive(Params);
Project.Deploy(Params);
PrintRunTime();
Project.Run(Params);
// Check everything in!
if (WorkingCL != -1)
{
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
}
}
protected void DoBuildCookRun(ProjectParams Params)
{
int WorkingCL = -1;
if (P4Enabled && AllowSubmit)
{
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} build from changelist {1}", Params.ShortProjectName, P4Env.Changelist));
}
Project.Build(this, Params, WorkingCL);
Project.Cook(Params);
Project.CopyBuildToStagingDirectory(Params);
Project.Package(Params, WorkingCL);
Project.Archive(Params);
Project.Deploy(Params);
PrintRunTime();
Project.Run(Params);
// Check everything in!
if (WorkingCL != -1)
{
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
}
}
protected void DoBuildCookRun(ProjectParams Params)
{
const ProjectBuildTargets ClientTargets = ProjectBuildTargets.ClientCooked | ProjectBuildTargets.ServerCooked;
bool bGenerateNativeScripts = Params.RunAssetNativization;
int WorkingCL = -1;
if (P4Enabled && GlobalCommandLine.Submit && AllowSubmit)
{
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} build from changelist {1}", Params.ShortProjectName, P4Env.Changelist));
}
Project.Build(this, Params, WorkingCL, bGenerateNativeScripts ? (ProjectBuildTargets.All & ~ClientTargets) : ProjectBuildTargets.All);
Project.Cook(Params);
if (bGenerateNativeScripts)
{
// crash reporter is built along with client targets, so we need to
// include that target flag here as well - note: that its not folded
// into ClientTargets because the editor needs its own CrashReporter
// as well (which would be built above)
Project.Build(this, Params, WorkingCL, ClientTargets | ProjectBuildTargets.CrashReporter);
}
else
{
ConfigHierarchy GameIni = ConfigCache.ReadHierarchy(ConfigHierarchyType.Game, Params.RawProjectPath.Directory, HostPlatform.Current.HostEditorPlatform);
if (GameIni != null)
{
List <string> NativizeBlueprintAssets = null;
string BlueprintNativizationMethod = "Disabled";
bool bWarnIfPackagedWithoutNativizationFlag = true;
GameIni.GetString("/Script/UnrealEd.ProjectPackagingSettings", "BlueprintNativizationMethod", out BlueprintNativizationMethod);
GameIni.GetBool("/Script/UnrealEd.ProjectPackagingSettings", "bWarnIfPackagedWithoutNativizationFlag", out bWarnIfPackagedWithoutNativizationFlag);
GameIni.GetArray("/Script/UnrealEd.ProjectPackagingSettings", "NativizeBlueprintAssets", out NativizeBlueprintAssets);
if (bWarnIfPackagedWithoutNativizationFlag && BlueprintNativizationMethod != "Disabled")
{
// Warn if we're cooking without the -nativizeAssets flag, when the project settings specify a nativization method.
// If the "exclusive" (whitelist) method is set, we only warn if at least one asset has been selected for conversion.
if (BlueprintNativizationMethod != "Exclusive" || (NativizeBlueprintAssets != null && NativizeBlueprintAssets.Count > 0))
{
LogWarning("Project is configured for Blueprint nativization, but the conversion flag (-nativizeAssets) was omitted from the command line. No nativized assets have been built as a result.");
}
}
}
}
Project.CopyBuildToStagingDirectory(Params);
Project.Package(Params, WorkingCL);
Project.Archive(Params);
Project.Deploy(Params);
PrintRunTime();
Project.Run(Params);
// Check everything in!
if (WorkingCL != -1)
{
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
}
}
public override void ExecuteBuild()
{
LogInformation("P4CLIENT={0}", GetEnvVar("P4CLIENT"));
LogInformation("P4PORT={0}", GetEnvVar("P4PORT"));
var CLDescription = "AutomationTool TestP4";
LogInformation("Creating new changelist \"{0}\" using client \"{1}\"", CLDescription, GetEnvVar("P4CLIENT"));
var ChangelistNumber = P4.CreateChange(Description: CLDescription);
LogInformation("Created changelist {0}", ChangelistNumber);
}
private void CheckOutMaps(ProjectParams Params)
{
Log("Running Step:- RebuildLightMaps::CheckOutMaps");
// Setup a P4 Cl we will use to submit the new lightmaps
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} rebuilding lightmaps from changelist {1}", Params.ShortProjectName, P4Env.Changelist));
Log("Working in {0}", WorkingCL);
string AllMapsWildcardCmdline = String.Format("{0}\\...\\*.umap", Params.RawProjectPath.Directory);
string Output = "";
P4.P4Output(out Output, String.Format("edit -c {0} {1}", WorkingCL, AllMapsWildcardCmdline));
// We need to ensure that any error in the output log is observed.
// P4 is still successful if it manages to run the operation.
if (FoundCheckOutErrorInP4Output(Output) == true)
{
LogError("Failed to check out every one of the project maps.");
throw new AutomationException("Failed to check out every one of the project maps.");
}
}
protected void DoBuildCookRun(ProjectParams Params)
{
const ProjectBuildTargets ClientTargets = ProjectBuildTargets.ClientCooked | ProjectBuildTargets.ServerCooked;
bool bGenerateNativeScripts = Params.RunAssetNativization;
int WorkingCL = -1;
if (P4Enabled && GlobalCommandLine.Submit && AllowSubmit)
{
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("{0} build from changelist {1}", Params.ShortProjectName, P4Env.Changelist));
}
Project.Build(this, Params, WorkingCL, bGenerateNativeScripts ? (ProjectBuildTargets.All & ~ClientTargets) : ProjectBuildTargets.All);
Project.Cook(Params);
if (bGenerateNativeScripts)
{
// crash reporter is built along with client targets, so we need to
// include that target flag here as well - note: that its not folded
// into ClientTargets because the editor needs its own CrashReporter
// as well (which would be built above)
Project.Build(this, Params, WorkingCL, ClientTargets | ProjectBuildTargets.CrashReporter);
}
Project.CopyBuildToStagingDirectory(Params);
Project.Package(Params, WorkingCL);
Project.Archive(Params);
Project.Deploy(Params);
PrintRunTime();
Project.Run(Params);
Project.GetFile(Params);
// Check everything in!
if (WorkingCL != -1)
{
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
}
}
public override void ExecuteBuild()
{
int WorkingCL = P4.CreateChange(P4Env.Client, String.Format("TestP4_StrandCheckout, head={0}", P4Env.Changelist));
LogInformation("Build from {0} Working in {1}", P4Env.Changelist, WorkingCL);
List <string> Sign = new List <string>();
Sign.Add(CombinePaths(CmdEnv.LocalRoot, @"\Engine\Binaries\DotNET\AgentInterface.dll"));
LogInformation("Signing and adding {0} build products to changelist {1}...", Sign.Count, WorkingCL);
CodeSign.SignMultipleIfEXEOrDLL(this, Sign);
foreach (var File in Sign)
{
P4.Sync("-f -k " + File + "#head"); // sync the file without overwriting local one
if (!FileExists(File))
{
throw new AutomationException("BUILD FAILED {0} was a build product but no longer exists", File);
}
P4.ReconcileNoDeletes(WorkingCL, File);
}
}
public override void ExecuteBuild()
{
string BucketName = ParseRequiredStringParam("Bucket");
FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile");
string CredentialsKey = ParseRequiredStringParam("CredentialsKey");
DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir");
DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir");
int Days = ParseParamInt("Days", 7);
int MaxFileSize = ParseParamInt("MaxFileSize", 0);
string RootManifestPath = ParseRequiredStringParam("Manifest");
string KeyPrefix = ParseParamValue("KeyPrefix", "");
bool bReset = ParseParam("Reset");
// The credentials to upload with
AWSCredentials Credentials;
// Try to get the credentials by the key passed in from the script
CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName);
if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials))
{
throw new AutomationException("Unknown credentials key: {0}", CredentialsKey);
}
// Create the new client
using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region))
{
using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4))
{
// Read the filters
HashSet <string> Paths = new HashSet <string>();
foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt"))
{
TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc;
if (Age < TimeSpan.FromDays(3))
{
Log.TraceInformation("Reading {0}", FilterFile.FullName);
string[] Lines = File.ReadAllLines(FilterFile.FullName);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim().Replace('\\', '/');
if (TrimLine.Length > 0)
{
Paths.Add(TrimLine);
}
}
}
else if (Age > TimeSpan.FromDays(5))
{
try
{
Log.TraceInformation("Deleting {0}", FilterFile.FullName);
FilterFile.Delete();
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to delete: {0}", Ex.Message);
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
}
}
}
Log.TraceInformation("Found {0:n0} files", Paths.Count);
// Enumerate all the files that are in the network DDC
Log.TraceInformation("");
Log.TraceInformation("Filtering files in {0}...", CacheDir);
List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag));
// Filter to the maximum size
if (MaxFileSize != 0)
{
int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize);
Log.TraceInformation("");
Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize);
}
// Create the working directory
DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC");
DirectoryReference.CreateDirectory(WorkingDir);
// Get the path to the manifest
FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath);
// Read the old root manifest
RootManifest OldRootManifest = new RootManifest();
if (FileReference.Exists(RootManifestFile))
{
OldRootManifest.Read(JsonObject.Read(RootManifestFile));
}
// Read the old bundle manifest
BundleManifest OldBundleManifest = new BundleManifest();
if (OldRootManifest.Entries.Count > 0)
{
FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json");
if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest))
{
OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest));
}
}
// Create the new manifest
BundleManifest NewBundleManifest = new BundleManifest();
// Try to download the old manifest, and add all the bundles we want to keep to the new manifest
if (!bReset)
{
foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries)
{
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
if (!FileReference.Exists(BundleFile))
{
Log.TraceInformation("Downloading {0}", BundleFile);
FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz");
if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile))
{
Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey);
continue;
}
FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming");
try
{
DecompressFile(TempCompressedFile, TempUncompressedFile);
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString());
continue;
}
FileReference.Move(TempUncompressedFile, BundleFile);
}
NewBundleManifest.Entries.Add(Bundle);
}
}
// Figure out all the item digests that we already have
Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>();
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
BinaryReader Reader = new BinaryReader(Stream);
uint Signature = Reader.ReadUInt32();
if (Signature != BundleSignatureV1)
{
throw new Exception(String.Format("Invalid signature for {0}", BundleFile));
}
int NumEntries = Reader.ReadInt32();
for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++)
{
byte[] Digest = new byte[ContentHash.LengthSHA1];
if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1)
{
throw new Exception("Unexpected EOF");
}
KeyHashes.Add(new ContentHash(Digest));
Stream.Seek(4, SeekOrigin.Current);
}
}
BundleToKeyHashes[Bundle] = KeyHashes;
}
// Calculate the download size of the manifest
long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
// Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these
// files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times.
foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes)
{
long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length);
if (ValidBundleSize < MinBundleSize)
{
NewBundleManifest.Entries.Remove(Pair.Key);
}
}
// Find all the valid digests
HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]);
}
// Remove all the files which already exist
int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash));
if (NumRemovedExist > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist);
}
// Read all the files we want to include
List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
if (Files.Count > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024));
FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y)));
}
// Generate new data
using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider())
{
// Flag for whether to update the manifest
bool bUpdateManifest = false;
// Upload the new bundle
Log.TraceInformation("");
if (FilesToInclude.Count == 0)
{
Log.TraceInformation("No new files to add.");
}
else
{
// Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together.
Log.TraceInformation("Sorting input files");
List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList();
// Get the target bundle size
long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length);
int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize);
long TargetBundleSize = TotalSize / NumBundles;
// Split the input data into bundles
List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >();
long BundleSize = 0;
for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++)
{
BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]);
BundleSize += SortedFilesToInclude[FileIdx].Item2.Length;
}
BundleFilesToIncludeList.Add(BundleFilesToInclude);
}
// Upload each bundle
DateTime NewBundleTime = DateTime.UtcNow;
for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx];
// Get the new bundle info
string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : "";
string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix);
// Create a random number for the object key
string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto);
// Create the bundle header
byte[] Header;
using (MemoryStream HeaderStream = new MemoryStream())
{
BinaryWriter Writer = new BinaryWriter(HeaderStream);
Writer.Write(BundleSignatureV1);
Writer.Write(BundleFilesToInclude.Count);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1);
Writer.Write((int)FileToInclude.Item2.Length);
}
Header = HeaderStream.ToArray();
}
// Create the output file
FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz");
Log.TraceInformation("Writing {0}", NewBundleFile);
using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true))
{
ZipStream.Write(Header, 0, Header.Length);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length);
}
}
}
// Upload the file
long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length;
long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length);
Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f));
UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null);
// Add the bundle to the new manifest
BundleManifest.Entry Bundle = new BundleManifest.Entry();
Bundle.Name = NewBundleName;
Bundle.ObjectKey = NewBundleObjectKey;
Bundle.Time = NewBundleTime;
Bundle.CompressedLength = (int)NewBundleCompressedLength;
Bundle.UncompressedLength = (int)NewBundleUncompressedLength;
NewBundleManifest.Entries.Add(Bundle);
// Mark the manifest as requiring an update
bUpdateManifest = true;
}
}
// Update the manifest
if (bUpdateManifest)
{
DateTime UtcNow = DateTime.UtcNow;
DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0);
// Update the root manifest
RootManifest NewRootManifest = new RootManifest();
NewRootManifest.AccessKey = OldRootManifest.AccessKey;
NewRootManifest.SecretKey = OldRootManifest.SecretKey;
foreach (RootManifest.Entry Entry in OldRootManifest.Entries)
{
if (Entry.CreateTime >= RemoveBundleManifestsBefore)
{
NewRootManifest.Entries.Add(Entry);
}
}
// Make sure there's an entry for the last 24h
DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0);
if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter))
{
RootManifest.Entry NewEntry = new RootManifest.Entry();
NewEntry.CreateTime = UtcNow;
NewEntry.Key = KeyPrefix + CreateObjectName(Crypto);
NewRootManifest.Entries.Add(NewEntry);
}
// Save out the new bundle manifest
FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json");
NewBundleManifest.Save(NewBundleManifestFile);
// Update all the bundle manifests still valid
foreach (RootManifest.Entry Entry in NewRootManifest.Entries)
{
Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key);
UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null);
}
// Overwrite all the existing manifests
if (AllowSubmit)
{
List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
// Create a changelist containing the new manifest
int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest");
if (ExistingFiles.Count > 0)
{
P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
NewRootManifest.Save(RootManifestFile);
}
else
{
NewRootManifest.Save(RootManifestFile);
P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
}
// Submit it
int SubmittedChangeNumber;
P4.Submit(ChangeNumber, out SubmittedChangeNumber, true);
if (SubmittedChangeNumber <= 0)
{
throw new AutomationException("Failed to submit change");
}
// Delete any bundles that are no longer referenced
HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey));
foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries)
{
if (!KeepObjectKeys.Contains(OldEntry.ObjectKey))
{
Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey);
DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null);
}
}
// Delete any bundle manifests which are no longer referenced
HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key));
foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries)
{
if (!KeepManifestKeys.Contains(OldEntry.Key))
{
Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key);
DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null);
}
}
}
else
{
// Skip submitting
Log.TraceWarning("Skipping manifest submit due to missing -Submit argument.");
}
// Update the new download size
DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
}
}
// Print some stats about the final manifest
Log.TraceInformation("");
Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0));
Log.TraceInformation("");
}
}
}
public override void ExecuteBuild()
{
int WorkingCL = -1;
if (P4Enabled && AllowSubmit)
{
string CmdLine = "";
foreach (var Arg in Params)
{
CmdLine += Arg.ToString() + " ";
}
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("MegaXGE build from changelist {0} - Params: {1}", P4Env.Changelist, CmdLine));
}
LogInformation("************************* MegaXGE");
bool Clean = ParseParam("Clean");
string CleanToolLocation = CombinePaths(CmdEnv.LocalRoot, "Engine", "Build", "Batchfiles", "Clean.bat");
bool ShowProgress = ParseParam("Progress");
var UE4Build = new UE4Build(this);
var Agenda = new UE4Build.BuildAgenda();
// we need to always build UHT when we use mega XGE
var ProgramTargets = new string[]
{
"UnrealHeaderTool",
};
Agenda.AddTargets(ProgramTargets, UnrealTargetPlatform.Win64, UnrealTargetConfiguration.Development);
if (Clean)
{
LogSetProgress(ShowProgress, "Cleaning previous builds...");
foreach (var CurTarget in ProgramTargets)
{
string Args = String.Format("{0} {1} {2}", CurTarget, UnrealTargetPlatform.Win64.ToString(), UnrealTargetConfiguration.Development.ToString());
RunAndLog(CmdEnv, CleanToolLocation, Args);
}
}
LogInformation("*************************");
for (int Arg = 1; Arg < 100; Arg++)
{
string Parm = String.Format("Target{0}", Arg);
string Target = ParseParamValue(Parm, "");
if (String.IsNullOrEmpty(Target))
{
break;
}
FileReference ProjectFile = null;
string ProjectFileParam = ParseParamValue(String.Format("Project{0}", Arg), null);
if (ProjectFileParam != null)
{
ProjectFile = new FileReference(ProjectFileParam);
if (!FileReference.Exists(ProjectFile))
{
throw new AutomationException("Project file '{0}' could not be found");
}
}
var Parts = Target.Split(' ');
string JustTarget = Parts[0];
if (String.IsNullOrEmpty(JustTarget))
{
throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target);
}
var Targets = JustTarget.Split('|');
if (Targets.Length < 1)
{
throw new AutomationException("BUILD FAILED target option '{0}' not parsed.", Target);
}
var Platforms = new List <UnrealTargetPlatform>();
var Configurations = new List <UnrealTargetConfiguration>();
for (int Part = 1; Part < Parts.Length; Part++)
{
if (!String.IsNullOrEmpty(Parts[Part]))
{
var SubParts = Parts[Part].Split('|');
foreach (var SubPart in SubParts)
{
UnrealTargetPlatform Platform;
if (UnrealTargetPlatform.TryParse(SubPart, out Platform))
{
Platforms.Add(Platform);
}
else
{
switch (SubPart.ToUpperInvariant())
{
case "DEBUG":
Configurations.Add(UnrealTargetConfiguration.Debug);
break;
case "DEBUGGAME":
Configurations.Add(UnrealTargetConfiguration.DebugGame);
break;
case "DEVELOPMENT":
Configurations.Add(UnrealTargetConfiguration.Development);
break;
case "SHIPPING":
Configurations.Add(UnrealTargetConfiguration.Shipping);
break;
case "TEST":
Configurations.Add(UnrealTargetConfiguration.Test);
break;
default:
throw new AutomationException("BUILD FAILED target option {0} not recognized.", SubPart);
}
}
}
}
}
if (Platforms.Count < 1)
{
Platforms.Add(UnrealTargetPlatform.Win64);
}
if (Configurations.Count < 1)
{
Configurations.Add(UnrealTargetConfiguration.Development);
}
foreach (var Platform in Platforms)
{
foreach (var CurTarget in Targets)
{
foreach (var Configuration in Configurations)
{
Agenda.AddTargets(new string[] { CurTarget }, Platform, Configuration, ProjectFile);
LogInformation("Target {0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString());
if (Clean)
{
string Args = String.Format("{0} {1} {2}", CurTarget, Platform.ToString(), Configuration.ToString());
RunAndLog(CmdEnv, CleanToolLocation, Args);
}
}
}
}
}
LogInformation("*************************");
UE4Build.Build(Agenda, InUpdateVersionFiles: IsBuildMachine, InUseParallelExecutor: ParseParam("useparallelexecutor"), InShowProgress: ShowProgress);
// if (WorkingCL > 0) // only move UAT files if we intend to check in some build products
// {
// UE4Build.CopyUATFilesAndAddToBuildProducts();
// }
UE4Build.CheckBuildProducts(UE4Build.BuildProductFiles);
if (WorkingCL > 0)
{
// Sign everything we built
CodeSign.SignMultipleIfEXEOrDLL(this, UE4Build.BuildProductFiles);
// Open files for add or edit
UE4Build.AddBuildProductsToChangelist(WorkingCL, UE4Build.BuildProductFiles);
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
}
PrintRunTime();
}
public override void ExecuteBuild()
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
if (P4Enabled)
{
Log("Sync necessary content to head revision");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Config/...");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Content/...");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Source/...");
Log("Localize from label {0}", P4Env.LabelToSync);
}
OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find("OneSkyConfig_EpicGames");
var oneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret);
var projectGroup = GetProjectGroup(oneSkyService, "Unreal Engine");
// Create changelist for backed up POs from OneSky.
if (P4Enabled)
{
OneSkyDownloadedPOChangeList = P4.CreateChange(P4Env.Client, "OneSky downloaded PO backup.");
}
// Export all text from OneSky
ExportProjectToDirectory(oneSkyService, projectGroup, "Engine");
ExportProjectToDirectory(oneSkyService, projectGroup, "Editor");
ExportProjectToDirectory(oneSkyService, projectGroup, "EditorTutorials");
ExportProjectToDirectory(oneSkyService, projectGroup, "PropertyNames");
ExportProjectToDirectory(oneSkyService, projectGroup, "ToolTips");
ExportProjectToDirectory(oneSkyService, projectGroup, "Category");
ExportProjectToDirectory(oneSkyService, projectGroup, "Keywords");
// Submit changelist for backed up POs from OneSky.
if (P4Enabled)
{
int SubmittedChangeList;
P4.Submit(OneSkyDownloadedPOChangeList, out SubmittedChangeList);
}
// Setup editor arguments for SCC.
string EditorArguments = String.Empty;
if (P4Enabled)
{
EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken());
}
else
{
EditorArguments = String.Format("-SCCProvider={0}", "None");
}
// Setup commandlet arguments for SCC.
string CommandletSCCArguments = String.Empty;
if (P4Enabled)
{
CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC";
}
if (!AllowSubmit)
{
CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit";
}
// Setup commandlet arguments with configurations.
var CommandletArgumentSets = new string[]
{
String.Format("-config={0}", @"./Config/Localization/Engine.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/Editor.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/EditorTutorials.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/PropertyNames.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/ToolTips.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/Category.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/Keywords.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
};
// Execute commandlet for each set of arguments.
foreach (var CommandletArguments in CommandletArgumentSets)
{
Log("Localization for {0} {1}", EditorArguments, CommandletArguments);
Log("Running UE4Editor to generate localization data");
string Arguments = String.Format("-run=GatherText {0} {1}", EditorArguments, CommandletArguments);
var RunResult = Run(EditorExe, Arguments);
if (RunResult.ExitCode != 0)
{
throw new AutomationException("Error while executing localization commandlet '{0}'", Arguments);
}
}
// Upload all text to OneSky
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Engine"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Engine"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Editor"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Editor"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "EditorTutorials"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/EditorTutorials"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "PropertyNames"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/PropertyNames"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "ToolTips"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/ToolTips"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Category"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Category"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Keywords"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Keywords"), "*.po");
// Localisation statistics estimator.
if (P4Enabled)
{
// Available only for P4
var EstimatorExePath = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/TranslatedWordsCountEstimator.exe");
var StatisticsFilePath = @"\\epicgames.net\root\UE3\Localization\WordCounts\udn.csv";
var Arguments = string.Format(
"{0} {1} {2} {3} {4}",
StatisticsFilePath,
P4Env.P4Port,
P4Env.User,
P4Env.Client,
Environment.GetEnvironmentVariable("P4PASSWD"));
var RunResult = Run(EstimatorExePath, Arguments);
if (RunResult.ExitCode != 0)
{
throw new AutomationException("Error while executing TranslatedWordsCountEstimator with arguments '{0}'", Arguments);
}
}
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "OneSky";
}
var LocalizationSteps = new List <string>();
{
var LocalizationStepsStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepsStr == null)
{
LocalizationSteps.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepsStr.Split(','))
{
LocalizationSteps.Add(StepName.Trim());
}
}
LocalizationSteps.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = CombinePaths(UEProjectRoot, UEProjectDirectory, "Plugins");
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(new DirectoryReference(PluginsRootDirectory), UEProjectName.Length == 0 ? PluginLoadedFrom.Engine : PluginLoadedFrom.GameProject);
// Add a batch for each plugin that meets our criteria
foreach (var PluginInfo in AllPlugins)
{
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
}
// Create a single changelist to use for all changes, and hash the current PO files on disk so we can work out whether they actually change
int PendingChangeList = 0;
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
PendingChangeList = P4.CreateChange(P4Env.Client, "Localization Automation");
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Process each localization batch
foreach (var LocalizationBatch in LocalizationBatches)
{
ProcessLocalizationProjects(LocalizationBatch, PendingChangeList, UEProjectRoot, UEProjectName, LocalizationProviderName, LocalizationSteps, AdditionalCommandletArguments);
}
// Submit that single changelist now
if (P4Enabled && AllowSubmit)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "";
}
var LocalizationStepNames = new List <string>();
{
var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepNamesStr == null)
{
LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepNamesStr.Split(','))
{
LocalizationStepNames.Add(StepName.Trim());
}
}
LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var ShouldGatherPlatforms = ParseParam("IncludePlatforms");
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var EnableParallelGather = ParseParam("ParallelGather");
var StartTime = DateTime.UtcNow;
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for platforms
if (ShouldGatherPlatforms)
{
var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms"));
if (DirectoryReference.Exists(PlatformsRootDirectory))
{
foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory))
{
// Find the localization targets defined for this platform
var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization")));
if (PlatformTargetNames.Count > 0)
{
var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames));
}
}
}
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory));
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project);
// Add a batch for each plugin that meets our criteria
var AvailablePluginNames = new HashSet <string>();
foreach (var PluginInfo in AllPlugins)
{
AvailablePluginNames.Add(PluginInfo.Name);
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
// If we had an explicit list of plugins to include, warn if any were missing
foreach (string PluginName in IncludePlugins)
{
if (!AvailablePluginNames.Contains(PluginName))
{
LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName);
}
}
}
// Create a single changelist to use for all changes
int PendingChangeList = 0;
if (P4Enabled)
{
var ChangeListCommitMessage = String.Format("Localization Automation using CL {0}", P4Env.Changelist);
if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Restricted/NotForLicensees/Build/EpicInternal.txt")))
{
ChangeListCommitMessage += "\n#okforgithub ignore";
}
PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage);
}
// Prepare to process each localization batch
var LocalizationTasks = new List <LocalizationTask>();
foreach (var LocalizationBatch in LocalizationBatches)
{
var LocalizationTask = new LocalizationTask(LocalizationBatch, UEProjectRoot, LocalizationProviderName, PendingChangeList, this);
LocalizationTasks.Add(LocalizationTask);
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
LogInformation("Sync necessary content to head revision");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Config/Localization/...");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
foreach (var ProjectName in LocalizationTask.Batch.LocalizationProjectNames)
{
LocalizationTask.ProjectInfos.Add(GenerateProjectInfo(LocalizationTask.RootLocalizationTargetDirectory, ProjectName, LocalizationStepNames));
}
}
// Hash the current PO files on disk so we can work out whether they actually change
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Download the latest translations from our localization provider
if (LocalizationStepNames.Contains("Download"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
LocalizationTask.LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo);
}
}
}
}
// Begin the gather command for each task
// These can run in parallel when ParallelGather is enabled
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
// Set the common basic editor arguments
var EditorArguments = P4Enabled
? String.Format("-SCCProvider=Perforce -P4Port={0} -P4User={1} -P4Client={2} -P4Passwd={3} -P4Changelist={4} -EnableSCC -DisableSCCSubmit", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList)
: "-SCCProvider=None";
if (IsBuildMachine)
{
EditorArguments += " -BuildMachine";
}
EditorArguments += " -Unattended -LogLocalizationConflicts";
if (EnableParallelGather)
{
EditorArguments += " -multiprocess";
}
if (!String.IsNullOrEmpty(AdditionalCommandletArguments))
{
EditorArguments += " " + AdditionalCommandletArguments;
}
// Set the common process run options
var CommandletRunOptions = ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand; // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves later)
if (EnableParallelGather)
{
CommandletRunOptions |= ERunOptions.NoWaitForExit;
}
foreach (var LocalizationTask in LocalizationTasks)
{
var ProjectArgument = String.IsNullOrEmpty(UEProjectName) ? "" : String.Format("\"{0}\"", Path.Combine(LocalizationTask.RootWorkingDirectory, String.Format("{0}.uproject", UEProjectName)));
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
var LocalizationConfigFiles = new List <string>();
foreach (var LocalizationStep in ProjectInfo.LocalizationSteps)
{
if (LocalizationStepNames.Contains(LocalizationStep.Name))
{
LocalizationConfigFiles.Add(LocalizationStep.LocalizationConfigFile);
}
}
if (LocalizationConfigFiles.Count > 0)
{
var Arguments = String.Format("{0} -run=GatherText -config=\"{1}\" {2}", ProjectArgument, String.Join(";", LocalizationConfigFiles), EditorArguments);
LogInformation("Running localization commandlet for '{0}': {1}", ProjectInfo.ProjectName, Arguments);
LocalizationTask.GatherProcessResults.Add(Run(EditorExe, Arguments, null, CommandletRunOptions));
}
else
{
LocalizationTask.GatherProcessResults.Add(null);
}
}
}
}
// Wait for each commandlet process to finish and report the result.
// This runs even for non-parallel execution to log the exit state of the process.
foreach (var LocalizationTask in LocalizationTasks)
{
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null)
{
RunResult.WaitForExit();
RunResult.OnProcessExited();
RunResult.DisposeProcess();
if (RunResult.ExitCode == 0)
{
LogInformation("The localization commandlet for '{0}' exited with code 0.", ProjectInfo.ProjectName);
}
else
{
LogWarning("The localization commandlet for '{0}' exited with code {1} which likely indicates a crash.", ProjectInfo.ProjectName, RunResult.ExitCode);
}
}
}
}
// Upload the latest sources to our localization provider
if (LocalizationStepNames.Contains("Upload"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
// Upload all text to our localization provider
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null && RunResult.ExitCode == 0)
{
// Recalculate the split platform paths before doing the upload, as the export may have changed them
ProjectInfo.ExportInfo.CalculateSplitPlatformNames(LocalizationTask.RootLocalizationTargetDirectory);
LocalizationTask.LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo);
}
else
{
LogWarning("Skipping upload to the localization provider for '{0}' due to an earlier commandlet failure.", ProjectInfo.ProjectName);
}
}
}
}
}
// Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to)
if (P4Enabled)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
// Submit that single changelist now
if (AllowSubmit)
{
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
LogInformation("Localize command finished in {0} seconds", RunDuration / 1000);
}
public override void ExecuteBuild()
{
SetupStaticBuildEnvironment();
bool bBuildSolutions = true;
if (ParseParam("SkipBuildSolutions"))
{
bBuildSolutions = false;
}
bool bBuildLibraries = true;
if (ParseParam("SkipBuild"))
{
bBuildLibraries = false;
}
bool bAutoCreateChangelist = true;
if (ParseParam("SkipCreateChangelist"))
{
bAutoCreateChangelist = false;
}
bool bAutoSubmit = false; // bAutoCreateChangelist;
if (ParseParam("SkipSubmit"))
{
bAutoSubmit = false;
}
// if we don't pass anything, we'll just merge by default
string RobomergeCommand = ParseParamValue("Robomerge", "").ToLower();
if (!string.IsNullOrEmpty(RobomergeCommand))
{
// for merge default action, add flag to make sure buildmachine commit isn't skipped
if (RobomergeCommand == "merge")
{
RobomergeCommand = "#robomerge[all] #DisregardExcludedAuthors";
}
// otherwise add hashtags
else if (RobomergeCommand == "ignore")
{
RobomergeCommand = "#robomerge #ignore";
}
else if (RobomergeCommand == "null")
{
RobomergeCommand = "#robomerge #null";
}
// otherwise the submit will likely fail.
else
{
throw new AutomationException("Invalid Robomerge param passed in {0}. Must be \"merge\", \"null\", or \"ignore\"", RobomergeCommand);
}
}
// get the platforms we want to build for
List <TargetPlatformData> TargetPlatforms = GetTargetPlatforms();
// get the platforms we want to build for
List <WindowsCompiler> TargetWindowsCompilers = GetTargetWindowsCompilers();
// get the configurations we want to build for
List <string> TargetConfigurations = GetTargetConfigurations();
if (bBuildSolutions)
{
// build target lib for all platforms
foreach (TargetPlatformData TargetData in TargetPlatforms)
{
if (!PlatformSupportsTargetLib(TargetData))
{
continue;
}
SetupBuildForTargetLibAndPlatform(TargetData, TargetConfigurations, TargetWindowsCompilers, false);
}
}
HashSet <FileReference> FilesToReconcile = new HashSet <FileReference>();
if (bBuildLibraries)
{
// build target lib for all platforms
foreach (TargetPlatformData TargetData in TargetPlatforms)
{
if (!PlatformSupportsTargetLib(TargetData))
{
continue;
}
HashSet <FileReference> FilesToDelete = new HashSet <FileReference>();
foreach (string TargetConfiguration in TargetConfigurations)
{
// Delete output files before building them
if (TargetData.Platform == UnrealTargetPlatform.Win64)
{
foreach (WindowsCompiler TargetCompiler in TargetWindowsCompilers)
{
FindOutputFiles(FilesToDelete, TargetData, TargetConfiguration, TargetCompiler);
}
}
else
{
FindOutputFiles(FilesToDelete, TargetData, TargetConfiguration);
}
}
foreach (FileReference FileToDelete in FilesToDelete)
{
FilesToReconcile.Add(FileToDelete);
InternalUtils.SafeDeleteFile(FileToDelete.ToString());
}
BuildTargetLibForPlatform(TargetData, TargetConfigurations, TargetWindowsCompilers);
if (DoesPlatformUseMSBuild(TargetData))
{
foreach (WindowsCompiler TargetWindowsCompiler in TargetWindowsCompilers)
{
CopyLibsToFinalDestination(TargetData, TargetConfigurations, TargetWindowsCompiler);
}
}
else
{
CopyLibsToFinalDestination(TargetData, TargetConfigurations);
}
}
}
int P4ChangeList = InvalidChangeList;
if (bAutoCreateChangelist)
{
string RobomergeLine = string.Empty;
if (!string.IsNullOrEmpty(RobomergeCommand))
{
RobomergeLine = Environment.NewLine + RobomergeCommand;
}
P4ChangeList = P4.CreateChange(P4Env.Client, "BuildHlslcc.Automation: Deploying hlslcc libs." + Environment.NewLine + "#rb none" + Environment.NewLine + "#lockdown Nick.Penwarden" + Environment.NewLine + "#tests none" + Environment.NewLine + "#jira none" + Environment.NewLine + "#okforgithub ignore" + RobomergeLine);
}
if (P4ChangeList != InvalidChangeList)
{
foreach (string TargetConfiguration in TargetConfigurations)
{
//Add any new files that p4 is not yet tracking.
foreach (TargetPlatformData TargetData in TargetPlatforms)
{
if (!PlatformSupportsTargetLib(TargetData))
{
continue;
}
if (TargetData.Platform == UnrealTargetPlatform.Win64)
{
foreach (WindowsCompiler TargetCompiler in TargetWindowsCompilers)
{
FindOutputFiles(FilesToReconcile, TargetData, TargetConfiguration, TargetCompiler);
}
}
else
{
FindOutputFiles(FilesToReconcile, TargetData, TargetConfiguration);
}
}
}
foreach (FileReference FileToReconcile in FilesToReconcile)
{
P4.Reconcile(P4ChangeList, FileToReconcile.ToString());
}
}
if (bAutoSubmit && (P4ChangeList != InvalidChangeList))
{
if (!P4.TryDeleteEmptyChange(P4ChangeList))
{
LogInformation("Submitting changelist " + P4ChangeList.ToString());
int SubmittedChangeList = InvalidChangeList;
P4.Submit(P4ChangeList, out SubmittedChangeList);
}
else
{
LogInformation("Nothing to submit!");
}
}
}
public override void ExecuteBuild()
{
string[] ProjectParams = ParseParamValues("Project");
string UpdateDirParam = ParseParamValue("UpdateDir", null);
if (UpdateDirParam == null)
{
throw new AutomationException("Missing -UpdateDir=... parameter");
}
DirectoryReference UpdateDir = new DirectoryReference(UpdateDirParam);
bool bWrite = ParseParam("Write");
// Get all the root dirs
HashSet <DirectoryReference> RootDirs = new HashSet <DirectoryReference>();
RootDirs.Add(EngineDirectory);
// Add the enterprise edirectory
DirectoryReference EnterpriseDirectory = DirectoryReference.Combine(RootDirectory, "Enterprise");
if (DirectoryReference.Exists(EnterpriseDirectory))
{
RootDirs.Add(EnterpriseDirectory);
}
// Add the project directories
foreach (string ProjectParam in ProjectParams)
{
FileReference ProjectLocation = new FileReference(ProjectParam);
if (!FileReference.Exists(ProjectLocation))
{
throw new AutomationException("Unable to find project '{0}'", ProjectLocation);
}
RootDirs.Add(ProjectLocation.Directory);
}
// Find all the modules
HashSet <DirectoryReference> ModuleDirs = new HashSet <DirectoryReference>();
foreach (DirectoryReference RootDir in RootDirs)
{
// Find all the modules from the source folder
DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Source");
if (DirectoryReference.Exists(SourceDir))
{
foreach (FileReference ModuleFile in DirectoryReference.EnumerateFiles(SourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(ModuleFile.Directory);
}
}
// Find all the modules under the plugins folder
DirectoryReference PluginsDir = DirectoryReference.Combine(RootDir, "Plugins");
foreach (FileReference PluginFile in DirectoryReference.EnumerateFiles(PluginsDir, "*.uplugin", SearchOption.AllDirectories))
{
DirectoryReference PluginSourceDir = DirectoryReference.Combine(PluginFile.Directory, "Source");
if (DirectoryReference.Exists(PluginSourceDir))
{
foreach (FileReference PluginModuleFile in DirectoryReference.EnumerateFiles(PluginSourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(PluginModuleFile.Directory);
}
}
}
}
// Find a mapping from old to new include paths
Dictionary <string, Tuple <string, FileReference> > RemapIncludePaths = new Dictionary <string, Tuple <string, FileReference> >(StringComparer.InvariantCultureIgnoreCase);
foreach (DirectoryReference ModuleDir in ModuleDirs)
{
DirectoryReference ModulePublicDir = DirectoryReference.Combine(ModuleDir, "Public");
if (DirectoryReference.Exists(ModulePublicDir))
{
foreach (FileReference HeaderFile in DirectoryReference.EnumerateFiles(ModulePublicDir, "*.h", SearchOption.AllDirectories))
{
string BaseIncludeFile = HeaderFile.GetFileName();
Tuple <string, FileReference> ExistingIncludeName;
if (RemapIncludePaths.TryGetValue(BaseIncludeFile, out ExistingIncludeName))
{
LogWarning("Multiple include paths for {0}: {1}, {2}", BaseIncludeFile, ExistingIncludeName.Item2, HeaderFile);
}
else
{
RemapIncludePaths.Add(BaseIncludeFile, Tuple.Create(HeaderFile.MakeRelativeTo(ModulePublicDir).Replace('\\', '/'), HeaderFile));
}
}
}
}
// List of folders to exclude from updates
string[] ExcludeFoldersFromUpdate =
{
"Intermediate",
"ThirdParty"
};
// Enumerate all the files to update
HashSet <FileReference> UpdateFiles = new HashSet <FileReference>();
foreach (FileReference UpdateFile in DirectoryReference.EnumerateFiles(UpdateDir, "*", SearchOption.AllDirectories))
{
if (!UpdateFile.ContainsAnyNames(ExcludeFoldersFromUpdate, UpdateDir))
{
if (UpdateFile.HasExtension(".cpp") | UpdateFile.HasExtension(".h") || UpdateFile.HasExtension(".inl"))
{
UpdateFiles.Add(UpdateFile);
}
}
}
// Process all the source files
Dictionary <FileReference, string[]> ModifiedFiles = new Dictionary <FileReference, string[]>();
foreach (FileReference UpdateFile in UpdateFiles)
{
bool bModifiedFile = false;
string[] Lines = FileReference.ReadAllLines(UpdateFile);
for (int Idx = 0; Idx < Lines.Length; Idx++)
{
Match Match = Regex.Match(Lines[Idx], "^(\\s*#\\s*include\\s+\\\")([^\"]+)(\\\".*)$");
if (Match.Success)
{
string IncludePath = Match.Groups[2].Value;
Tuple <string, FileReference> NewIncludePath;
if (RemapIncludePaths.TryGetValue(IncludePath, out NewIncludePath))
{
if (IncludePath != NewIncludePath.Item1)
{
// Log("{0}: Changing {1} -> {2}", UpdateFile, IncludePath, NewIncludePath.Item1);
Lines[Idx] = String.Format("{0}{1}{2}", Match.Groups[1].Value, NewIncludePath.Item1, Match.Groups[3].Value);
bModifiedFile = true;
}
}
}
}
if (bModifiedFile)
{
ModifiedFiles.Add(UpdateFile, Lines);
}
}
// Output them all to disk
if (bWrite && ModifiedFiles.Count > 0)
{
LogInformation("Updating {0} files...", ModifiedFiles.Count);
List <FileReference> FilesToCheckOut = new List <FileReference>();
foreach (FileReference ModifiedFile in ModifiedFiles.Keys)
{
if ((FileReference.GetAttributes(ModifiedFile) & FileAttributes.ReadOnly) != 0)
{
FilesToCheckOut.Add(ModifiedFile);
}
}
if (FilesToCheckOut.Count > 0)
{
if (!P4Enabled)
{
throw new AutomationException("{0} files have been modified, but are read only. Run with -P4 to enable Perforce checkout.\n{1}", FilesToCheckOut.Count, String.Join("\n", FilesToCheckOut.Select(x => " " + x)));
}
LogInformation("Checking out files from Perforce");
int ChangeNumber = P4.CreateChange(Description: "Updating source files");
P4.Edit(ChangeNumber, FilesToCheckOut.Select(x => x.FullName).ToList(), false);
}
foreach (KeyValuePair <FileReference, string[]> FileToWrite in ModifiedFiles)
{
LogInformation("Writing {0}", FileToWrite.Key);
FileReference.WriteAllLines(FileToWrite.Key, FileToWrite.Value);
}
}
}
/// <summary>
/// Main entry point for the command
/// </summary>
public override void ExecuteBuild()
{
string FileSpec = ParseRequiredStringParam("Files");
// Make sure the patterns are a valid syntax
if (!FileSpec.StartsWith("//"))
{
throw new AutomationException("Files must be specified as full depot paths");
}
// Pick out the source and target prefixes
string Prefix;
if (FileSpec.EndsWith("*"))
{
Prefix = FileSpec.Substring(0, FileSpec.Length - 1);
}
else if (FileSpec.EndsWith("..."))
{
Prefix = FileSpec.Substring(0, FileSpec.Length - 3);
}
else
{
Prefix = FileSpec;
}
// Make sure there aren't any other wildcards in the pattern
if (Prefix.Contains("?") || Prefix.Contains("*") || Prefix.Contains("..."))
{
throw new AutomationException("Wildcards are only permitted at the end of filespecs");
}
// Find all the source files
List <string> SourceFiles = P4.Files(String.Format("-e {0}", FileSpec));
if (SourceFiles.Count == 0)
{
throw new AutomationException("No files found matching {0}", FileSpec);
}
SourceFiles.RemoveAll(x => x.StartsWith(Prefix, StringComparison.Ordinal));
// Error if we didn't find anything
if (SourceFiles.Count == 0)
{
throw new AutomationException("No files found matching spec");
}
// Find all the target files
List <string> TargetFiles = new List <string>(SourceFiles.Count);
foreach (string SourceFile in SourceFiles)
{
if (SourceFile.StartsWith(Prefix, StringComparison.OrdinalIgnoreCase))
{
TargetFiles.Add(Prefix + SourceFile.Substring(Prefix.Length));
}
else
{
throw new AutomationException("Source file '{0}' does not start with '{1}'", SourceFile, Prefix);
}
}
// Print what we're going to do
LogInformation("Ready to rename {0} files:", SourceFiles.Count);
for (int Idx = 0; Idx < SourceFiles.Count; Idx++)
{
LogInformation("{0,3}: {1}", Idx, SourceFiles[Idx]);
LogInformation("{0,3} {1}", "", TargetFiles[Idx]);
}
// If we're not going through with it, print the renames
if (!AllowSubmit)
{
LogWarning("Skipping due to no -Submit option");
return;
}
// Force sync all the old files
foreach (string OldFile in SourceFiles)
{
P4.LogP4(String.Format("sync -f {0}", OldFile));
}
// Delete all the old files
int DeleteChangeNumber = P4.CreateChange(Description: String.Format("Fixing case of {0} (1/2){1}", FileSpec, BoilerplateText));
foreach (string OldFile in SourceFiles)
{
P4.LogP4(String.Format("delete -k -c {0} {1}", DeleteChangeNumber, OldFile));
}
P4.Submit(DeleteChangeNumber);
// Re-add all the files in the new location
int AddChangeNumber = P4.CreateChange(Description: String.Format("Fixing case of {0} (2/2){1}", FileSpec, BoilerplateText));
foreach (string NewFile in TargetFiles)
{
P4.LogP4(String.Format("add -c {0} {1}", AddChangeNumber, NewFile));
}
P4.Submit(AddChangeNumber);
}
public override void ExecuteBuild()
{
LogInformation("************************* Reworking stats code");
var Wildcards = new List <string>()
{
"*.h", "*.cpp", "*.inl"
};
// @todo: Add support for STATCAT_
var DeclareStrings = new List <string>()
{
"DECLARE_CYCLE_STAT", "DECLARE_FLOAT_COUNTER_STAT", "DECLARE_DWORD_COUNTER_STAT", "DECLARE_FLOAT_ACCUMULATOR_STAT", "DECLARE_DWORD_ACCUMULATOR_STAT", "DECLARE_MEMORY_STAT", "DECLARE_MEMORY_STAT_POOL", "DECLARE_STATS_GROUP"
};
DirectoryInfo DirInfo = new DirectoryInfo(CmdEnv.LocalRoot);
var TopLevelDirs = DirInfo.GetDirectories();
var Dirs = new List <string>();
foreach (var TopLevelDir in TopLevelDirs)
{
if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source")))
{
Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source"));
}
}
var AllFiles = new List <string>();
foreach (var Dir in Dirs)
{
foreach (var Wildcard in Wildcards)
{
foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(Wildcard, true, Dir))
{
if (
!ThisFile.Contains(@"Runtime\Core\Public\Stats\Stats2") &&
!ThisFile.Contains(@"\ThirdParty\")
)
{
LogInformation("Source File: {0}", ThisFile);
AllFiles.Add(ThisFile);
}
}
}
}
var DeclareFiles = new Dictionary <string, string>();
var DeclareLines = new Dictionary <string, string>();
var EnumFiles = new Dictionary <string, string>();
var EnumLines = new Dictionary <string, string>();
var Broken = new Dictionary <string, string>();
foreach (var ThisFile in AllFiles)
{
var FileText = ReadAllText(ThisFile);
if (FileText.Contains("STAT_") || FileText.Contains("STATGROUP_"))
{
var Lines = ReadAllLines(ThisFile);
foreach (var LineWithWS in Lines)
{
var Line = LineWithWS.Trim();
if (Line.Contains("STAT_") || Line.Contains("STATGROUP_"))
{
string TypeString = "STAT_";
if (!Line.Contains(TypeString) || Line.Contains("DECLARE_STATS_GROUP"))
{
TypeString = "STATGROUP_";
}
bool bDeclareLine = false;
foreach (var DeclareString in DeclareStrings)
{
if (Line.Contains(DeclareString))
{
bDeclareLine = true;
break;
}
}
var Cut = Line;
string Exception = "DECLARE_MEMORY_STAT_POOL";
if (Line.StartsWith(Exception))
{
Cut = Line.Substring(Exception.Length);
}
Cut = Cut.Substring(Cut.IndexOf(TypeString));
int End = Cut.IndexOf(",");
if (End < 0)
{
End = Cut.Length;
}
int EndEq = Cut.IndexOf("=");
int EndParen = Cut.IndexOf(")");
int EndParen2 = Cut.IndexOf("(");
if (EndEq > 0 && EndEq < End)
{
End = EndEq;
}
if (EndParen > 0 && EndParen < End)
{
End = EndParen;
}
if (EndParen2 > 0 && EndParen2 < End)
{
End = EndParen2;
}
string StatName = Cut.Substring(0, End).Trim();
bool bEnumLine = false;
if (!bDeclareLine)
{
if ((Line.EndsWith(",") || Line == StatName) && Line.StartsWith(TypeString))
{
bEnumLine = true;
}
}
if (bEnumLine || bDeclareLine)
{
LogInformation("{0} {1} Line: {2} : {3}", bEnumLine ? "Enum" : "Declare", TypeString, StatName, Line);
if (bEnumLine)
{
if (EnumFiles.ContainsKey(StatName))
{
if (!Broken.ContainsKey(StatName))
{
Broken.Add(StatName, Line);
}
}
else
{
EnumFiles.Add(StatName, ThisFile);
EnumLines.Add(StatName, Line);
}
}
else
{
if (DeclareFiles.ContainsKey(StatName))
{
if (!Broken.ContainsKey(StatName))
{
Broken.Add(StatName, Line);
}
}
else
{
DeclareFiles.Add(StatName, ThisFile);
DeclareLines.Add(StatName, Line);
}
}
}
}
}
}
}
var AllGoodStats = new List <string>();
var AllGoodGroups = new List <string>();
foreach (var DeclareLine in DeclareLines)
{
if (!Broken.ContainsKey(DeclareLine.Key))
{
if (EnumFiles.ContainsKey(DeclareLine.Key))
{
if (DeclareLine.Key.StartsWith("STATGROUP_"))
{
AllGoodGroups.Add(DeclareLine.Key);
}
else
{
AllGoodStats.Add(DeclareLine.Key);
}
}
else
{
Broken.Add(DeclareLine.Key, DeclareLine.Value);
}
}
}
var ToCheckOuts = new HashSet <string>();
LogInformation("Stats *************************");
foreach (var AllGoodStat in AllGoodStats)
{
LogInformation("{0}", AllGoodStat);
ToCheckOuts.Add(DeclareFiles[AllGoodStat]);
ToCheckOuts.Add(EnumFiles[AllGoodStat]);
}
LogInformation("Groups *************************");
foreach (var AllGoodGroup in AllGoodGroups)
{
LogInformation("{0}", AllGoodGroup);
ToCheckOuts.Add(DeclareFiles[AllGoodGroup]);
ToCheckOuts.Add(EnumFiles[AllGoodGroup]);
}
LogInformation("Broken *************************");
foreach (var BrokenItem in Broken)
{
LogInformation("{0}", BrokenItem.Key);
}
LogInformation("*************************");
int WorkingCL = -1;
if (P4Enabled)
{
WorkingCL = P4.CreateChange(P4Env.Client, "Stat code surgery");
LogInformation("Working in {0}", WorkingCL);
}
else
{
throw new AutomationException("this command needs to run with P4.");
}
var CheckedOuts = new HashSet <string>();
foreach (var ToCheckOut in ToCheckOuts)
{
if (P4.Edit_NoExceptions(WorkingCL, ToCheckOut))
{
CheckedOuts.Add(ToCheckOut);
}
}
LogInformation("Checked Out *************************");
foreach (var CheckedOut in CheckedOuts)
{
LogInformation("{0}", CheckedOut);
}
LogInformation("Failed to check out *************************");
foreach (var ToCheckOut in ToCheckOuts)
{
if (!CheckedOuts.Contains(ToCheckOut))
{
LogInformation("{0}", ToCheckOut);
}
}
LogInformation("*************************");
foreach (var AllGoodStat in AllGoodStats)
{
if (EnumFiles[AllGoodStat].EndsWith(".cpp", StringComparison.InvariantCultureIgnoreCase))
{
var DeclareFileText = ReadAllText(DeclareFiles[AllGoodStat]);
DeclareFileText = DeclareFileText.Replace(DeclareLines[AllGoodStat], "");
WriteAllText(DeclareFiles[AllGoodStat], DeclareFileText);
var EnumFileText = ReadAllText(EnumFiles[AllGoodStat]);
EnumFileText = EnumFileText.Replace(EnumLines[AllGoodStat], DeclareLines[AllGoodStat]);
WriteAllText(EnumFiles[AllGoodStat], EnumFileText);
}
else
{
var DeclareFileText = ReadAllText(DeclareFiles[AllGoodStat]);
DeclareFileText = DeclareFileText.Replace(DeclareLines[AllGoodStat], "DEFINE_STAT(" + AllGoodStat + ");");
WriteAllText(DeclareFiles[AllGoodStat], DeclareFileText);
var EnumFileText = ReadAllText(EnumFiles[AllGoodStat]);
var ExternDeclare = DeclareLines[AllGoodStat];
int Paren = ExternDeclare.IndexOf("(");
ExternDeclare = ExternDeclare.Substring(0, Paren) + "_EXTERN" + ExternDeclare.Substring(Paren);
Paren = ExternDeclare.LastIndexOf(")");
ExternDeclare = ExternDeclare.Substring(0, Paren) + ", " + ExternDeclare.Substring(Paren);
EnumFileText = EnumFileText.Replace(EnumLines[AllGoodStat], ExternDeclare);
WriteAllText(EnumFiles[AllGoodStat], EnumFileText);
}
}
foreach (var AllGoodGroup in AllGoodGroups)
{
var DeclareFileText = ReadAllText(DeclareFiles[AllGoodGroup]);
DeclareFileText = DeclareFileText.Replace(DeclareLines[AllGoodGroup], "");
WriteAllText(DeclareFiles[AllGoodGroup], DeclareFileText);
var EnumFileText = ReadAllText(EnumFiles[AllGoodGroup]);
EnumFileText = EnumFileText.Replace(EnumLines[AllGoodGroup], DeclareLines[AllGoodGroup]);
WriteAllText(EnumFiles[AllGoodGroup], EnumFileText);
}
LogInformation("*************************");
}
public override void ExecuteBuild()
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
// Parse out the required command line arguments
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var OneSkyConfigName = ParseParamValue("OneSkyConfigName");
if (OneSkyConfigName == null)
{
throw new AutomationException("Missing required command line argument: 'OneSkyConfigName'");
}
var OneSkyProjectGroupName = ParseParamValue("OneSkyProjectGroupName");
if (OneSkyProjectGroupName == null)
{
throw new AutomationException("Missing required command line argument: 'OneSkyProjectGroupName'");
}
var OneSkyProjectNames = new List <string>();
{
var OneSkyProjectNamesStr = ParseParamValue("OneSkyProjectNames");
if (OneSkyProjectNamesStr == null)
{
throw new AutomationException("Missing required command line argument: 'OneSkyProjectNames'");
}
foreach (var ProjectName in OneSkyProjectNamesStr.Split(','))
{
OneSkyProjectNames.Add(ProjectName.Trim());
}
}
var OneSkyBranchSuffix = ParseParamValue("OneSkyBranchSuffix");
var RootWorkingDirectory = CombinePaths(CmdEnv.LocalRoot, UEProjectDirectory);
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
Log("Sync necessary content to head revision");
P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Config/Localization/...");
P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
var ProjectInfos = new List <ProjectInfo>();
foreach (var ProjectName in OneSkyProjectNames)
{
ProjectInfos.Add(GenerateProjectInfo(RootWorkingDirectory, ProjectName));
}
OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find(OneSkyConfigName);
var OneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret);
var OneSkyProjectGroup = GetOneSkyProjectGroup(OneSkyService, OneSkyProjectGroupName);
// Create changelist for backed up POs from OneSky.
if (P4Enabled)
{
OneSkyDownloadedPOChangeList = P4.CreateChange(P4Env.Client, "OneSky downloaded PO backup.");
}
// Export all text from OneSky
foreach (var ProjectInfo in ProjectInfos)
{
ExportOneSkyProjectToDirectory(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo);
}
// Submit changelist for backed up POs from OneSky.
if (P4Enabled)
{
int SubmittedChangeList;
P4.Submit(OneSkyDownloadedPOChangeList, out SubmittedChangeList);
}
// Setup editor arguments for SCC.
string EditorArguments = String.Empty;
if (P4Enabled)
{
EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken());
}
else
{
EditorArguments = String.Format("-SCCProvider={0}", "None");
}
// Setup commandlet arguments for SCC.
string CommandletSCCArguments = String.Empty;
if (P4Enabled)
{
CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC";
}
if (!AllowSubmit)
{
CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit";
}
// Execute commandlet for each config in each project.
foreach (var ProjectInfo in ProjectInfos)
{
foreach (var LocalizationConfigFile in ProjectInfo.LocalizationConfigFiles)
{
var CommandletArguments = String.Format("-config={0}", LocalizationConfigFile) + (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments);
Log("Localization for {0} {1}", EditorArguments, CommandletArguments);
Log("Running UE4Editor to generate localization data");
string Arguments = String.Format("{0} -run=GatherText {1} {2}", UEProjectName, EditorArguments, CommandletArguments);
var RunResult = Run(EditorExe, Arguments);
if (RunResult.ExitCode != 0)
{
Console.WriteLine("[ERROR] Error while executing localization commandlet '{0}'", Arguments);
}
}
}
// Upload all text to OneSky
foreach (var ProjectInfo in ProjectInfos)
{
UploadProjectToOneSky(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo);
}
}
public override void ExecuteBuild()
{
Log("************************* Build Third Party Libs");
// figure out what batch/script to run
string CompileScriptName;
switch (UnrealBuildTool.BuildHostPlatform.Current.Platform)
{
case UnrealTargetPlatform.Win64:
CompileScriptName = WindowsCompileScript;
break;
case UnrealTargetPlatform.Mac:
CompileScriptName = MacCompileScript;
break;
case UnrealTargetPlatform.Linux:
CompileScriptName = LinuxCompileScript;
break;
default:
throw new AutomationException("Unknown runtime platform!");
}
// look for changelist on the command line
int WorkingCL = Int32.Parse(ParseParamValue("Changelist", "-1"));
// if not specified, make one
if (WorkingCL == -1)
{
WorkingCL = P4.CreateChange(P4Env.Client, String.Format("Third party libs built from changelist {0}", P4Env.Changelist));
}
Log("Build from {0} Working in {1}", P4Env.Changelist, WorkingCL);
// go to the third party lib dir
string SearchLibraryDir = ParseParamValue("SearchDir", DefaultLibraryDir);
CommandUtils.PushDir(SearchLibraryDir);
// figure out what libraries to compile
string LibsToCompileString = ParseParamValue("Libs");
// hunt down build batch files if the caller didn't specify a list to compile
List <string> LibsToCompile = new List <string>();
if (string.IsNullOrEmpty(LibsToCompileString))
{
// loop over third party directories looking for the right batch files
foreach (string Dir in Directory.EnumerateDirectories("."))
{
if (File.Exists(Path.Combine(Dir, CompileScriptName)))
{
LibsToCompile.Add(Path.GetFileName(Dir));
}
}
}
else
{
// just split up the param and make sure the batch file exists
string[] Libs = LibsToCompileString.Split('+');
bool bHadError = false;
foreach (string Dir in Libs)
{
if (File.Exists(Path.Combine(Dir, CompileScriptName)))
{
LibsToCompile.Add(Path.GetFileName(Dir));
}
else
{
LogError("Error: Requested lib {0} does not have a {1}", Dir, CompileScriptName);
bHadError = true;
}
}
if (bHadError)
{
// error out so that we don't fail to build some and have it lost in the noise
throw new AutomationException("One or more libs were not set up to compile.");
}
}
// set an envvar so that the inner batch files can check out files into a shared changelist
Environment.SetEnvironmentVariable("THIRD_PARTY_CHANGELIST", string.Format("-c {0}", WorkingCL));
// now go through and run each batch file,
foreach (string Lib in LibsToCompile)
{
Log("Building {0}", Lib);
// go into the lib dir
CommandUtils.PushDir(Lib);
// run the builder batch file
CommandUtils.RunAndLog(CmdEnv, CompileScriptName, "", "ThirdPartyLib_" + Lib);
// go back to ThirdParty dir
CommandUtils.PopDir();
}
// undo the SearchLibraryDir push
CommandUtils.PopDir();
PrintRunTime();
// revert any unchanged files
P4.RevertUnchanged(WorkingCL);
if (AllowSubmit)
{
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL, true, true);
Log("Submitted changelist {0}", SubmittedCL);
}
}
public override void ExecuteBuild()
{
bool bAutoCreateChangelist = true;
if (ParseParam("SkipCreateChangelist"))
{
bAutoCreateChangelist = false;
}
bool bAutoSubmit = bAutoCreateChangelist;
if (ParseParam("SkipSubmit"))
{
bAutoSubmit = false;
}
// if we don't pass anything, we'll just merge by default
string RobomergeCommand = ParseParamValue("Robomerge", "").ToLower();
if (!string.IsNullOrEmpty(RobomergeCommand))
{
// for merge default action, add flag to make sure buildmachine commit isn't skipped
if (RobomergeCommand == "merge")
{
RobomergeCommand = "#robomerge[all] #DisregardExcludedAuthors";
}
// otherwise add hashtags
else if (RobomergeCommand == "ignore")
{
RobomergeCommand = "#robomerge #ignore";
}
else if (RobomergeCommand == "null")
{
RobomergeCommand = "#robomerge #null";
}
// otherwise the submit will likely fail.
else
{
throw new AutomationException("Invalid Robomerge param passed in {0}. Must be \"merge\", \"null\", or \"ignore\"", RobomergeCommand);
}
}
SetupBuildEnvironment();
TargetPlatform Platform = GetTargetPlatform();
TargetLib TargetLib = GetTargetLib();
List <string> TargetConfigurations = GetTargetConfigurations();
if (Platform.SeparateProjectPerConfig)
{
foreach (string TargetConfiguration in TargetConfigurations)
{
Platform.SetupTargetLib(TargetLib, TargetConfiguration);
}
}
else
{
Platform.SetupTargetLib(TargetLib, null);
}
HashSet <FileReference> FilesToReconcile = new HashSet <FileReference>();
foreach (string TargetConfiguration in TargetConfigurations)
{
foreach (FileReference FileToDelete in Platform.EnumerateOutputFiles(TargetLib, TargetConfiguration).Distinct())
{
FilesToReconcile.Add(FileToDelete);
// Also clean the output files
InternalUtils.SafeDeleteFile(FileToDelete.FullName);
}
}
foreach (string TargetConfiguration in TargetConfigurations)
{
Platform.BuildTargetLib(TargetLib, TargetConfiguration);
}
Platform.CleanupTargetLib(TargetLib, null);
const int InvalidChangeList = -1;
int P4ChangeList = InvalidChangeList;
if (bAutoCreateChangelist)
{
string LibDeploymentDesc = TargetLib.Name + " " + Platform.FriendlyName;
var Builder = new StringBuilder();
Builder.AppendFormat("BuildCMakeLib.Automation: Deploying {0} libs.{1}", LibDeploymentDesc, Environment.NewLine);
Builder.AppendLine("#rb none");
Builder.AppendLine("#lockdown Nick.Penwarden");
Builder.AppendLine("#tests none");
Builder.AppendLine("#jira none");
Builder.AppendLine("#okforgithub ignore");
if (!string.IsNullOrEmpty(RobomergeCommand))
{
Builder.AppendLine(RobomergeCommand);
}
P4ChangeList = P4.CreateChange(P4Env.Client, Builder.ToString());
}
if (P4ChangeList != InvalidChangeList)
{
foreach (FileReference FileToReconcile in FilesToReconcile)
{
P4.Reconcile(P4ChangeList, FileToReconcile.FullName);
}
if (bAutoSubmit)
{
if (!P4.TryDeleteEmptyChange(P4ChangeList))
{
LogInformation("Submitting changelist " + P4ChangeList.ToString());
int SubmittedChangeList = InvalidChangeList;
P4.Submit(P4ChangeList, out SubmittedChangeList);
}
else
{
LogInformation("Nothing to submit!");
}
}
}
}
public override void ExecuteBuild()
{
Log("************************* UpdateCopyright");
var Wildcards = new List <string>()
{
"*.h", "*.cpp", "*.inl", "*.cs"
};
var Dirs = new List <string>();
Dirs.Add(CombinePaths(CmdEnv.LocalRoot, "Engine", "Plugins"));
{
DirectoryInfo DirInfo = new DirectoryInfo(CmdEnv.LocalRoot);
var TopLevelDirs = DirInfo.GetDirectories();
foreach (var TopLevelDir in TopLevelDirs)
{
if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source")))
{
Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source"));
Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts"));
}
}
}
{
DirectoryInfo DirInfo = new DirectoryInfo(CombinePaths(CmdEnv.LocalRoot, "Samples", "SampleGames"));
var TopLevelDirs = DirInfo.GetDirectories();
foreach (var TopLevelDir in TopLevelDirs)
{
if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source")))
{
Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source"));
Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts"));
}
}
}
{
DirectoryInfo DirInfo = new DirectoryInfo(CombinePaths(CmdEnv.LocalRoot, "Samples", "Showcases"));
var TopLevelDirs = DirInfo.GetDirectories();
foreach (var TopLevelDir in TopLevelDirs)
{
if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source")))
{
Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source"));
Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts"));
}
}
}
{
DirectoryInfo DirInfo = new DirectoryInfo(CombinePaths(CmdEnv.LocalRoot, "Templates"));
var TopLevelDirs = DirInfo.GetDirectories();
foreach (var TopLevelDir in TopLevelDirs)
{
if (DirectoryExists_NoExceptions(CombinePaths(TopLevelDir.FullName, "Source")))
{
Dirs.Add(CombinePaths(TopLevelDir.FullName, "Source"));
Dirs.Add(CombinePaths(TopLevelDir.FullName, "Build", "Scripts"));
}
}
}
var AllFiles = new List <string>();
foreach (var Dir in Dirs)
{
foreach (var Wildcard in Wildcards)
{
foreach (var ThisFile in CommandUtils.FindFiles_NoExceptions(Wildcard, true, Dir))
{
if (!ThisFile.Contains(@"\ThirdParty\"))
{
Log("Source File: {0}", ThisFile);
AllFiles.Add(ThisFile);
}
}
}
}
int WorkingCL = -1;
if (P4Enabled)
{
WorkingCL = P4.CreateChange(P4Env.Client, "Stat code surgery");
Log("Working in {0}", WorkingCL);
}
else
{
throw new AutomationException("this command needs to run with P4.");
}
foreach (var ToCheckOut in AllFiles)
{
if (P4.Edit_NoExceptions(WorkingCL, ToCheckOut))
{
Log("Checked out {0}", ToCheckOut);
}
else if (ToCheckOut.Contains("CodeSurgery"))
{
Log("Couldn't check out {0}", ToCheckOut);
}
else
{
P4.RevertAll(WorkingCL);
throw new AutomationException("Couldn't check out {0}", ToCheckOut);
}
}
var Normal = new HashSet <string>();
var Missing = new HashSet <string>();
foreach (var ThisFile in AllFiles)
{
var FileText = ReadAllText(ThisFile);
if (FileText.Contains("2013 Epic Games, Inc. All Rights Reserved."))
{
FileText = FileText.Replace("2013 Epic Games, Inc. All Rights Reserved.", "2014 Epic Games, Inc. All Rights Reserved.");
WriteAllText(ThisFile, FileText);
Normal.Add(ThisFile);
}
else
{
var Lines = new List <string>(ReadAllLines(ThisFile));
Lines.Insert(0, "// Copyright 1998-2014 Epic Games, Inc. All Rights Reserved.");
WriteAllLines(ThisFile, Lines.ToArray());
Missing.Add(ThisFile);
}
}
Log("Normal Copyright *************************");
foreach (var ThisFile in Normal)
{
Log(" {0}", ThisFile);
}
Log("Missing Copyright *************************");
foreach (var ThisFile in Missing)
{
Log(" {0}", ThisFile);
}
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "";
}
var LocalizationStepNames = new List <string>();
{
var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepNamesStr == null)
{
LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepNamesStr.Split(','))
{
LocalizationStepNames.Add(StepName.Trim());
}
}
LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var ShouldGatherPlatforms = ParseParam("IncludePlatforms");
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for platforms
if (ShouldGatherPlatforms)
{
var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms"));
if (DirectoryReference.Exists(PlatformsRootDirectory))
{
foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory))
{
// Find the localization targets defined for this platform
var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization")));
if (PlatformTargetNames.Count > 0)
{
var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames));
}
}
}
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory));
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project);
// Add a batch for each plugin that meets our criteria
var AvailablePluginNames = new HashSet <string>();
foreach (var PluginInfo in AllPlugins)
{
AvailablePluginNames.Add(PluginInfo.Name);
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
// If we had an explicit list of plugins to include, warn if any were missing
foreach (string PluginName in IncludePlugins)
{
if (!AvailablePluginNames.Contains(PluginName))
{
LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName);
}
}
}
// Create a single changelist to use for all changes, and hash the current PO files on disk so we can work out whether they actually change
int PendingChangeList = 0;
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
var ChangeListCommitMessage = "Localization Automation";
if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Build/NotForLicensees/EpicInternal.txt")))
{
ChangeListCommitMessage += "\n#okforgithub ignore";
}
PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage);
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Process each localization batch
foreach (var LocalizationBatch in LocalizationBatches)
{
ProcessLocalizationProjects(LocalizationBatch, PendingChangeList, UEProjectRoot, UEProjectName, LocalizationProviderName, LocalizationStepNames, AdditionalCommandletArguments);
}
// Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to)
if (P4Enabled)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
// Submit that single changelist now
if (AllowSubmit)
{
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
}
