static Patcher CreatePatcher(Options opts)
{
var patcher = new Patcher(opts.WorkingPath);
if (opts.P4Edit)
{
var p4 = new P4(opts.WorkingPath);
patcher.BeforeApplyingPatches += (filepaths) => {
foreach (var f in filepaths)
{
p4.Edit(f);
}
};
}
return(patcher);
}
public override void ExecuteBuild()
{
string[] ProjectParams = ParseParamValues("Project");
string UpdateDirParam = ParseParamValue("UpdateDir", null);
if (UpdateDirParam == null)
{
throw new AutomationException("Missing -UpdateDir=... parameter");
}
DirectoryReference UpdateDir = new DirectoryReference(UpdateDirParam);
bool bWrite = ParseParam("Write");
// Get all the root dirs
HashSet <DirectoryReference> RootDirs = new HashSet <DirectoryReference>();
RootDirs.Add(EngineDirectory);
// Add the enterprise edirectory
DirectoryReference EnterpriseDirectory = DirectoryReference.Combine(RootDirectory, "Enterprise");
if (DirectoryReference.Exists(EnterpriseDirectory))
{
RootDirs.Add(EnterpriseDirectory);
}
// Add the project directories
foreach (string ProjectParam in ProjectParams)
{
FileReference ProjectLocation = new FileReference(ProjectParam);
if (!FileReference.Exists(ProjectLocation))
{
throw new AutomationException("Unable to find project '{0}'", ProjectLocation);
}
RootDirs.Add(ProjectLocation.Directory);
}
// Find all the modules
HashSet <DirectoryReference> ModuleDirs = new HashSet <DirectoryReference>();
foreach (DirectoryReference RootDir in RootDirs)
{
// Find all the modules from the source folder
DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Source");
if (DirectoryReference.Exists(SourceDir))
{
foreach (FileReference ModuleFile in DirectoryReference.EnumerateFiles(SourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(ModuleFile.Directory);
}
}
// Find all the modules under the plugins folder
DirectoryReference PluginsDir = DirectoryReference.Combine(RootDir, "Plugins");
foreach (FileReference PluginFile in DirectoryReference.EnumerateFiles(PluginsDir, "*.uplugin", SearchOption.AllDirectories))
{
DirectoryReference PluginSourceDir = DirectoryReference.Combine(PluginFile.Directory, "Source");
if (DirectoryReference.Exists(PluginSourceDir))
{
foreach (FileReference PluginModuleFile in DirectoryReference.EnumerateFiles(PluginSourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(PluginModuleFile.Directory);
}
}
}
}
// Find a mapping from old to new include paths
Dictionary <string, Tuple <string, FileReference> > RemapIncludePaths = new Dictionary <string, Tuple <string, FileReference> >(StringComparer.InvariantCultureIgnoreCase);
foreach (DirectoryReference ModuleDir in ModuleDirs)
{
DirectoryReference ModulePublicDir = DirectoryReference.Combine(ModuleDir, "Public");
if (DirectoryReference.Exists(ModulePublicDir))
{
foreach (FileReference HeaderFile in DirectoryReference.EnumerateFiles(ModulePublicDir, "*.h", SearchOption.AllDirectories))
{
string BaseIncludeFile = HeaderFile.GetFileName();
Tuple <string, FileReference> ExistingIncludeName;
if (RemapIncludePaths.TryGetValue(BaseIncludeFile, out ExistingIncludeName))
{
LogWarning("Multiple include paths for {0}: {1}, {2}", BaseIncludeFile, ExistingIncludeName.Item2, HeaderFile);
}
else
{
RemapIncludePaths.Add(BaseIncludeFile, Tuple.Create(HeaderFile.MakeRelativeTo(ModulePublicDir).Replace('\\', '/'), HeaderFile));
}
}
}
}
// List of folders to exclude from updates
string[] ExcludeFoldersFromUpdate =
{
"Intermediate",
"ThirdParty"
};
// Enumerate all the files to update
HashSet <FileReference> UpdateFiles = new HashSet <FileReference>();
foreach (FileReference UpdateFile in DirectoryReference.EnumerateFiles(UpdateDir, "*", SearchOption.AllDirectories))
{
if (!UpdateFile.ContainsAnyNames(ExcludeFoldersFromUpdate, UpdateDir))
{
if (UpdateFile.HasExtension(".cpp") | UpdateFile.HasExtension(".h") || UpdateFile.HasExtension(".inl"))
{
UpdateFiles.Add(UpdateFile);
}
}
}
// Process all the source files
Dictionary <FileReference, string[]> ModifiedFiles = new Dictionary <FileReference, string[]>();
foreach (FileReference UpdateFile in UpdateFiles)
{
bool bModifiedFile = false;
string[] Lines = FileReference.ReadAllLines(UpdateFile);
for (int Idx = 0; Idx < Lines.Length; Idx++)
{
Match Match = Regex.Match(Lines[Idx], "^(\\s*#\\s*include\\s+\\\")([^\"]+)(\\\".*)$");
if (Match.Success)
{
string IncludePath = Match.Groups[2].Value;
Tuple <string, FileReference> NewIncludePath;
if (RemapIncludePaths.TryGetValue(IncludePath, out NewIncludePath))
{
if (IncludePath != NewIncludePath.Item1)
{
// Log("{0}: Changing {1} -> {2}", UpdateFile, IncludePath, NewIncludePath.Item1);
Lines[Idx] = String.Format("{0}{1}{2}", Match.Groups[1].Value, NewIncludePath.Item1, Match.Groups[3].Value);
bModifiedFile = true;
}
}
}
}
if (bModifiedFile)
{
ModifiedFiles.Add(UpdateFile, Lines);
}
}
// Output them all to disk
if (bWrite && ModifiedFiles.Count > 0)
{
LogInformation("Updating {0} files...", ModifiedFiles.Count);
List <FileReference> FilesToCheckOut = new List <FileReference>();
foreach (FileReference ModifiedFile in ModifiedFiles.Keys)
{
if ((FileReference.GetAttributes(ModifiedFile) & FileAttributes.ReadOnly) != 0)
{
FilesToCheckOut.Add(ModifiedFile);
}
}
if (FilesToCheckOut.Count > 0)
{
if (!P4Enabled)
{
throw new AutomationException("{0} files have been modified, but are read only. Run with -P4 to enable Perforce checkout.\n{1}", FilesToCheckOut.Count, String.Join("\n", FilesToCheckOut.Select(x => " " + x)));
}
LogInformation("Checking out files from Perforce");
int ChangeNumber = P4.CreateChange(Description: "Updating source files");
P4.Edit(ChangeNumber, FilesToCheckOut.Select(x => x.FullName).ToList(), false);
}
foreach (KeyValuePair <FileReference, string[]> FileToWrite in ModifiedFiles)
{
LogInformation("Writing {0}", FileToWrite.Key);
FileReference.WriteAllLines(FileToWrite.Key, FileToWrite.Value);
}
}
}
public override void ExecuteBuild()
{
string BucketName = ParseRequiredStringParam("Bucket");
FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile");
string CredentialsKey = ParseRequiredStringParam("CredentialsKey");
DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir");
DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir");
int Days = ParseParamInt("Days", 7);
int MaxFileSize = ParseParamInt("MaxFileSize", 0);
string RootManifestPath = ParseRequiredStringParam("Manifest");
string KeyPrefix = ParseParamValue("KeyPrefix", "");
bool bReset = ParseParam("Reset");
// The credentials to upload with
AWSCredentials Credentials;
// Try to get the credentials by the key passed in from the script
CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName);
if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials))
{
throw new AutomationException("Unknown credentials key: {0}", CredentialsKey);
}
// Create the new client
using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region))
{
using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4))
{
// Read the filters
HashSet <string> Paths = new HashSet <string>();
foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt"))
{
TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc;
if (Age < TimeSpan.FromDays(3))
{
Log.TraceInformation("Reading {0}", FilterFile.FullName);
string[] Lines = File.ReadAllLines(FilterFile.FullName);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim().Replace('\\', '/');
if (TrimLine.Length > 0)
{
Paths.Add(TrimLine);
}
}
}
else if (Age > TimeSpan.FromDays(5))
{
try
{
Log.TraceInformation("Deleting {0}", FilterFile.FullName);
FilterFile.Delete();
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to delete: {0}", Ex.Message);
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
}
}
}
Log.TraceInformation("Found {0:n0} files", Paths.Count);
// Enumerate all the files that are in the network DDC
Log.TraceInformation("");
Log.TraceInformation("Filtering files in {0}...", CacheDir);
List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag));
// Filter to the maximum size
if (MaxFileSize != 0)
{
int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize);
Log.TraceInformation("");
Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize);
}
// Create the working directory
DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC");
DirectoryReference.CreateDirectory(WorkingDir);
// Get the path to the manifest
FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath);
// Read the old root manifest
RootManifest OldRootManifest = new RootManifest();
if (FileReference.Exists(RootManifestFile))
{
OldRootManifest.Read(JsonObject.Read(RootManifestFile));
}
// Read the old bundle manifest
BundleManifest OldBundleManifest = new BundleManifest();
if (OldRootManifest.Entries.Count > 0)
{
FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json");
if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest))
{
OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest));
}
}
// Create the new manifest
BundleManifest NewBundleManifest = new BundleManifest();
// Try to download the old manifest, and add all the bundles we want to keep to the new manifest
if (!bReset)
{
foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries)
{
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
if (!FileReference.Exists(BundleFile))
{
Log.TraceInformation("Downloading {0}", BundleFile);
FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz");
if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile))
{
Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey);
continue;
}
FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming");
try
{
DecompressFile(TempCompressedFile, TempUncompressedFile);
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString());
continue;
}
FileReference.Move(TempUncompressedFile, BundleFile);
}
NewBundleManifest.Entries.Add(Bundle);
}
}
// Figure out all the item digests that we already have
Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>();
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
BinaryReader Reader = new BinaryReader(Stream);
uint Signature = Reader.ReadUInt32();
if (Signature != BundleSignatureV1)
{
throw new Exception(String.Format("Invalid signature for {0}", BundleFile));
}
int NumEntries = Reader.ReadInt32();
for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++)
{
byte[] Digest = new byte[ContentHash.LengthSHA1];
if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1)
{
throw new Exception("Unexpected EOF");
}
KeyHashes.Add(new ContentHash(Digest));
Stream.Seek(4, SeekOrigin.Current);
}
}
BundleToKeyHashes[Bundle] = KeyHashes;
}
// Calculate the download size of the manifest
long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
// Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these
// files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times.
foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes)
{
long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length);
if (ValidBundleSize < MinBundleSize)
{
NewBundleManifest.Entries.Remove(Pair.Key);
}
}
// Find all the valid digests
HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]);
}
// Remove all the files which already exist
int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash));
if (NumRemovedExist > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist);
}
// Read all the files we want to include
List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
if (Files.Count > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024));
FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y)));
}
// Generate new data
using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider())
{
// Flag for whether to update the manifest
bool bUpdateManifest = false;
// Upload the new bundle
Log.TraceInformation("");
if (FilesToInclude.Count == 0)
{
Log.TraceInformation("No new files to add.");
}
else
{
// Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together.
Log.TraceInformation("Sorting input files");
List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList();
// Get the target bundle size
long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length);
int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize);
long TargetBundleSize = TotalSize / NumBundles;
// Split the input data into bundles
List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >();
long BundleSize = 0;
for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++)
{
BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]);
BundleSize += SortedFilesToInclude[FileIdx].Item2.Length;
}
BundleFilesToIncludeList.Add(BundleFilesToInclude);
}
// Upload each bundle
DateTime NewBundleTime = DateTime.UtcNow;
for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx];
// Get the new bundle info
string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : "";
string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix);
// Create a random number for the object key
string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto);
// Create the bundle header
byte[] Header;
using (MemoryStream HeaderStream = new MemoryStream())
{
BinaryWriter Writer = new BinaryWriter(HeaderStream);
Writer.Write(BundleSignatureV1);
Writer.Write(BundleFilesToInclude.Count);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1);
Writer.Write((int)FileToInclude.Item2.Length);
}
Header = HeaderStream.ToArray();
}
// Create the output file
FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz");
Log.TraceInformation("Writing {0}", NewBundleFile);
using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true))
{
ZipStream.Write(Header, 0, Header.Length);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length);
}
}
}
// Upload the file
long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length;
long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length);
Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f));
UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null);
// Add the bundle to the new manifest
BundleManifest.Entry Bundle = new BundleManifest.Entry();
Bundle.Name = NewBundleName;
Bundle.ObjectKey = NewBundleObjectKey;
Bundle.Time = NewBundleTime;
Bundle.CompressedLength = (int)NewBundleCompressedLength;
Bundle.UncompressedLength = (int)NewBundleUncompressedLength;
NewBundleManifest.Entries.Add(Bundle);
// Mark the manifest as requiring an update
bUpdateManifest = true;
}
}
// Update the manifest
if (bUpdateManifest)
{
DateTime UtcNow = DateTime.UtcNow;
DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0);
// Update the root manifest
RootManifest NewRootManifest = new RootManifest();
NewRootManifest.AccessKey = OldRootManifest.AccessKey;
NewRootManifest.SecretKey = OldRootManifest.SecretKey;
foreach (RootManifest.Entry Entry in OldRootManifest.Entries)
{
if (Entry.CreateTime >= RemoveBundleManifestsBefore)
{
NewRootManifest.Entries.Add(Entry);
}
}
// Make sure there's an entry for the last 24h
DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0);
if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter))
{
RootManifest.Entry NewEntry = new RootManifest.Entry();
NewEntry.CreateTime = UtcNow;
NewEntry.Key = KeyPrefix + CreateObjectName(Crypto);
NewRootManifest.Entries.Add(NewEntry);
}
// Save out the new bundle manifest
FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json");
NewBundleManifest.Save(NewBundleManifestFile);
// Update all the bundle manifests still valid
foreach (RootManifest.Entry Entry in NewRootManifest.Entries)
{
Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key);
UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null);
}
// Overwrite all the existing manifests
if (AllowSubmit)
{
List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
// Create a changelist containing the new manifest
int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest");
if (ExistingFiles.Count > 0)
{
P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
NewRootManifest.Save(RootManifestFile);
}
else
{
NewRootManifest.Save(RootManifestFile);
P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
}
// Submit it
int SubmittedChangeNumber;
P4.Submit(ChangeNumber, out SubmittedChangeNumber, true);
if (SubmittedChangeNumber <= 0)
{
throw new AutomationException("Failed to submit change");
}
// Delete any bundles that are no longer referenced
HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey));
foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries)
{
if (!KeepObjectKeys.Contains(OldEntry.ObjectKey))
{
Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey);
DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null);
}
}
// Delete any bundle manifests which are no longer referenced
HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key));
foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries)
{
if (!KeepManifestKeys.Contains(OldEntry.Key))
{
Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key);
DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null);
}
}
}
else
{
// Skip submitting
Log.TraceWarning("Skipping manifest submit due to missing -Submit argument.");
}
// Update the new download size
DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
}
}
// Print some stats about the final manifest
Log.TraceInformation("");
Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0));
Log.TraceInformation("");
}
}
}
