protected T Retry <T>(Func <T> FuncToRetry, int MaxAttempts)
{
int NumAttempts = 0;
for (; ;)
{
try
{
NumAttempts++;
return(FuncToRetry());
}
catch (Exception Ex)
{
if (NumAttempts < MaxAttempts)
{
Log.TraceLog("Attempt {0}: {1}", NumAttempts, ExceptionUtils.FormatExceptionDetails(Ex));
}
else
{
throw;
}
}
Thread.Sleep(TimeSpan.FromSeconds(5.0));
}
}
protected void DeleteFile(AmazonS3Client Client, string BucketName, string Key, SemaphoreSlim Semaphore, DeleteState State)
{
try
{
DeleteFileInner(Client, BucketName, Key, Semaphore, State);
}
catch (Exception Ex)
{
Log.TraceWarning("Exception trying to delete {0}: {1}", Key, ExceptionUtils.FormatExceptionDetails(Ex));
}
}
protected void UploadFile(AmazonS3Client Client, string BucketName, FileReference File, long Length, string Key, SemaphoreSlim Semaphore, UploadState State)
{
try
{
Retry(() => UploadFileInner(Client, BucketName, File, Length, Key, Semaphore, State), 10);
}
catch (Exception Ex)
{
Log.TraceWarning("Exception trying to upload {0}: {1}", File, ExceptionUtils.FormatExceptionDetails(Ex));
}
}
static Repository CreateOrLoadRepository(string ServerAndPort, string UserName, DirectoryReference BaseDir, bool bOverwrite)
{
if (Repository.Exists(BaseDir))
{
try
{
return(Repository.Load(ServerAndPort, UserName, BaseDir));
}
catch (Exception Ex)
{
if (bOverwrite)
{
Log.TraceWarning("Unable to load existing repository: {0}", Ex.Message);
Log.TraceVerbose("{0}", ExceptionUtils.FormatExceptionDetails(Ex));
}
else
{
throw;
}
}
}
return(Repository.Create(ServerAndPort, UserName, BaseDir));
}
public static int Main(string[] Arguments)
{
// Ensure UTF8Output flag is respected, since we are initializing logging early in the program.
if (CommandUtils.ParseParam(Arguments, "-Utf8output"))
{
Console.OutputEncoding = new System.Text.UTF8Encoding(false, false);
}
// Parse the log level argument
if (CommandUtils.ParseParam(Arguments, "-Verbose"))
{
Log.OutputLevel = LogEventType.Verbose;
}
if (CommandUtils.ParseParam(Arguments, "-VeryVerbose"))
{
Log.OutputLevel = LogEventType.VeryVerbose;
}
// Initialize the log system, buffering the output until we can create the log file
StartupTraceListener StartupListener = new StartupTraceListener();
Trace.Listeners.Add(StartupListener);
// Configure log timestamps
Log.IncludeTimestamps = CommandUtils.ParseParam(Arguments, "-Timestamps");
// Enter the main program section
ExitCode ReturnCode = ExitCode.Success;
try
{
// Set the working directory to the UE4 root
Environment.CurrentDirectory = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().GetOriginalLocation()), "..", "..", ".."));
// Ensure we can resolve any external assemblies as necessary.
string PathToBinariesDotNET = Path.GetDirectoryName(Assembly.GetEntryAssembly().GetOriginalLocation());
AssemblyUtils.InstallAssemblyResolver(PathToBinariesDotNET);
AssemblyUtils.InstallRecursiveAssemblyResolver(PathToBinariesDotNET);
// Initialize the host platform layer
HostPlatform.Initialize();
// Log the operating environment. Since we usually compile to AnyCPU, we may be executed using different system paths under WOW64.
Log.TraceVerbose("{2}: Running on {0} as a {1}-bit process.", HostPlatform.Current.GetType().Name, Environment.Is64BitProcess ? 64 : 32, DateTime.UtcNow.ToString("o"));
// Log if we're running from the launcher
string ExecutingAssemblyLocation = Assembly.GetExecutingAssembly().Location;
if (string.Compare(ExecutingAssemblyLocation, Assembly.GetEntryAssembly().GetOriginalLocation(), StringComparison.OrdinalIgnoreCase) != 0)
{
Log.TraceVerbose("Executed from AutomationToolLauncher ({0})", ExecutingAssemblyLocation);
}
Log.TraceVerbose("CWD={0}", Environment.CurrentDirectory);
// Hook up exit callbacks
AppDomain Domain = AppDomain.CurrentDomain;
Domain.ProcessExit += Domain_ProcessExit;
Domain.DomainUnload += Domain_ProcessExit;
HostPlatform.Current.SetConsoleCtrlHandler(CtrlHandlerDelegateInstance);
// Log the application version
FileVersionInfo Version = AssemblyUtils.ExecutableVersion;
Log.TraceVerbose("{0} ver. {1}", Version.ProductName, Version.ProductVersion);
// Don't allow simultaneous execution of AT (in the same branch)
ReturnCode = InternalUtils.RunSingleInstance(() => MainProc(Arguments, StartupListener));
}
catch (AutomationException Ex)
{
// Output the message in the desired format
if (Ex.OutputFormat == AutomationExceptionOutputFormat.Silent)
{
Log.TraceLog("{0}", ExceptionUtils.FormatExceptionDetails(Ex));
}
else if (Ex.OutputFormat == AutomationExceptionOutputFormat.Minimal)
{
Log.TraceInformation("{0}", Ex.ToString().Replace("\n", "\n "));
Log.TraceLog("{0}", ExceptionUtils.FormatExceptionDetails(Ex));
}
else
{
Log.WriteException(Ex, LogUtils.FinalLogFileName);
}
// Take the exit code from the exception
ReturnCode = Ex.ErrorCode;
}
catch (Exception Ex)
{
// Use a default exit code
Log.WriteException(Ex, LogUtils.FinalLogFileName);
ReturnCode = ExitCode.Error_Unknown;
}
finally
{
// In all cases, do necessary shut down stuff, but don't let any additional exceptions leak out while trying to shut down.
// Make sure there's no directories on the stack.
NoThrow(() => CommandUtils.ClearDirStack(), "Clear Dir Stack");
// Try to kill process before app domain exits to leave the other KillAll call to extreme edge cases
NoThrow(() => { if (ShouldKillProcesses && !Utils.IsRunningOnMono)
{
ProcessManager.KillAll();
}
}, "Kill All Processes");
// Write the exit code
Log.TraceInformation("AutomationTool exiting with ExitCode={0} ({1})", (int)ReturnCode, ReturnCode);
// Can't use NoThrow here because the code logs exceptions. We're shutting down logging!
Trace.Close();
}
return((int)ReturnCode);
}
public override void ExecuteBuild()
{
string BucketName = ParseRequiredStringParam("Bucket");
FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile");
string CredentialsKey = ParseRequiredStringParam("CredentialsKey");
DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir");
DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir");
int Days = ParseParamInt("Days", 7);
int MaxFileSize = ParseParamInt("MaxFileSize", 0);
string RootManifestPath = ParseRequiredStringParam("Manifest");
string KeyPrefix = ParseParamValue("KeyPrefix", "");
bool bReset = ParseParam("Reset");
// The credentials to upload with
AWSCredentials Credentials;
// Try to get the credentials by the key passed in from the script
CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName);
if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials))
{
throw new AutomationException("Unknown credentials key: {0}", CredentialsKey);
}
// Create the new client
using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region))
{
using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4))
{
// Read the filters
HashSet <string> Paths = new HashSet <string>();
foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt"))
{
TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc;
if (Age < TimeSpan.FromDays(3))
{
Log.TraceInformation("Reading {0}", FilterFile.FullName);
string[] Lines = File.ReadAllLines(FilterFile.FullName);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim().Replace('\\', '/');
if (TrimLine.Length > 0)
{
Paths.Add(TrimLine);
}
}
}
else if (Age > TimeSpan.FromDays(5))
{
try
{
Log.TraceInformation("Deleting {0}", FilterFile.FullName);
FilterFile.Delete();
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to delete: {0}", Ex.Message);
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
}
}
}
Log.TraceInformation("Found {0:n0} files", Paths.Count);
// Enumerate all the files that are in the network DDC
Log.TraceInformation("");
Log.TraceInformation("Filtering files in {0}...", CacheDir);
List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag));
// Filter to the maximum size
if (MaxFileSize != 0)
{
int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize);
Log.TraceInformation("");
Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize);
}
// Create the working directory
DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC");
DirectoryReference.CreateDirectory(WorkingDir);
// Get the path to the manifest
FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath);
// Read the old root manifest
RootManifest OldRootManifest = new RootManifest();
if (FileReference.Exists(RootManifestFile))
{
OldRootManifest.Read(JsonObject.Read(RootManifestFile));
}
// Read the old bundle manifest
BundleManifest OldBundleManifest = new BundleManifest();
if (OldRootManifest.Entries.Count > 0)
{
FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json");
if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest))
{
OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest));
}
}
// Create the new manifest
BundleManifest NewBundleManifest = new BundleManifest();
// Try to download the old manifest, and add all the bundles we want to keep to the new manifest
if (!bReset)
{
foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries)
{
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
if (!FileReference.Exists(BundleFile))
{
Log.TraceInformation("Downloading {0}", BundleFile);
FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz");
if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile))
{
Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey);
continue;
}
FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming");
try
{
DecompressFile(TempCompressedFile, TempUncompressedFile);
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString());
continue;
}
FileReference.Move(TempUncompressedFile, BundleFile);
}
NewBundleManifest.Entries.Add(Bundle);
}
}
// Figure out all the item digests that we already have
Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>();
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
BinaryReader Reader = new BinaryReader(Stream);
uint Signature = Reader.ReadUInt32();
if (Signature != BundleSignatureV1)
{
throw new Exception(String.Format("Invalid signature for {0}", BundleFile));
}
int NumEntries = Reader.ReadInt32();
for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++)
{
byte[] Digest = new byte[ContentHash.LengthSHA1];
if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1)
{
throw new Exception("Unexpected EOF");
}
KeyHashes.Add(new ContentHash(Digest));
Stream.Seek(4, SeekOrigin.Current);
}
}
BundleToKeyHashes[Bundle] = KeyHashes;
}
// Calculate the download size of the manifest
long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
// Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these
// files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times.
foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes)
{
long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length);
if (ValidBundleSize < MinBundleSize)
{
NewBundleManifest.Entries.Remove(Pair.Key);
}
}
// Find all the valid digests
HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]);
}
// Remove all the files which already exist
int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash));
if (NumRemovedExist > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist);
}
// Read all the files we want to include
List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
if (Files.Count > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024));
FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y)));
}
// Generate new data
using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider())
{
// Flag for whether to update the manifest
bool bUpdateManifest = false;
// Upload the new bundle
Log.TraceInformation("");
if (FilesToInclude.Count == 0)
{
Log.TraceInformation("No new files to add.");
}
else
{
// Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together.
Log.TraceInformation("Sorting input files");
List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList();
// Get the target bundle size
long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length);
int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize);
long TargetBundleSize = TotalSize / NumBundles;
// Split the input data into bundles
List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >();
long BundleSize = 0;
for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++)
{
BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]);
BundleSize += SortedFilesToInclude[FileIdx].Item2.Length;
}
BundleFilesToIncludeList.Add(BundleFilesToInclude);
}
// Upload each bundle
DateTime NewBundleTime = DateTime.UtcNow;
for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx];
// Get the new bundle info
string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : "";
string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix);
// Create a random number for the object key
string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto);
// Create the bundle header
byte[] Header;
using (MemoryStream HeaderStream = new MemoryStream())
{
BinaryWriter Writer = new BinaryWriter(HeaderStream);
Writer.Write(BundleSignatureV1);
Writer.Write(BundleFilesToInclude.Count);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1);
Writer.Write((int)FileToInclude.Item2.Length);
}
Header = HeaderStream.ToArray();
}
// Create the output file
FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz");
Log.TraceInformation("Writing {0}", NewBundleFile);
using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true))
{
ZipStream.Write(Header, 0, Header.Length);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length);
}
}
}
// Upload the file
long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length;
long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length);
Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f));
UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null);
// Add the bundle to the new manifest
BundleManifest.Entry Bundle = new BundleManifest.Entry();
Bundle.Name = NewBundleName;
Bundle.ObjectKey = NewBundleObjectKey;
Bundle.Time = NewBundleTime;
Bundle.CompressedLength = (int)NewBundleCompressedLength;
Bundle.UncompressedLength = (int)NewBundleUncompressedLength;
NewBundleManifest.Entries.Add(Bundle);
// Mark the manifest as requiring an update
bUpdateManifest = true;
}
}
// Update the manifest
if (bUpdateManifest)
{
DateTime UtcNow = DateTime.UtcNow;
DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0);
// Update the root manifest
RootManifest NewRootManifest = new RootManifest();
NewRootManifest.AccessKey = OldRootManifest.AccessKey;
NewRootManifest.SecretKey = OldRootManifest.SecretKey;
foreach (RootManifest.Entry Entry in OldRootManifest.Entries)
{
if (Entry.CreateTime >= RemoveBundleManifestsBefore)
{
NewRootManifest.Entries.Add(Entry);
}
}
// Make sure there's an entry for the last 24h
DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0);
if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter))
{
RootManifest.Entry NewEntry = new RootManifest.Entry();
NewEntry.CreateTime = UtcNow;
NewEntry.Key = KeyPrefix + CreateObjectName(Crypto);
NewRootManifest.Entries.Add(NewEntry);
}
// Save out the new bundle manifest
FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json");
NewBundleManifest.Save(NewBundleManifestFile);
// Update all the bundle manifests still valid
foreach (RootManifest.Entry Entry in NewRootManifest.Entries)
{
Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key);
UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null);
}
// Overwrite all the existing manifests
if (AllowSubmit)
{
List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
// Create a changelist containing the new manifest
int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest");
if (ExistingFiles.Count > 0)
{
P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
NewRootManifest.Save(RootManifestFile);
}
else
{
NewRootManifest.Save(RootManifestFile);
P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
}
// Submit it
int SubmittedChangeNumber;
P4.Submit(ChangeNumber, out SubmittedChangeNumber, true);
if (SubmittedChangeNumber <= 0)
{
throw new AutomationException("Failed to submit change");
}
// Delete any bundles that are no longer referenced
HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey));
foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries)
{
if (!KeepObjectKeys.Contains(OldEntry.ObjectKey))
{
Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey);
DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null);
}
}
// Delete any bundle manifests which are no longer referenced
HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key));
foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries)
{
if (!KeepManifestKeys.Contains(OldEntry.Key))
{
Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key);
DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null);
}
}
}
else
{
// Skip submitting
Log.TraceWarning("Skipping manifest submit due to missing -Submit argument.");
}
// Update the new download size
DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
}
}
// Print some stats about the final manifest
Log.TraceInformation("");
Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0));
Log.TraceInformation("");
}
}
}