public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List<string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return Files;
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List<string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List<string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List<string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return DestFiles;
}
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List<string> Files)
{
var Saver = new TempStorageManifest();
Saver.Create(Files, RootDir);
if (Saver.GetFileCount() != Files.Count)
{
throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count);
}
var TempFilename = FinalFilename + ".temp";
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest file already exists {0}", TempFilename);
}
CreateDirectory(true, Path.GetDirectoryName(FinalFilename));
Saver.Save(TempFilename);
var Tester = new TempStorageManifest();
Tester.Load(TempFilename, true);
if (!Saver.Compare(Tester))
{
throw new AutomationException("Temp manifest differs {0}", TempFilename);
}
RenameFile(TempFilename, FinalFilename, true);
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest didn't go away {0}", TempFilename);
}
var FinalTester = new TempStorageManifest();
FinalTester.Load(FinalFilename, true);
if (!Saver.Compare(FinalTester))
{
throw new AutomationException("Final manifest differs {0}", TempFilename);
}
Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize());
return Saver;
}
public static List <string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List <string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return(Files);
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List <string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List <string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List <string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return(DestFiles);
}
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List <string> Files)
{
var Saver = new TempStorageManifest();
Saver.Create(Files, RootDir);
if (Saver.GetFileCount() != Files.Count)
{
throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count);
}
var TempFilename = FinalFilename + ".temp";
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest file already exists {0}", TempFilename);
}
CreateDirectory(true, Path.GetDirectoryName(FinalFilename));
Saver.Save(TempFilename);
var Tester = new TempStorageManifest();
Tester.Load(TempFilename, true);
if (!Saver.Compare(Tester))
{
throw new AutomationException("Temp manifest differs {0}", TempFilename);
}
RenameFile(TempFilename, FinalFilename, true);
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest didn't go away {0}", TempFilename);
}
var FinalTester = new TempStorageManifest();
FinalTester.Load(FinalFilename, true);
if (!Saver.Compare(FinalTester))
{
throw new AutomationException("Final manifest differs {0}", TempFilename);
}
Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize());
return(Saver);
}
/// <summary>
/// Saves the given files (that should be rooted at the branch root) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="NodeName">The node which created the storage block</param>
/// <param name="BlockName">Name of the block to retrieve. May be null or empty.</param>
/// <param name="BuildProducts">Array of build products to be archived</param>
/// <param name="bPushToRemote">Allow skipping the copying of this manifest to shared storage, because it's not required by any other agent</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
public TempStorageManifest Archive(string NodeName, string BlockName, FileReference[] BuildProducts, bool bPushToRemote = true)
{
using(TelemetryStopwatch TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage"))
{
// Create a manifest for the given build products
FileInfo[] Files = BuildProducts.Select(x => new FileInfo(x.FullName)).ToArray();
TempStorageManifest Manifest = new TempStorageManifest(Files, RootDir);
// Create the local directory for this node
DirectoryReference LocalNodeDir = GetDirectoryForNode(LocalDir, NodeName);
LocalNodeDir.CreateDirectory();
// Compress the files and copy to shared storage if necessary
bool bRemote = SharedDir != null && bPushToRemote && bWriteToSharedStorage;
if(bRemote)
{
// Create the shared directory for this node
FileReference SharedManifestFile = GetManifestLocation(SharedDir, NodeName, BlockName);
SharedManifestFile.Directory.CreateDirectory();
// Zip all the build products
FileInfo[] ZipFiles = ParallelZipFiles(Files, RootDir, SharedManifestFile.Directory, LocalNodeDir, SharedManifestFile.GetFileNameWithoutExtension());
Manifest.ZipFiles = ZipFiles.Select(x => new TempStorageZipFile(x)).ToArray();
// Save the shared manifest
CommandUtils.Log("Saving shared manifest to {0}", SharedManifestFile.FullName);
Manifest.Save(SharedManifestFile);
}
// Save the local manifest
FileReference LocalManifestFile = GetManifestLocation(LocalDir, NodeName, BlockName);
CommandUtils.Log("Saving local manifest to {0}", LocalManifestFile.FullName);
Manifest.Save(LocalManifestFile);
// Update the stats
long ZipFilesTotalSize = (Manifest.ZipFiles == null)? 0 : Manifest.ZipFiles.Sum(x => x.Length);
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Files.Length, Manifest.GetTotalSize(), ZipFilesTotalSize, bRemote? "Remote" : "Local", 0, 0, BlockName));
return Manifest;
}
}
/// <summary>
/// Saves the given files (that should be rooted at the branch root) to a shared temp storage manifest with the given temp storage node and game.
/// </summary>
/// <param name="NodeName">The node which these build products belong to</param>
/// <param name="OutputName">The output name of the node.</param>
/// <param name="BuildProducts">Array of build products to be archived</param>
/// <param name="bPushToRemote">Allow skipping the copying of this manifest to shared storage, because it's not required by any other agent</param>
/// <returns>The created manifest instance (which has already been saved to disk).</returns>
public TempStorageManifest Archive(string NodeName, string OutputName, FileReference[] BuildProducts, bool bPushToRemote = true)
{
using (TelemetryStopwatch TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage"))
{
// Create a manifest for the given build products
FileInfo[] Files = BuildProducts.Select(x => new FileInfo(x.FullName)).ToArray();
TempStorageManifest Manifest = new TempStorageManifest(Files, RootDir);
// Create the local directory for this node
DirectoryReference LocalNodeDir = GetDirectoryForNode(LocalDir, NodeName);
LocalNodeDir.CreateDirectory();
// Compress the files and copy to shared storage if necessary
bool bRemote = SharedDir != null && bPushToRemote && bWriteToSharedStorage;
if (bRemote)
{
// Create the shared directory for this node
DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName);
SharedNodeDir.CreateDirectory();
// Zip all the build products
FileInfo[] ZipFiles = ParallelZipFiles(Files, RootDir, SharedNodeDir, LocalNodeDir, OutputName);
Manifest.ZipFiles = ZipFiles.Select(x => new TempStorageZipFile(x)).ToArray();
// Save the shared manifest
FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName);
CommandUtils.Log("Saving shared manifest to {0}", SharedManifestFile.FullName);
Manifest.Save(SharedManifestFile);
}
// Save the local manifest
FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName);
CommandUtils.Log("Saving local manifest to {0}", LocalManifestFile.FullName);
Manifest.Save(LocalManifestFile);
// Update the stats
long ZipFilesTotalSize = (Manifest.ZipFiles == null)? 0 : Manifest.ZipFiles.Sum(x => x.Length);
TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Files.Length, Manifest.GetTotalSize(), ZipFilesTotalSize, bRemote? "Remote" : "Local", 0, 0, OutputName));
return(Manifest);
}
}
