/// <summary> /// Saves the given files (that should be rooted at the branch root) to a shared temp storage manifest with the given temp storage node and game. /// </summary> /// <param name="NodeName">The node which these build products belong to</param> /// <param name="OutputName">The output name of the node.</param> /// <param name="BuildProducts">Array of build products to be archived</param> /// <param name="bPushToRemote">Allow skipping the copying of this manifest to shared storage, because it's not required by any other agent</param> /// <returns>The created manifest instance (which has already been saved to disk).</returns> public TempStorageManifest Archive(string NodeName, string OutputName, FileReference[] BuildProducts, bool bPushToRemote = true) { using (TelemetryStopwatch TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage")) { // Create a manifest for the given build products FileInfo[] Files = BuildProducts.Select(x => new FileInfo(x.FullName)).ToArray(); TempStorageManifest Manifest = new TempStorageManifest(Files, RootDir); // Create the local directory for this node DirectoryReference LocalNodeDir = GetDirectoryForNode(LocalDir, NodeName); LocalNodeDir.CreateDirectory(); // Compress the files and copy to shared storage if necessary bool bRemote = SharedDir != null && bPushToRemote && bWriteToSharedStorage; if (bRemote) { // Create the shared directory for this node DirectoryReference SharedNodeDir = GetDirectoryForNode(SharedDir, NodeName); SharedNodeDir.CreateDirectory(); // Zip all the build products FileInfo[] ZipFiles = ParallelZipFiles(Files, RootDir, SharedNodeDir, LocalNodeDir, OutputName); Manifest.ZipFiles = ZipFiles.Select(x => new TempStorageZipFile(x)).ToArray(); // Save the shared manifest FileReference SharedManifestFile = GetManifestFile(SharedDir, NodeName, OutputName); CommandUtils.Log("Saving shared manifest to {0}", SharedManifestFile.FullName); Manifest.Save(SharedManifestFile); } // Save the local manifest FileReference LocalManifestFile = GetManifestFile(LocalDir, NodeName, OutputName); CommandUtils.Log("Saving local manifest to {0}", LocalManifestFile.FullName); Manifest.Save(LocalManifestFile); // Update the stats long ZipFilesTotalSize = (Manifest.ZipFiles == null)? 0 : Manifest.ZipFiles.Sum(x => x.Length); TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Files.Length, Manifest.GetTotalSize(), ZipFilesTotalSize, bRemote? "Remote" : "Local", 0, 0, OutputName)); return(Manifest); } }
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List <string> Files) { var Saver = new TempStorageManifest(); Saver.Create(Files, RootDir); if (Saver.GetFileCount() != Files.Count) { throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count); } var TempFilename = FinalFilename + ".temp"; if (FileExists_NoExceptions(true, TempFilename)) { throw new AutomationException("Temp manifest file already exists {0}", TempFilename); } CreateDirectory(true, Path.GetDirectoryName(FinalFilename)); Saver.Save(TempFilename); var Tester = new TempStorageManifest(); Tester.Load(TempFilename, true); if (!Saver.Compare(Tester)) { throw new AutomationException("Temp manifest differs {0}", TempFilename); } RenameFile(TempFilename, FinalFilename, true); if (FileExists_NoExceptions(true, TempFilename)) { throw new AutomationException("Temp manifest didn't go away {0}", TempFilename); } var FinalTester = new TempStorageManifest(); FinalTester.Load(FinalFilename, true); if (!Saver.Compare(FinalTester)) { throw new AutomationException("Final manifest differs {0}", TempFilename); } Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize()); return(Saver); }
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List<string> Files) { var Saver = new TempStorageManifest(); Saver.Create(Files, RootDir); if (Saver.GetFileCount() != Files.Count) { throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count); } var TempFilename = FinalFilename + ".temp"; if (FileExists_NoExceptions(true, TempFilename)) { throw new AutomationException("Temp manifest file already exists {0}", TempFilename); } CreateDirectory(true, Path.GetDirectoryName(FinalFilename)); Saver.Save(TempFilename); var Tester = new TempStorageManifest(); Tester.Load(TempFilename, true); if (!Saver.Compare(Tester)) { throw new AutomationException("Temp manifest differs {0}", TempFilename); } RenameFile(TempFilename, FinalFilename, true); if (FileExists_NoExceptions(true, TempFilename)) { throw new AutomationException("Temp manifest didn't go away {0}", TempFilename); } var FinalTester = new TempStorageManifest(); FinalTester.Load(FinalFilename, true); if (!Saver.Compare(FinalTester)) { throw new AutomationException("Final manifest differs {0}", TempFilename); } Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize()); return Saver; }
/// <summary> /// Saves the given files (that should be rooted at the branch root) to a shared temp storage manifest with the given temp storage node and game. /// </summary> /// <param name="NodeName">The node which created the storage block</param> /// <param name="BlockName">Name of the block to retrieve. May be null or empty.</param> /// <param name="BuildProducts">Array of build products to be archived</param> /// <param name="bPushToRemote">Allow skipping the copying of this manifest to shared storage, because it's not required by any other agent</param> /// <returns>The created manifest instance (which has already been saved to disk).</returns> public TempStorageManifest Archive(string NodeName, string BlockName, FileReference[] BuildProducts, bool bPushToRemote = true) { using(TelemetryStopwatch TelemetryStopwatch = new TelemetryStopwatch("StoreToTempStorage")) { // Create a manifest for the given build products FileInfo[] Files = BuildProducts.Select(x => new FileInfo(x.FullName)).ToArray(); TempStorageManifest Manifest = new TempStorageManifest(Files, RootDir); // Create the local directory for this node DirectoryReference LocalNodeDir = GetDirectoryForNode(LocalDir, NodeName); LocalNodeDir.CreateDirectory(); // Compress the files and copy to shared storage if necessary bool bRemote = SharedDir != null && bPushToRemote && bWriteToSharedStorage; if(bRemote) { // Create the shared directory for this node FileReference SharedManifestFile = GetManifestLocation(SharedDir, NodeName, BlockName); SharedManifestFile.Directory.CreateDirectory(); // Zip all the build products FileInfo[] ZipFiles = ParallelZipFiles(Files, RootDir, SharedManifestFile.Directory, LocalNodeDir, SharedManifestFile.GetFileNameWithoutExtension()); Manifest.ZipFiles = ZipFiles.Select(x => new TempStorageZipFile(x)).ToArray(); // Save the shared manifest CommandUtils.Log("Saving shared manifest to {0}", SharedManifestFile.FullName); Manifest.Save(SharedManifestFile); } // Save the local manifest FileReference LocalManifestFile = GetManifestLocation(LocalDir, NodeName, BlockName); CommandUtils.Log("Saving local manifest to {0}", LocalManifestFile.FullName); Manifest.Save(LocalManifestFile); // Update the stats long ZipFilesTotalSize = (Manifest.ZipFiles == null)? 0 : Manifest.ZipFiles.Sum(x => x.Length); TelemetryStopwatch.Finish(string.Format("StoreToTempStorage.{0}.{1}.{2}.{3}.{4}.{5}.{6}", Files.Length, Manifest.GetTotalSize(), ZipFilesTotalSize, bRemote? "Remote" : "Local", 0, 0, BlockName)); return Manifest; } }