public override void ExecuteBuild()
{
string BucketName = ParseRequiredStringParam("Bucket");
FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile");
string CredentialsKey = ParseRequiredStringParam("CredentialsKey");
DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir");
DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir");
int Days = ParseParamInt("Days", 7);
int MaxFileSize = ParseParamInt("MaxFileSize", 0);
string RootManifestPath = ParseRequiredStringParam("Manifest");
string KeyPrefix = ParseParamValue("KeyPrefix", "");
bool bReset = ParseParam("Reset");
// The credentials to upload with
AWSCredentials Credentials;
// Try to get the credentials by the key passed in from the script
CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName);
if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials))
{
throw new AutomationException("Unknown credentials key: {0}", CredentialsKey);
}
// Create the new client
using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region))
{
using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4))
{
// Read the filters
HashSet <string> Paths = new HashSet <string>();
foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt"))
{
TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc;
if (Age < TimeSpan.FromDays(3))
{
Log.TraceInformation("Reading {0}", FilterFile.FullName);
string[] Lines = File.ReadAllLines(FilterFile.FullName);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim().Replace('\\', '/');
if (TrimLine.Length > 0)
{
Paths.Add(TrimLine);
}
}
}
else if (Age > TimeSpan.FromDays(5))
{
try
{
Log.TraceInformation("Deleting {0}", FilterFile.FullName);
FilterFile.Delete();
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to delete: {0}", Ex.Message);
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
}
}
}
Log.TraceInformation("Found {0:n0} files", Paths.Count);
// Enumerate all the files that are in the network DDC
Log.TraceInformation("");
Log.TraceInformation("Filtering files in {0}...", CacheDir);
List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag));
// Filter to the maximum size
if (MaxFileSize != 0)
{
int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize);
Log.TraceInformation("");
Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize);
}
// Create the working directory
DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC");
DirectoryReference.CreateDirectory(WorkingDir);
// Get the path to the manifest
FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath);
// Read the old root manifest
RootManifest OldRootManifest = new RootManifest();
if (FileReference.Exists(RootManifestFile))
{
OldRootManifest.Read(JsonObject.Read(RootManifestFile));
}
// Read the old bundle manifest
BundleManifest OldBundleManifest = new BundleManifest();
if (OldRootManifest.Entries.Count > 0)
{
FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json");
if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest))
{
OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest));
}
}
// Create the new manifest
BundleManifest NewBundleManifest = new BundleManifest();
// Try to download the old manifest, and add all the bundles we want to keep to the new manifest
if (!bReset)
{
foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries)
{
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
if (!FileReference.Exists(BundleFile))
{
Log.TraceInformation("Downloading {0}", BundleFile);
FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz");
if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile))
{
Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey);
continue;
}
FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming");
try
{
DecompressFile(TempCompressedFile, TempUncompressedFile);
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString());
continue;
}
FileReference.Move(TempUncompressedFile, BundleFile);
}
NewBundleManifest.Entries.Add(Bundle);
}
}
// Figure out all the item digests that we already have
Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>();
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
BinaryReader Reader = new BinaryReader(Stream);
uint Signature = Reader.ReadUInt32();
if (Signature != BundleSignatureV1)
{
throw new Exception(String.Format("Invalid signature for {0}", BundleFile));
}
int NumEntries = Reader.ReadInt32();
for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++)
{
byte[] Digest = new byte[ContentHash.LengthSHA1];
if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1)
{
throw new Exception("Unexpected EOF");
}
KeyHashes.Add(new ContentHash(Digest));
Stream.Seek(4, SeekOrigin.Current);
}
}
BundleToKeyHashes[Bundle] = KeyHashes;
}
// Calculate the download size of the manifest
long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
// Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these
// files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times.
foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes)
{
long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length);
if (ValidBundleSize < MinBundleSize)
{
NewBundleManifest.Entries.Remove(Pair.Key);
}
}
// Find all the valid digests
HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]);
}
// Remove all the files which already exist
int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash));
if (NumRemovedExist > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist);
}
// Read all the files we want to include
List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
if (Files.Count > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024));
FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y)));
}
// Generate new data
using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider())
{
// Flag for whether to update the manifest
bool bUpdateManifest = false;
// Upload the new bundle
Log.TraceInformation("");
if (FilesToInclude.Count == 0)
{
Log.TraceInformation("No new files to add.");
}
else
{
// Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together.
Log.TraceInformation("Sorting input files");
List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList();
// Get the target bundle size
long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length);
int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize);
long TargetBundleSize = TotalSize / NumBundles;
// Split the input data into bundles
List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >();
long BundleSize = 0;
for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++)
{
BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]);
BundleSize += SortedFilesToInclude[FileIdx].Item2.Length;
}
BundleFilesToIncludeList.Add(BundleFilesToInclude);
}
// Upload each bundle
DateTime NewBundleTime = DateTime.UtcNow;
for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx];
// Get the new bundle info
string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : "";
string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix);
// Create a random number for the object key
string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto);
// Create the bundle header
byte[] Header;
using (MemoryStream HeaderStream = new MemoryStream())
{
BinaryWriter Writer = new BinaryWriter(HeaderStream);
Writer.Write(BundleSignatureV1);
Writer.Write(BundleFilesToInclude.Count);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1);
Writer.Write((int)FileToInclude.Item2.Length);
}
Header = HeaderStream.ToArray();
}
// Create the output file
FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz");
Log.TraceInformation("Writing {0}", NewBundleFile);
using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true))
{
ZipStream.Write(Header, 0, Header.Length);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length);
}
}
}
// Upload the file
long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length;
long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length);
Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f));
UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null);
// Add the bundle to the new manifest
BundleManifest.Entry Bundle = new BundleManifest.Entry();
Bundle.Name = NewBundleName;
Bundle.ObjectKey = NewBundleObjectKey;
Bundle.Time = NewBundleTime;
Bundle.CompressedLength = (int)NewBundleCompressedLength;
Bundle.UncompressedLength = (int)NewBundleUncompressedLength;
NewBundleManifest.Entries.Add(Bundle);
// Mark the manifest as requiring an update
bUpdateManifest = true;
}
}
// Update the manifest
if (bUpdateManifest)
{
DateTime UtcNow = DateTime.UtcNow;
DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0);
// Update the root manifest
RootManifest NewRootManifest = new RootManifest();
NewRootManifest.AccessKey = OldRootManifest.AccessKey;
NewRootManifest.SecretKey = OldRootManifest.SecretKey;
foreach (RootManifest.Entry Entry in OldRootManifest.Entries)
{
if (Entry.CreateTime >= RemoveBundleManifestsBefore)
{
NewRootManifest.Entries.Add(Entry);
}
}
// Make sure there's an entry for the last 24h
DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0);
if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter))
{
RootManifest.Entry NewEntry = new RootManifest.Entry();
NewEntry.CreateTime = UtcNow;
NewEntry.Key = KeyPrefix + CreateObjectName(Crypto);
NewRootManifest.Entries.Add(NewEntry);
}
// Save out the new bundle manifest
FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json");
NewBundleManifest.Save(NewBundleManifestFile);
// Update all the bundle manifests still valid
foreach (RootManifest.Entry Entry in NewRootManifest.Entries)
{
Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key);
UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null);
}
// Overwrite all the existing manifests
if (AllowSubmit)
{
List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
// Create a changelist containing the new manifest
int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest");
if (ExistingFiles.Count > 0)
{
P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
NewRootManifest.Save(RootManifestFile);
}
else
{
NewRootManifest.Save(RootManifestFile);
P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
}
// Submit it
int SubmittedChangeNumber;
P4.Submit(ChangeNumber, out SubmittedChangeNumber, true);
if (SubmittedChangeNumber <= 0)
{
throw new AutomationException("Failed to submit change");
}
// Delete any bundles that are no longer referenced
HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey));
foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries)
{
if (!KeepObjectKeys.Contains(OldEntry.ObjectKey))
{
Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey);
DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null);
}
}
// Delete any bundle manifests which are no longer referenced
HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key));
foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries)
{
if (!KeepManifestKeys.Contains(OldEntry.Key))
{
Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key);
DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null);
}
}
}
else
{
// Skip submitting
Log.TraceWarning("Skipping manifest submit due to missing -Submit argument.");
}
// Update the new download size
DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
}
}
// Print some stats about the final manifest
Log.TraceInformation("");
Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0));
Log.TraceInformation("");
}
}
}