Ejemplo n.º 1
0
        public override void ExecuteBuild()
        {
            string             BucketName      = ParseRequiredStringParam("Bucket");
            FileReference      CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile");
            string             CredentialsKey  = ParseRequiredStringParam("CredentialsKey");
            DirectoryReference CacheDir        = ParseRequiredDirectoryReferenceParam("CacheDir");
            DirectoryReference FilterDir       = ParseRequiredDirectoryReferenceParam("FilterDir");
            int    Days             = ParseParamInt("Days", 7);
            int    MaxFileSize      = ParseParamInt("MaxFileSize", 0);
            string RootManifestPath = ParseRequiredStringParam("Manifest");
            string KeyPrefix        = ParseParamValue("KeyPrefix", "");
            bool   bReset           = ParseParam("Reset");

            // The credentials to upload with
            AWSCredentials Credentials;

            // Try to get the credentials by the key passed in from the script
            CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName);

            if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials))
            {
                throw new AutomationException("Unknown credentials key: {0}", CredentialsKey);
            }

            // Create the new client
            using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region))
            {
                using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4))
                {
                    // Read the filters
                    HashSet <string> Paths = new HashSet <string>();
                    foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt"))
                    {
                        TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc;
                        if (Age < TimeSpan.FromDays(3))
                        {
                            Log.TraceInformation("Reading {0}", FilterFile.FullName);

                            string[] Lines = File.ReadAllLines(FilterFile.FullName);
                            foreach (string Line in Lines)
                            {
                                string TrimLine = Line.Trim().Replace('\\', '/');
                                if (TrimLine.Length > 0)
                                {
                                    Paths.Add(TrimLine);
                                }
                            }
                        }
                        else if (Age > TimeSpan.FromDays(5))
                        {
                            try
                            {
                                Log.TraceInformation("Deleting {0}", FilterFile.FullName);
                                FilterFile.Delete();
                            }
                            catch (Exception Ex)
                            {
                                Log.TraceWarning("Unable to delete: {0}", Ex.Message);
                                Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
                            }
                        }
                    }
                    Log.TraceInformation("Found {0:n0} files", Paths.Count);

                    // Enumerate all the files that are in the network DDC
                    Log.TraceInformation("");
                    Log.TraceInformation("Filtering files in {0}...", CacheDir);
                    List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag));

                    // Filter to the maximum size
                    if (MaxFileSize != 0)
                    {
                        int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize);
                        Log.TraceInformation("");
                        Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize);
                    }

                    // Create the working directory
                    DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC");
                    DirectoryReference.CreateDirectory(WorkingDir);

                    // Get the path to the manifest
                    FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath);

                    // Read the old root manifest
                    RootManifest OldRootManifest = new RootManifest();
                    if (FileReference.Exists(RootManifestFile))
                    {
                        OldRootManifest.Read(JsonObject.Read(RootManifestFile));
                    }

                    // Read the old bundle manifest
                    BundleManifest OldBundleManifest = new BundleManifest();
                    if (OldRootManifest.Entries.Count > 0)
                    {
                        FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json");
                        if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest))
                        {
                            OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest));
                        }
                    }

                    // Create the new manifest
                    BundleManifest NewBundleManifest = new BundleManifest();

                    // Try to download the old manifest, and add all the bundles we want to keep to the new manifest
                    if (!bReset)
                    {
                        foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries)
                        {
                            FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
                            if (!FileReference.Exists(BundleFile))
                            {
                                Log.TraceInformation("Downloading {0}", BundleFile);

                                FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz");
                                if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile))
                                {
                                    Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey);
                                    continue;
                                }

                                FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming");
                                try
                                {
                                    DecompressFile(TempCompressedFile, TempUncompressedFile);
                                }
                                catch (Exception Ex)
                                {
                                    Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString());
                                    continue;
                                }

                                FileReference.Move(TempUncompressedFile, BundleFile);
                            }
                            NewBundleManifest.Entries.Add(Bundle);
                        }
                    }

                    // Figure out all the item digests that we already have
                    Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >();
                    foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
                    {
                        HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>();

                        FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
                        using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
                        {
                            BinaryReader Reader = new BinaryReader(Stream);

                            uint Signature = Reader.ReadUInt32();
                            if (Signature != BundleSignatureV1)
                            {
                                throw new Exception(String.Format("Invalid signature for {0}", BundleFile));
                            }

                            int NumEntries = Reader.ReadInt32();
                            for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++)
                            {
                                byte[] Digest = new byte[ContentHash.LengthSHA1];
                                if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1)
                                {
                                    throw new Exception("Unexpected EOF");
                                }
                                KeyHashes.Add(new ContentHash(Digest));
                                Stream.Seek(4, SeekOrigin.Current);
                            }
                        }

                        BundleToKeyHashes[Bundle] = KeyHashes;
                    }

                    // Calculate the download size of the manifest
                    long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);

                    // Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these
                    // files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times.
                    foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes)
                    {
                        long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length);
                        if (ValidBundleSize < MinBundleSize)
                        {
                            NewBundleManifest.Entries.Remove(Pair.Key);
                        }
                    }

                    // Find all the valid digests
                    HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>();
                    foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
                    {
                        ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]);
                    }

                    // Remove all the files which already exist
                    int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash));
                    if (NumRemovedExist > 0)
                    {
                        Log.TraceInformation("");
                        Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist);
                    }

                    // Read all the files we want to include
                    List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
                    if (Files.Count > 0)
                    {
                        Log.TraceInformation("");
                        Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024));
                        FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y)));
                    }

                    // Generate new data
                    using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider())
                    {
                        // Flag for whether to update the manifest
                        bool bUpdateManifest = false;

                        // Upload the new bundle
                        Log.TraceInformation("");
                        if (FilesToInclude.Count == 0)
                        {
                            Log.TraceInformation("No new files to add.");
                        }
                        else
                        {
                            // Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together.
                            Log.TraceInformation("Sorting input files");
                            List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList();

                            // Get the target bundle size
                            long TotalSize        = SortedFilesToInclude.Sum(x => (long)x.Item2.Length);
                            int  NumBundles       = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize);
                            long TargetBundleSize = TotalSize / NumBundles;

                            // Split the input data into bundles
                            List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >();
                            long BundleSize = 0;
                            for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize)
                            {
                                List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
                                for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++)
                                {
                                    BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]);
                                    BundleSize += SortedFilesToInclude[FileIdx].Item2.Length;
                                }
                                BundleFilesToIncludeList.Add(BundleFilesToInclude);
                            }

                            // Upload each bundle
                            DateTime NewBundleTime = DateTime.UtcNow;
                            for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++)
                            {
                                List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx];

                                // Get the new bundle info
                                string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : "";
                                string NewBundleName   = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix);

                                // Create a random number for the object key
                                string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto);

                                // Create the bundle header
                                byte[] Header;
                                using (MemoryStream HeaderStream = new MemoryStream())
                                {
                                    BinaryWriter Writer = new BinaryWriter(HeaderStream);
                                    Writer.Write(BundleSignatureV1);
                                    Writer.Write(BundleFilesToInclude.Count);

                                    foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
                                    {
                                        Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1);
                                        Writer.Write((int)FileToInclude.Item2.Length);
                                    }

                                    Header = HeaderStream.ToArray();
                                }

                                // Create the output file
                                FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz");
                                Log.TraceInformation("Writing {0}", NewBundleFile);
                                using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read))
                                {
                                    using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true))
                                    {
                                        ZipStream.Write(Header, 0, Header.Length);
                                        foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
                                        {
                                            ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length);
                                        }
                                    }
                                }

                                // Upload the file
                                long NewBundleCompressedLength   = NewBundleFile.ToFileInfo().Length;
                                long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length);
                                Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f));
                                UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null);

                                // Add the bundle to the new manifest
                                BundleManifest.Entry Bundle = new BundleManifest.Entry();
                                Bundle.Name               = NewBundleName;
                                Bundle.ObjectKey          = NewBundleObjectKey;
                                Bundle.Time               = NewBundleTime;
                                Bundle.CompressedLength   = (int)NewBundleCompressedLength;
                                Bundle.UncompressedLength = (int)NewBundleUncompressedLength;
                                NewBundleManifest.Entries.Add(Bundle);

                                // Mark the manifest as requiring an update
                                bUpdateManifest = true;
                            }
                        }

                        // Update the manifest
                        if (bUpdateManifest)
                        {
                            DateTime UtcNow = DateTime.UtcNow;
                            DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0);

                            // Update the root manifest
                            RootManifest NewRootManifest = new RootManifest();
                            NewRootManifest.AccessKey = OldRootManifest.AccessKey;
                            NewRootManifest.SecretKey = OldRootManifest.SecretKey;
                            foreach (RootManifest.Entry Entry in OldRootManifest.Entries)
                            {
                                if (Entry.CreateTime >= RemoveBundleManifestsBefore)
                                {
                                    NewRootManifest.Entries.Add(Entry);
                                }
                            }

                            // Make sure there's an entry for the last 24h
                            DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0);
                            if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter))
                            {
                                RootManifest.Entry NewEntry = new RootManifest.Entry();
                                NewEntry.CreateTime = UtcNow;
                                NewEntry.Key        = KeyPrefix + CreateObjectName(Crypto);
                                NewRootManifest.Entries.Add(NewEntry);
                            }

                            // Save out the new bundle manifest
                            FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json");
                            NewBundleManifest.Save(NewBundleManifestFile);

                            // Update all the bundle manifests still valid
                            foreach (RootManifest.Entry Entry in NewRootManifest.Entries)
                            {
                                Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key);
                                UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null);
                            }

                            // Overwrite all the existing manifests
                            if (AllowSubmit)
                            {
                                List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));

                                // Create a changelist containing the new manifest
                                int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest");
                                if (ExistingFiles.Count > 0)
                                {
                                    P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
                                    NewRootManifest.Save(RootManifestFile);
                                }
                                else
                                {
                                    NewRootManifest.Save(RootManifestFile);
                                    P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
                                }

                                // Submit it
                                int SubmittedChangeNumber;
                                P4.Submit(ChangeNumber, out SubmittedChangeNumber, true);

                                if (SubmittedChangeNumber <= 0)
                                {
                                    throw new AutomationException("Failed to submit change");
                                }

                                // Delete any bundles that are no longer referenced
                                HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey));
                                foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries)
                                {
                                    if (!KeepObjectKeys.Contains(OldEntry.ObjectKey))
                                    {
                                        Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey);
                                        DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null);
                                    }
                                }

                                // Delete any bundle manifests which are no longer referenced
                                HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key));
                                foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries)
                                {
                                    if (!KeepManifestKeys.Contains(OldEntry.Key))
                                    {
                                        Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key);
                                        DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null);
                                    }
                                }
                            }
                            else
                            {
                                // Skip submitting
                                Log.TraceWarning("Skipping manifest submit due to missing -Submit argument.");
                            }

                            // Update the new download size
                            DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
                        }
                    }

                    // Print some stats about the final manifest
                    Log.TraceInformation("");
                    Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0));
                    Log.TraceInformation("");
                }
            }
        }