/// <summary>
/// </summary>
public void UnregisterManifest(Guid ManifestId)
{
BuildManifest Manifest = GetManifestById(ManifestId);
if (Manifest == null)
{
return;
}
string FilePath = Path.Combine(RootPath, ManifestId + ".manifest");
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Unregistering manifest: {0}", FilePath);
try
{
File.Delete(FilePath);
string MetaFile = FilePath + ".metadata";
if (File.Exists(MetaFile))
{
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Unregistering manifest metadata: {0}", MetaFile);
File.Delete(MetaFile);
}
}
catch (Exception Ex)
{
Logger.Log(LogLevel.Error, LogCategory.Manifest, "Failed to delete unregistered manifest file {0} with error: {1}", FilePath, Ex.Message);
}
ManifestFileSystem.RemoveNode(Manifest.VirtualPath);
Manifests.Remove(Manifest);
}
/// <summary>
/// </summary>
/// <param name="Manifest"></param>
public void AddManifest(BuildManifest Manifest)
{
ManifestFileSystem.InsertNode(Manifest.VirtualPath, DateTime.UtcNow, Manifest);
Manifests.Add(Manifest);
PruneManifests();
}
/// <summary>
/// </summary>
public void PruneUnseenManifests(int MaxDaysOld)
{
lock (ManifestLastSeenTimes)
{
for (int i = 0; i < Manifests.Count; i++)
{
BuildManifest Manifest = Manifests[i];
string Id = Manifest.Guid.ToString();
if (ManifestLastSeenTimes.ContainsKey(Id))
{
TimeSpan Elapsed = DateTime.UtcNow - ManifestLastSeenTimes[Id];
if (Elapsed.TotalDays > MaxDaysOld)
{
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Pruning manifest '{0}' '{1}' as it's {2} days since it was last seen on any peer.", Manifest.Guid.ToString(), Manifest.VirtualPath, (int)Elapsed.TotalDays);
UnregisterManifest(Manifest.Guid);
// ManifestFileSystem.RemoveNode(Manifest.VirtualPath);
//Manifests.RemoveAt(i);
i--;
ManifestLastSeenTimesDirty = true;
}
}
else
{
// Put an initial timestamp in for this manifest.
MarkAsSeen(Manifest.Guid);
ManifestLastSeenTimesDirty = true;
}
}
}
}
/// <summary>
///
/// </summary>
/// <param name="Manifest"></param>
/// <param name="Tag"></param>
public void UntagManifest(Guid ManifestId, Guid TagId)
{
BuildManifest Manifest = GetManifestById(ManifestId);
if (Manifest == null)
{
return;
}
Tag Tag = TagRegistry.GetTagById(TagId);
if (Tag == null)
{
return;
}
if (Manifest.Metadata != null && Manifest.Metadata.TagIds.Contains(TagId))
{
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Unagging manifest {0} with {1}", ManifestId.ToString(), Tag.Name);
Manifest.Metadata.ModifiedTime = DateTime.UtcNow;
Manifest.Metadata.TagIds.Remove(TagId);
StoreMetadata(Manifest);
}
}
/// <summary>
///
/// </summary>
/// <param name="Manifest"></param>
/// <param name="Tag"></param>
public void TagManifest(Guid ManifestId, Guid TagId)
{
BuildManifest Manifest = GetManifestById(ManifestId);
if (Manifest == null)
{
return;
}
Tag Tag = TagRegistry.GetTagById(TagId);
if (Tag == null)
{
return;
}
if (Manifest.Metadata != null && !Manifest.Metadata.TagIds.Contains(TagId))
{
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Tagging manifest {0} with {1}", ManifestId.ToString(), Tag.Name);
Manifest.Metadata.ModifiedTime = DateTime.UtcNow;
Manifest.Metadata.TagIds.Add(TagId);
StoreMetadata(Manifest);
}
if (Tag.Unique)
{
string ParentPath = VirtualFileSystem.GetParentPath(Manifest.VirtualPath);
string NodeName = VirtualFileSystem.GetNodeName(Manifest.VirtualPath);
List <string> Children = GetVirtualPathChildren(ParentPath);
Tag DecayTag = TagRegistry.GetTagById(Tag.DecayTagId);
foreach (string ChildName in Children)
{
if (ChildName == Manifest.VirtualPath)
{
continue;
}
BuildManifest ChildManifest = GetManifestByPath(ChildName);
if (ChildManifest != null)
{
if (ChildManifest.Metadata != null && ChildManifest.Metadata.TagIds.Contains(TagId))
{
UntagManifest(ChildManifest.Guid, TagId);
if (DecayTag != null)
{
TagManifest(ChildManifest.Guid, DecayTag.Id);
}
}
}
}
}
}
/// <summary>
///
/// </summary>
private void StoreMetadata(BuildManifest Manifest)
{
string MetaFile = Path.Combine(RootPath, Manifest.Guid + ".manifest") + ".metadata";
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Storing manifest metadata: {0}", MetaFile);
if (Manifest.Metadata != null)
{
Manifest.Metadata.WriteToFile(MetaFile);
}
}
/// <summary>
/// </summary>
/// <param name="FilePath"></param>
/// <returns></returns>
public static BuildManifest FromByteArray(byte[] ByteArray)
{
BuildManifest Manifest = FileUtils.ReadFromArray <BuildManifest>(ByteArray);
if (Manifest != null)
{
Manifest.UpgradeVersion();
Manifest.CacheBlockInfo();
Manifest.CacheSizeInfo();
}
return(Manifest);
}
/// <summary>
/// </summary>
public void PruneManifests()
{
if (Manifests.Count <= MaximumManifests)
{
return;
}
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Pruning manifests to max limits");
// Create a flat list of folders and all builds they contain, we will prune starting from the
// folders with the largest number of builds.
Dictionary <string, List <BuildManifest> > Folders = new Dictionary <string, List <BuildManifest> >();
foreach (BuildManifest Manifest in Manifests)
{
string Folder = VirtualFileSystem.GetParentPath(Manifest.VirtualPath);
if (!Folders.ContainsKey(Folder))
{
Folders.Add(Folder, new List <BuildManifest>());
}
List <BuildManifest> List = Folders[Folder];
List.Add(Manifest);
}
// Sort all folders.
foreach (KeyValuePair <string, List <BuildManifest> > Entry in Folders)
{
Entry.Value.Sort((Item1, Item2) => - Item1.CreateTime.CompareTo(Item2.CreateTime));
}
// Prune until we are back in space constraints.
while (Manifests.Count > MaximumManifests)
{
// Find folder with the most entries.
List <BuildManifest> Folder = null;
foreach (KeyValuePair <string, List <BuildManifest> > Entry in Folders)
{
if (Folder == null || Entry.Value.Count > Folder.Count)
{
Folder = Entry.Value;
}
}
// Last entry is oldests.
BuildManifest Manifest = Folder[Folder.Count - 1];
Folder.RemoveAt(Folder.Count - 1);
UnregisterManifest(Manifest.Guid);
}
}
/// <summary>
/// </summary>
/// <param name="FilePath"></param>
/// <returns></returns>
public static BuildManifest ReadFromFile(string FilePath)
{
BuildManifest Manifest = FileUtils.ReadFromBinaryFile <BuildManifest>(FilePath);
if (Manifest != null)
{
Manifest.OriginalFilePath = FilePath;
Manifest.UpgradeVersion();
Manifest.CacheSizeInfo();
Manifest.TrimBlockInfo();
}
return(Manifest);
}
/// <summary>
/// </summary>
public void RegisterManifest(BuildManifest Manifest)
{
string FilePath = Path.Combine(RootPath, Manifest.Guid + ".manifest");
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Registering manifest: {0}", FilePath);
if (Manifest.Metadata == null)
{
Manifest.Metadata = new BuildManifestMetadata();
}
Manifest.WriteToFile(FilePath);
AddManifest(Manifest);
StoreMetadata(Manifest);
}
/// <summary>
///
/// </summary>
/// <returns></returns>
public List <BuildManifestFileDiff> Diff(BuildManifest Other)
{
LazyCacheBlockInfo();
if (Other != null)
{
Other.LazyCacheBlockInfo();
}
List <BuildManifestFileDiff> Result = new List <BuildManifestFileDiff>();
if (Other == null)
{
foreach (BuildManifestFileInfo FileInfo in Files)
{
BuildManifestFileDiff FileDiff = new BuildManifestFileDiff();
FileDiff.Type = BuildManifestFileDiffType.Unchanged;
FileDiff.FileInfo = FileInfo;
Result.Add(FileDiff);
}
}
else
{
// Get removed files.
foreach (BuildManifestFileInfo FileInfo in Files)
{
BuildManifestFileDiff FileDiff = new BuildManifestFileDiff();
BuildManifestFileInfo OtherFileInfo = Other.GetFileInfo(FileInfo.Path);
if (OtherFileInfo == null)
{
FileDiff.FileInfo = FileInfo;
FileDiff.Type = BuildManifestFileDiffType.Removed;
Result.Add(FileDiff);
}
}
// Get added/modified files.
foreach (BuildManifestFileInfo OtherFileInfo in Other.Files)
{
BuildManifestFileDiff FileDiff = new BuildManifestFileDiff();
FileDiff.FileInfo = OtherFileInfo;
BuildManifestFileInfo FileInfo = GetFileInfo(OtherFileInfo.Path);
if (FileInfo == null)
{
FileDiff.Type = BuildManifestFileDiffType.Added;
}
else
{
if (FileInfo.Checksum == OtherFileInfo.Checksum &&
FileInfo.Size == OtherFileInfo.Size)
{
FileDiff.Type = BuildManifestFileDiffType.Unchanged;
}
else
{
FileDiff.Type = BuildManifestFileDiffType.Modified;
}
}
Result.Add(FileDiff);
}
}
return(Result);
}
/// <summary>
/// </summary>
public static BuildManifest BuildFromDirectory(Guid NewManifestId, string RootPath, string VirtualPath, AsyncIOQueue IOQueue, BuildManfiestProgressCallbackHandler Callback = null)
{
string[] FileNames = Directory.GetFiles(RootPath, "*", SearchOption.AllDirectories);
long TotalSize = 0;
// Try and order in the most efficient packing order.
List <FileInfo> RemainingFiles = new List <FileInfo>();
List <FileInfo> OrderedList = new List <FileInfo>();
for (int i = 0; i < FileNames.Length; i++)
{
FileInfo Info = new FileInfo(FileNames[i]);
RemainingFiles.Add(Info);
TotalSize += Info.Length;
}
// Order main list from largest to smallest.
RemainingFiles.Sort((Item1, Item2) => - Item1.Length.CompareTo(Item2.Length));
// Block size.
long ActualBlockSize = DefaultBlockSize;
// Try and add in optimal block packing format.
long BlockIndex = 0;
while (RemainingFiles.Count > 0)
{
FileInfo Info = RemainingFiles[0];
RemainingFiles.RemoveAt(0);
OrderedList.Add(Info);
//Console.WriteLine("Block[{0}] {1}", BlockIndex, Info.Name);
long BlockCount = (Info.Length + (ActualBlockSize - 1)) / ActualBlockSize;
long BytesRemaining = (Info.Length % ActualBlockSize) == 0 ? 0 : ActualBlockSize - Info.Length % ActualBlockSize;
BlockIndex += BlockCount;
long SubBlockCount = BlockCount;
// Try and fit some smaller files into the remaining block space.
for (int i = 0; i < RemainingFiles.Count && BytesRemaining > 0 && SubBlockCount < MaxSubBlockCount; i++)
{
FileInfo PotentialFile = RemainingFiles[i];
if (PotentialFile.Length <= BytesRemaining)
{
BytesRemaining -= PotentialFile.Length;
SubBlockCount++;
//Console.WriteLine("\tSubblock[{0}] {1}", BlockIndex, PotentialFile.Name);
RemainingFiles.RemoveAt(i);
OrderedList.Add(PotentialFile);
i--;
}
}
}
// Our final list!
FileInfo[] FileInfos = OrderedList.ToArray();
BuildManifest Manifest = new BuildManifest();
Manifest.Guid = NewManifestId;
Manifest.VirtualPath = VirtualPath;
Manifest.BlockCount = BlockIndex;
Manifest.BlockChecksums = new uint[Manifest.BlockCount];
Manifest.CreateTime = DateTime.UtcNow;
#if USE_SPARSE_CHECKSUMS
Manifest.SparseBlockChecksums = new uint[Manifest.BlockCount];
#else
Manifest.SparseBlockChecksums = null;
#endif
Manifest.Version = BuildManifest.CurrentVersion;
Manifest.BlockSize = ActualBlockSize;
List <Task> ChecksumTasks = new List <Task>();
int FileCounter = 0;
int BlockCounter = 0;
for (int i = 0; i < FileInfos.Length; i++)
{
Manifest.Files.Add(new BuildManifestFileInfo());
}
for (int i = 0; i < Environment.ProcessorCount; i++)
{
ChecksumTasks.Add(
Task.Run(
() =>
{
while (true)
{
int FileIndex = Interlocked.Increment(ref FileCounter) - 1;
if (FileIndex >= FileInfos.Length)
{
break;
}
FileInfo SubFileInfo = FileInfos[FileIndex];
string SubFile = SubFileInfo.FullName;
string RelativePath = SubFile.Substring(RootPath.Length).Trim('\\', '/');
if (Callback != null)
{
lock (Callback)
{
float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount);
Callback(RelativePath, Progress * 100);
}
}
BuildManifestFileInfo ManifestFileInfo = new BuildManifestFileInfo();
ManifestFileInfo.Path = RelativePath;
ManifestFileInfo.Size = new FileInfo(SubFile).Length;
ManifestFileInfo.Checksum = FileUtils.GetChecksum(SubFile, null);
lock (Manifest)
{
Manifest.Files[FileIndex] = ManifestFileInfo;
}
}
}
)
);
}
foreach (Task task in ChecksumTasks)
{
task.Wait();
}
ChecksumTasks.Clear();
// Calculate which data goes in eahc block.
Manifest.CacheBlockInfo();
Manifest.CacheSizeInfo();
Manifest.DebugCheck();
// Calculate checksum for each individual block.
for (int i = 0; i < Environment.ProcessorCount; i++)
{
ChecksumTasks.Add(
Task.Run(
() =>
{
byte[] Buffer = new byte[ActualBlockSize];
Stopwatch stopwatch = new Stopwatch();
while (true)
{
int CalculateBlockIndex = Interlocked.Increment(ref BlockCounter) - 1;
if (CalculateBlockIndex >= Manifest.BlockCount)
{
break;
}
long BufferLength = 0;
if (!Manifest.GetBlockData(CalculateBlockIndex, RootPath, IOQueue, Buffer, out BufferLength))
{
// We should never end up in this situation when publishing ...
Debug.Assert(false);
}
uint Checksum = 0;
if (Manifest.Version >= 2)
{
Checksum = Crc32Fast.Compute(Buffer, 0, (int)BufferLength);
}
else
{
Checksum = Crc32Slow.Compute(Buffer, (int)BufferLength);
}
Manifest.BlockChecksums[CalculateBlockIndex] = Checksum;
#if USE_SPARSE_CHECKSUMS
Manifest.SparseBlockChecksums[CalculateBlockIndex] = Crc32Slow.ComputeSparse(Buffer, (int)BufferLength);
#endif
if (Callback != null)
{
lock (Callback)
{
float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount);
Callback("Checksuming blocks", Progress * 100);
}
}
}
}
)
);
}
foreach (Task task in ChecksumTasks)
{
task.Wait();
}
return(Manifest);
}
/// <summary>
/// </summary>
private void CacheBlockInfo()
{
// Reload content from disk cache.
if (OriginalFilePath != "" && Files.Count == 0)
{
BuildManifest Manifest = FileUtils.ReadFromBinaryFile <BuildManifest>(OriginalFilePath);
if (Manifest == null)
{
Logger.Log(LogLevel.Error, LogCategory.Manifest, "Failed to reload trimmed block information from manifest, this may cause crashes if the data is unavailable: {0}", OriginalFilePath);
return;
}
BlockChecksums = Manifest.BlockChecksums;
SparseBlockChecksums = Manifest.SparseBlockChecksums;
Files = Manifest.Files;
}
// Store block information.
BlockInfo = new BuildManifestBlockInfo[BlockCount];
FilesByPath = new Dictionary <string, BuildManifestFileInfo>();
// Try and add in optimal block packing format.
//Console.WriteLine("======================================== CACHING BLOCK INFO ================================");
long BlockIndex = 0;
for (int fi = 0; fi < Files.Count;)
{
BuildManifestFileInfo Info = Files[fi];
Info.FirstBlockIndex = (int)BlockIndex;
fi++;
long BlockCount = (Info.Size + (BlockSize - 1)) / BlockSize;
long BytesRemaining = (Info.Size % BlockSize) == 0 ? 0 : BlockSize - Info.Size % BlockSize;
//Console.WriteLine("Block[{0}] {1}", BlockIndex, Info.Path);
// Fill info for all the "full blocks" for this file.
long Total = 0;
for (int i = 0; i < BlockCount; i++)
{
BuildManifestSubBlockInfo SubBlock;
SubBlock.File = Info;
SubBlock.FileOffset = i * BlockSize;
SubBlock.FileSize = Math.Min(BlockSize, Info.Size - SubBlock.FileOffset);
SubBlock.OffsetInBlock = 0;
Debug.Assert(SubBlock.FileOffset + SubBlock.FileSize <= Info.Size);
Debug.Assert(BlockInfo[BlockIndex].SubBlocks == null);
BlockInfo[BlockIndex].SubBlocks = new BuildManifestSubBlockInfo[1];
BlockInfo[BlockIndex].SubBlocks[0] = SubBlock;
BlockInfo[BlockIndex].TotalSize += SubBlock.FileSize;
Debug.Assert(BlockInfo[BlockIndex].TotalSize <= BlockSize);
Total += SubBlock.FileSize;
BlockIndex++;
}
Debug.Assert(Total == Info.Size);
Info.LastBlockIndex = (int)BlockIndex - 1;
int LastBlockIndex = Info.LastBlockIndex;
// Fill remaining space with blocks.
Debug.Assert(BlockInfo[LastBlockIndex].TotalSize <= BlockSize);
while (BytesRemaining > 0 && fi < Files.Count && BlockInfo[LastBlockIndex].SubBlocks.Length < BuildManifest.MaxSubBlockCount)
{
BuildManifestFileInfo NextInfo = Files[fi];
if (NextInfo.Size > BytesRemaining)
{
break;
}
if (NextInfo.Size > 0)
{
BuildManifestSubBlockInfo SubBlock;
SubBlock.File = NextInfo;
SubBlock.FileOffset = 0;
SubBlock.FileSize = NextInfo.Size;
SubBlock.OffsetInBlock = BlockInfo[LastBlockIndex].TotalSize;
Array.Resize(ref BlockInfo[LastBlockIndex].SubBlocks, BlockInfo[LastBlockIndex].SubBlocks.Length + 1);
BlockInfo[LastBlockIndex].SubBlocks[BlockInfo[LastBlockIndex].SubBlocks.Length - 1] = SubBlock;
BlockInfo[LastBlockIndex].TotalSize += SubBlock.FileSize;
Debug.Assert(BlockInfo[LastBlockIndex].TotalSize <= BlockSize);
//Console.WriteLine("\tSubblock[{0}] {1}", BlockIndex, SubBlock.File.Path);
}
NextInfo.FirstBlockIndex = (int)BlockIndex - 1;
NextInfo.LastBlockIndex = (int)BlockIndex - 1;
BytesRemaining -= NextInfo.Size;
//Console.WriteLine("\tRemaining: {0}", BytesRemaining);
fi++;
}
}
}
/// <summary>
/// </summary>
/// <param name="Path"></param>
public void Open(string Path, int InMaxManifests)
{
MaximumManifests = InMaxManifests;
RootPath = Path;
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Loading build manfiests from: {0}", Path);
if (!Directory.Exists(RootPath))
{
Directory.CreateDirectory(RootPath);
}
Stopwatch watch = new Stopwatch();
watch.Start();
string[] ManifestFilePaths = Directory.GetFiles(Path, "*.manifest", SearchOption.AllDirectories);
List <Task> Tasks = new List <Task>();
List <BuildManifest> Results = new List <BuildManifest>();
foreach (string FilePath in ManifestFilePaths)
{
Tasks.Add(Task.Run(() =>
{
try
{
Logger.Log(LogLevel.Verbose, LogCategory.Manifest, "Loading manifest: {0}", FilePath);
BuildManifest Manifest = BuildManifest.ReadFromFile(FilePath);
if (Manifest != null)
{
string MetaFile = FilePath + ".metadata";
if (File.Exists(MetaFile))
{
Logger.Log(LogLevel.Verbose, LogCategory.Manifest, "Loading manifest metadata: {0}", MetaFile);
Manifest.Metadata = BuildManifestMetadata.ReadFromFile(MetaFile);
}
if (Manifest.Metadata == null)
{
Manifest.Metadata = new BuildManifestMetadata();
}
lock (Results)
{
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Loaded Manifest (Version {2}): {0} -> {1}", Manifest.Guid.ToString(), Manifest.VirtualPath, Manifest.Version);
Results.Add(Manifest);
}
}
}
catch (Exception ex)
{
Logger.Log(LogLevel.Error, LogCategory.Manifest, "Failed to read manifest '{0}', due to error {1}", FilePath, ex.Message);
// File.Delete(FilePath);
}
}));
}
Task.WaitAll(Tasks.ToArray());
GC.Collect();
foreach (BuildManifest Manifest in Results)
{
AddManifest(Manifest);
}
watch.Stop();
Logger.Log(LogLevel.Info, LogCategory.Manifest, "Loaded all manifests in {0}ms", watch.ElapsedMilliseconds);
PruneManifests();
}
