/// <summary>
///
/// </summary>
public StorageManager(List <StorageLocation> InLocations, ManifestDownloadManager InDownloadManager, BuildManifestRegistry Registry, AsyncIOQueue InIOQueue, ManifestStorageHeuristic Heuristic, List <Guid> KeepBuilds, List <Guid> DeleteBuilds)
{
IOQueue = InIOQueue;
ManifestRegistry = Registry;
DownloadManager = InDownloadManager;
Locations = InLocations;
StoredLocations = new List <StorageLocation>(Locations);
StorageHeuristic = Heuristic;
StoragePrioritizeKeepingBuildTagIds = KeepBuilds;
StoragePrioritizeDeletingBuildTagIds = DeleteBuilds;
}
/// <summary>
/// </summary>
/// <param name="Index"></param>
/// <returns></returns>
public bool GetBlockData(int Index, string RootPath, AsyncIOQueue IOQueue, byte[] Data, out long DataLength)
{
LazyCacheBlockInfo();
DataLength = 0;
if (Index < 0 || Index >= BlockCount)
{
throw new ArgumentOutOfRangeException("Index", "Block index out of range.");
return(false);
}
BuildManifestBlockInfo Info = BlockInfo[Index];
Debug.Assert(Data.Length >= Info.TotalSize);
DataLength = Info.TotalSize;
ManualResetEvent CompleteEvent = new ManualResetEvent(false);
int QueuedReads = Info.SubBlocks.Length;
bool Success = true;
foreach (BuildManifestSubBlockInfo SubBlock in Info.SubBlocks)
{
string PathName = Path.Combine(RootPath, SubBlock.File.Path);
IOQueue.Read(
PathName, SubBlock.FileOffset, SubBlock.FileSize, Data, SubBlock.OffsetInBlock, bSuccess =>
{
int Result = Interlocked.Decrement(ref QueuedReads);
if (Result == 0)
{
CompleteEvent.Set();
}
if (!bSuccess)
{
Success = false;
Logger.Log(LogLevel.Error, LogCategory.Manifest, "Failed to read data for block (offset={0} size={1}) from file {2}", SubBlock.FileOffset, SubBlock.FileSize, SubBlock.File);
}
}
);
}
CompleteEvent.WaitOne();
return(Success);
}
/// <summary>
/// </summary>
public static void OnStart()
{
Statistic.Instantiate();
ScmManager = new ScmManager();
IOQueue = new AsyncIOQueue();
ManifestDownloadManager = new ManifestDownloadManager();
DownloadManager = new DownloadManager();
DownloadManager.OnRequestReplicatedBuilds += (List <Guid> SelectTags, List <Guid> IgnoreTags, DateTime NewerThan) =>
{
NetClient.RequestFilteredBuilds(SelectTags, IgnoreTags, NewerThan);
Settings.ReplicationNewerThanTime = DownloadManager.ReplicationNewerThanTime;
SaveSettings();
};
Logger.Log(LogLevel.Info, LogCategory.Main, "OnStart: Initializing settings");
InitSettings();
NetClient = new Core.Client.Client();
RemoteActionClient = new RemoteActionClient(NetClient, ManifestDownloadManager);
TagRegistry = new TagRegistry();
RouteRegistry = new RouteRegistry(null, TagRegistry);
BuildRegistry = new BuildManifestRegistry(TagRegistry);
StorageManager = new StorageManager(Settings.StorageLocations, ManifestDownloadManager, BuildRegistry, IOQueue, Settings.StorageHeuristic, Settings.PrioritizeKeepingBuildTagIds, Settings.PrioritizeDeletingBuildTagIds);
Logger.Log(LogLevel.Info, LogCategory.Main, "OnStart: Setting up network client");
BuildRegistry.Open(Path.Combine(AppDataDir, "Manifests"), int.MaxValue);
Logger.Log(LogLevel.Info, LogCategory.Main, "OnStart: Setting up network client");
NetClient.Start(
Settings.ServerHostname,
Settings.ServerPort,
Settings.ClientPortRangeMin,
Settings.ClientPortRangeMax,
Settings.AllowRemoteActions,
Settings.TagIds,
BuildRegistry,
StorageManager,
ManifestDownloadManager,
TagRegistry,
RouteRegistry
);
NetClient.TagIds = new List <Guid>(Settings.TagIds);
// Setup the virtual file system we will store our available builds in.
Logger.Log(LogLevel.Info, LogCategory.Main, "OnStart: Setting up build file system");
BuildFileSystem = new VirtualFileSystem();
NetClient.OnClientTagsUpdatedByServer += () =>
{
Settings.TagIds = NetClient.TagIds;
NetClient.RestartConnections();
SaveSettings();
};
NetClient.OnTagListRecieved += (List <Tag> InTags) =>
{
TagRenderer.InvalidateResources();
};
NetClient.OnPermissionsUpdated += () =>
{
BuildFileSystem.ForceRefresh();
DownloadManager.ForceRefresh();
};
NetClient.OnBuildPublished += (string Path, Guid Id) =>
{
BuildFileSystem.ForceRefresh();
DownloadManager.ForceRefresh();
};
NetClient.OnBuildUpdated += (string Path, Guid Id) =>
{
BuildFileSystem.ForceRefresh();
DownloadManager.ForceRefresh();
};
NetClient.OnConnectedToServer += () => { BuildFileSystem.ForceRefresh(); };
NetClient.OnFilteredBuildsRecieved += (Builds) =>
{
DownloadManager.RecieveReplicatedBuilds(Builds);
};
NetClient.OnBuildsRecieved += (RootPath, Builds) =>
{
List <VirtualFileSystemInsertChild> NewChildren = new List <VirtualFileSystemInsertChild>();
foreach (NetMessage_GetBuildsResponse.BuildInfo Build in Builds)
{
NewChildren.Add(
new VirtualFileSystemInsertChild
{
VirtualPath = Build.VirtualPath,
CreateTime = Build.Guid == Guid.Empty ? DateTime.UtcNow : Build.CreateTime,
Metadata = Build
}
);
}
BuildFileSystem.ReconcileChildren(RootPath, NewChildren);
};
BuildFileSystem.OnRequestChildren += (FileSystem, Path) =>
{
if (NetClient.IsConnected)
{
NetClient.RequestBuilds(Path);
}
};
BuildFileSystem.Init();
// Setup download managers for the manifest and app level.
Logger.Log(LogLevel.Info, LogCategory.Main, "OnStart: Setting up manifest download manager");
ManifestDownloadManager.Start(
StorageManager,
Settings.ManifestDownloadStates,
BuildRegistry,
IOQueue
);
Logger.Log(LogLevel.Info, LogCategory.Main, "OnStart: Setting up download manager");
DownloadManager.Start(
ManifestDownloadManager,
Settings.DownloadStates,
BuildFileSystem,
ScmManager,
Settings.ReplicationNewerThanTime
);
Logger.Log(LogLevel.Info, LogCategory.Main, "OnStart: Setting up update download");
// Ensure we are downloading the latest update.
string UpdateDownloadName = "$ Buildsync Update $";
foreach (DownloadState State in DownloadManager.States.States)
{
if (State.Name == UpdateDownloadName)
{
InternalUpdateDownload = State;
break;
}
}
if (InternalUpdateDownload == null)
{
InternalUpdateDownload = DownloadManager.AddDownload(UpdateDownloadName, "$Internal$/Updates", 2, BuildSelectionRule.Newest, BuildSelectionFilter.None, "", "", true, false, "", "", new List <Guid>(), new List <Guid>());
}
// Make sure we have to get the latest manifest id before updating.
InternalUpdateDownload.ActiveManifestId = Guid.Empty;
// Clean up any orphan builds.
StorageManager.CleanUpOrphanBuilds();
Logger.Log(LogLevel.Info, LogCategory.Main, "OnStart: Complete");
}
/// <summary>
/// </summary>
public List <int> Validate(string RootPath, RateTracker Tracker, AsyncIOQueue IOQueue, BuildManfiestValidateProgressCallbackHandler Callback = null)
{
LazyCacheBlockInfo();
List <int> FailedBlocks = new List <int>();
try
{
LockBlockInfo();
int TaskCount = Environment.ProcessorCount;
Task[] FileTasks = new Task[TaskCount];
int BlockCounter = 0;
long BytesValidated = 0;
bool Aborted = false;
// Check the size of each file.
for (int i = 0; i < Files.Count; i++)
{
BuildManifestFileInfo FileInfo = Files[i];
string FilePath = Path.Combine(RootPath, FileInfo.Path);
string DirPath = Path.GetDirectoryName(FilePath);
if (!Directory.Exists(DirPath))
{
Directory.CreateDirectory(DirPath);
Logger.Log(LogLevel.Warning, LogCategory.Manifest, "Expected directory {0} does not exist, creating.", DirPath);
}
FileInfo Info = new FileInfo(FilePath);
if (!Info.Exists || Info.Length != FileInfo.Size)
{
using (FileStream Stream = new FileStream(FilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read))
{
Logger.Log(LogLevel.Warning, LogCategory.Manifest, "File {0} is not of expected length {1} (is {2}) settting length.", FilePath, FileInfo.Size, Info.Length);
Stream.SetLength(FileInfo.Size);
}
}
}
// Check each individual block of data for validity.
for (int i = 0; i < TaskCount; i++)
{
FileTasks[i] = Task.Run(
() =>
{
byte[] Buffer = new byte[BlockSize];
while (!Aborted)
{
int BlockIndex = Interlocked.Increment(ref BlockCounter) - 1;
if (BlockIndex >= BlockCount)
{
break;
}
BuildManifestBlockInfo BInfo = BlockInfo[BlockIndex];
long BufferLength = 0;
bool Success = GetBlockData(BlockIndex, RootPath, IOQueue, Buffer, out BufferLength);
uint Checksum = 0;
if (Version >= 2)
{
Checksum = Crc32Fast.Compute(Buffer, 0, (int)BufferLength);
}
else
{
Checksum = Crc32Slow.Compute(Buffer, (int)BufferLength);
}
if (!Success || BlockChecksums[BlockIndex] != Checksum)
{
Logger.Log(LogLevel.Warning, LogCategory.Manifest, "Block {0} failed checksum, block contains following sub-blocks:", BlockIndex);
for (int SubBlock = 0; SubBlock < BInfo.SubBlocks.Length; SubBlock++)
{
BuildManifestSubBlockInfo SubBInfo = BInfo.SubBlocks[SubBlock];
Logger.Log(LogLevel.Warning, LogCategory.Manifest, "\tfile={0} offset={1} size={2}", SubBInfo.File.Path, SubBInfo.FileOffset, SubBInfo.FileSize);
}
lock (FailedBlocks)
{
FailedBlocks.Add(BlockIndex);
}
}
Interlocked.Add(ref BytesValidated, BInfo.TotalSize);
if (Callback != null)
{
if (!Callback.Invoke(BytesValidated, TotalSize, Guid, BlockIndex))
{
Aborted = true;
}
}
}
}
);
}
Task.WaitAll(FileTasks);
}
finally
{
UnlockBlockInfo();
}
return(FailedBlocks);
}
/// <summary>
/// </summary>
public void InitializeDirectory(string RootPath, AsyncIOQueue IOQueue, bool AllocateFiles, BuildManfiestInitProgressCallbackHandler Callback = null)
{
LazyCacheBlockInfo();
try
{
LockBlockInfo();
const int WriteChunkSize = 16 * 1024 * 1024;
byte[] ChunkArray = new byte[WriteChunkSize];
byte[] ChunkPattern = { 0xDE, 0xAD, 0xBE, 0xEF };
for (int i = 0; i < WriteChunkSize; i++)
{
ChunkArray[i] = ChunkPattern[i % ChunkPattern.Length];
}
long TotalBytes = 0;
foreach (BuildManifestFileInfo FileInfo in Files)
{
TotalBytes += FileInfo.Size;
}
long BytesWritten = 0;
foreach (BuildManifestFileInfo FileInfo in Files)
{
string FilePath = Path.Combine(RootPath, FileInfo.Path);
string FileDir = Path.GetDirectoryName(FilePath);
if (!Directory.Exists(FileDir))
{
Directory.CreateDirectory(FileDir);
}
if (AllocateFiles)
{
using (FileStream Stream = File.OpenWrite(FilePath))
{
long BytesRemaining = FileInfo.Size;
while (BytesRemaining > 0)
{
long Size = Math.Min(BytesRemaining, WriteChunkSize);
Stream.Write(ChunkArray, 0, (int)Size);
BytesWritten += Size;
BytesRemaining -= Size;
AsyncIOQueue.GlobalBandwidthStats.In(Size);
if (Callback != null)
{
if (!Callback.Invoke(BytesWritten, TotalBytes))
{
return;
}
}
}
}
}
}
}
finally
{
UnlockBlockInfo();
}
}
/// <summary>
/// </summary>
public static BuildManifest BuildFromDirectory(Guid NewManifestId, string RootPath, string VirtualPath, AsyncIOQueue IOQueue, BuildManfiestProgressCallbackHandler Callback = null)
{
string[] FileNames = Directory.GetFiles(RootPath, "*", SearchOption.AllDirectories);
long TotalSize = 0;
// Try and order in the most efficient packing order.
List <FileInfo> RemainingFiles = new List <FileInfo>();
List <FileInfo> OrderedList = new List <FileInfo>();
for (int i = 0; i < FileNames.Length; i++)
{
FileInfo Info = new FileInfo(FileNames[i]);
RemainingFiles.Add(Info);
TotalSize += Info.Length;
}
// Order main list from largest to smallest.
RemainingFiles.Sort((Item1, Item2) => - Item1.Length.CompareTo(Item2.Length));
// Block size.
long ActualBlockSize = DefaultBlockSize;
// Try and add in optimal block packing format.
long BlockIndex = 0;
while (RemainingFiles.Count > 0)
{
FileInfo Info = RemainingFiles[0];
RemainingFiles.RemoveAt(0);
OrderedList.Add(Info);
//Console.WriteLine("Block[{0}] {1}", BlockIndex, Info.Name);
long BlockCount = (Info.Length + (ActualBlockSize - 1)) / ActualBlockSize;
long BytesRemaining = (Info.Length % ActualBlockSize) == 0 ? 0 : ActualBlockSize - Info.Length % ActualBlockSize;
BlockIndex += BlockCount;
long SubBlockCount = BlockCount;
// Try and fit some smaller files into the remaining block space.
for (int i = 0; i < RemainingFiles.Count && BytesRemaining > 0 && SubBlockCount < MaxSubBlockCount; i++)
{
FileInfo PotentialFile = RemainingFiles[i];
if (PotentialFile.Length <= BytesRemaining)
{
BytesRemaining -= PotentialFile.Length;
SubBlockCount++;
//Console.WriteLine("\tSubblock[{0}] {1}", BlockIndex, PotentialFile.Name);
RemainingFiles.RemoveAt(i);
OrderedList.Add(PotentialFile);
i--;
}
}
}
// Our final list!
FileInfo[] FileInfos = OrderedList.ToArray();
BuildManifest Manifest = new BuildManifest();
Manifest.Guid = NewManifestId;
Manifest.VirtualPath = VirtualPath;
Manifest.BlockCount = BlockIndex;
Manifest.BlockChecksums = new uint[Manifest.BlockCount];
Manifest.CreateTime = DateTime.UtcNow;
#if USE_SPARSE_CHECKSUMS
Manifest.SparseBlockChecksums = new uint[Manifest.BlockCount];
#else
Manifest.SparseBlockChecksums = null;
#endif
Manifest.Version = BuildManifest.CurrentVersion;
Manifest.BlockSize = ActualBlockSize;
List <Task> ChecksumTasks = new List <Task>();
int FileCounter = 0;
int BlockCounter = 0;
for (int i = 0; i < FileInfos.Length; i++)
{
Manifest.Files.Add(new BuildManifestFileInfo());
}
for (int i = 0; i < Environment.ProcessorCount; i++)
{
ChecksumTasks.Add(
Task.Run(
() =>
{
while (true)
{
int FileIndex = Interlocked.Increment(ref FileCounter) - 1;
if (FileIndex >= FileInfos.Length)
{
break;
}
FileInfo SubFileInfo = FileInfos[FileIndex];
string SubFile = SubFileInfo.FullName;
string RelativePath = SubFile.Substring(RootPath.Length).Trim('\\', '/');
if (Callback != null)
{
lock (Callback)
{
float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount);
Callback(RelativePath, Progress * 100);
}
}
BuildManifestFileInfo ManifestFileInfo = new BuildManifestFileInfo();
ManifestFileInfo.Path = RelativePath;
ManifestFileInfo.Size = new FileInfo(SubFile).Length;
ManifestFileInfo.Checksum = FileUtils.GetChecksum(SubFile, null);
lock (Manifest)
{
Manifest.Files[FileIndex] = ManifestFileInfo;
}
}
}
)
);
}
foreach (Task task in ChecksumTasks)
{
task.Wait();
}
ChecksumTasks.Clear();
// Calculate which data goes in eahc block.
Manifest.CacheBlockInfo();
Manifest.CacheSizeInfo();
Manifest.DebugCheck();
// Calculate checksum for each individual block.
for (int i = 0; i < Environment.ProcessorCount; i++)
{
ChecksumTasks.Add(
Task.Run(
() =>
{
byte[] Buffer = new byte[ActualBlockSize];
Stopwatch stopwatch = new Stopwatch();
while (true)
{
int CalculateBlockIndex = Interlocked.Increment(ref BlockCounter) - 1;
if (CalculateBlockIndex >= Manifest.BlockCount)
{
break;
}
long BufferLength = 0;
if (!Manifest.GetBlockData(CalculateBlockIndex, RootPath, IOQueue, Buffer, out BufferLength))
{
// We should never end up in this situation when publishing ...
Debug.Assert(false);
}
uint Checksum = 0;
if (Manifest.Version >= 2)
{
Checksum = Crc32Fast.Compute(Buffer, 0, (int)BufferLength);
}
else
{
Checksum = Crc32Slow.Compute(Buffer, (int)BufferLength);
}
Manifest.BlockChecksums[CalculateBlockIndex] = Checksum;
#if USE_SPARSE_CHECKSUMS
Manifest.SparseBlockChecksums[CalculateBlockIndex] = Crc32Slow.ComputeSparse(Buffer, (int)BufferLength);
#endif
if (Callback != null)
{
lock (Callback)
{
float Progress = (FileCounter + BlockCounter) / (float)(FileInfos.Length + Manifest.BlockCount);
Callback("Checksuming blocks", Progress * 100);
}
}
}
}
)
);
}
foreach (Task task in ChecksumTasks)
{
task.Wait();
}
return(Manifest);
}
