// We return from this function before all uploads are complete. So we must wait to dispose until all uploads are finished. public void QueueUpload(CacheEntry entry, string artifactsPath, MemoryStream stream) { var item = new WorkItem(); string finalHash = HashingMethods.Calculate(entry.Hash, m_GlobalHash).ToString(); item.fileId = FileId.From(entry.Guid.ToString(), finalHash); item.artifactsPath = artifactsPath; item.stream = stream; ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.UploadQueue, UploadItem, item); }
private static FileId GenerateFileId() { var rand = new Random(); var guid = new byte[16]; var hash = new byte[16]; rand.NextBytes(guid); rand.NextBytes(hash); return(FileId.From(guid, hash)); }
// We return from this function before all uploads are complete. So we must wait to dispose until all uploads are finished. public void QueueUpload(CacheEntry entry, string artifactsPath, MemoryStream stream) { var item = new WorkItem(); item.fileId = FileId.From(entry.Guid.ToString(), entry.Hash.ToString()); item.artifactsPath = artifactsPath; item.stream = stream; lock (m_WorkItems) m_WorkItems.Enqueue(item); m_Semaphore.Release(); }
// We return from this function before all uploads are complete. So we must wait to dispose until all uploads are finished. public void QueueUpload(CacheEntry entry, string artifactsPath, MemoryStream stream) { var item = new WorkItem(); string finalHash = HashingMethods.Calculate(entry.Hash, m_GlobalHash).ToString(); item.fileId = FileId.From(entry.Guid.ToString(), finalHash); item.artifactsPath = artifactsPath; item.stream = stream; lock (m_WorkItems) m_WorkItems.Enqueue(item); m_Semaphore.Release(); }
private FileId GenerateFileId() { if (rand == null) { rand = new Random(); } var guid = new byte[16]; var hash = new byte[16]; rand.NextBytes(guid); rand.NextBytes(hash); return(FileId.From(guid, hash)); }
public void DonwloadFileNotFound() { var client = new Client(KTestHost, TestPort); client.Connect(); var fileId = FileId.From(new byte[16], new byte[16]); var mre = new ManualResetEvent(false); var downloadItem = new TestDownloadItem(fileId, FileType.Asset); client.QueueDownload(downloadItem); Exception err = null; client.DownloadFinished += (sender, args) => { try { Assert.AreEqual(args.Result, DownloadResult.FileNotFound); Assert.AreEqual(args.DownloadItem.Id, fileId); } catch (Exception e) { err = e; } finally { mre.Set(); } }; mre.WaitOne(500); if (err != null) { throw err; } }
// We don't return from this function until all downloads are processed. So it is safe to dispose immediately after. public void DownloadMissing(IList <CacheEntry> entries, IList <CachedInfo> cachedInfos) { Assert.AreEqual(entries.Count, cachedInfos.Count); Directory.CreateDirectory(k_CachePath); m_Semaphore = new Semaphore(0, entries.Count); m_Client.DownloadFinished += ThreadedDownloadFinished; // Queue up downloads for the missing or invalid local data for (var index = 0; index < entries.Count; index++) { // Only download data for cachedInfos that are invalid if (cachedInfos[index] != null) { continue; } var entry = entries[index]; string finalHash = HashingMethods.Calculate(entry.Hash, m_GlobalHash).ToHash128().ToString(); var fileId = FileId.From(entry.Guid.ToString(), finalHash); // Download artifacts before info to ensure both are available when download for info returns var downloadArtifact = new FileDownloadItem(fileId, FileType.Resource, GetCachedArtifactsFile(entry)); m_Client.QueueDownload(downloadArtifact); var downloadInfo = new FileDownloadItem(fileId, FileType.Info, GetCachedInfoFile(entry)); m_Client.QueueDownload(downloadInfo); } // Check downloads to see if it is usable data var formatter = new BinaryFormatter(); for (var index = 0; index < entries.Count; index++) { // find the next invalid cachedInfo while (index < entries.Count && cachedInfos[index] != null) { index++; } // make sure we didn't go out of bounds looking for invalid entries if (index >= entries.Count) { break; } // Wait for info download m_Semaphore.WaitOne(); string tempInfoFile = GetCachedInfoFile(entries[index]); if (!File.Exists(tempInfoFile)) { continue; } try { CachedInfo info; using (var fileStream = new FileStream(tempInfoFile, FileMode.Open, FileAccess.Read)) info = formatter.Deserialize(fileStream) as CachedInfo; if (m_Cache.HasAssetOrDependencyChanged(info)) { continue; } // Not every info file will have artifacts. So just check to see if we downloaded something. // TODO: May want to extend CachedInfo with Artifact knowledge if there is a performance benefit? string tempArtifactFile = GetCachedArtifactsFile(entries[index]); string tempArtifactDir = Path.ChangeExtension(tempArtifactFile, ""); if (File.Exists(tempArtifactFile) && !FileCompressor.Decompress(tempArtifactFile, tempArtifactDir)) { continue; } // All valid, move downloaded data into place cachedInfos[index] = info; string targetInfoFile = m_Cache.GetCachedInfoFile(info.Asset); if (File.Exists(targetInfoFile)) { File.Delete(targetInfoFile); } else { Directory.CreateDirectory(Path.GetDirectoryName(targetInfoFile)); } File.Move(tempInfoFile, targetInfoFile); if (Directory.Exists(tempArtifactDir)) { string targetArtifactDir = m_Cache.GetCachedArtifactsDirectory(info.Asset); if (Directory.Exists(targetArtifactDir)) { Directory.Delete(targetArtifactDir, true); } Directory.Move(tempArtifactDir, targetArtifactDir); } } catch (Exception e) { BuildLogger.LogException(e); } } m_Client.ResetDownloadFinishedEventHandler(); ((IDisposable)m_Semaphore).Dispose(); m_Semaphore = null; Directory.Delete(k_CachePath, true); }