/// <summary> /// Schedules a download to be downloaded /// </summary> /// <param name="dl">The download to be downloaded</param> public static void ScheduleDownload(DownloadJob dl) { lock (downloads) { downloads.Push (dl); StartNextJob(); } }
/// <summary> /// Schedules a single file to be downloaded /// </summary> /// <param name="name">The name</param> /// <param name="url">The URL to download from</param> /// <param name="MCRequire">Wheter Minecraft requires this before starting</param> public static DownloadJob ScheduleDownload(string Name, string Url, string Description, Action<DownloadJob,byte[]> finished = null) { DownloadJob dl = new DownloadJob(Url,Name,Description); if(finished == null) dl.DownloadComplete += finished; ScheduleDownload(dl); return dl; }
public void DistributeHeaders_WithALotOfPeersAndForkInTheMiddle() { var peers = new List <INetworkPeer>(); for (int i = 0; i < 10; i++) { INetworkPeer peer = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior); peers.Add(peer); } List <ChainedHeader> chainA = ChainedHeadersHelper.CreateConsecutiveHeaders(10000); List <ChainedHeader> chainB = ChainedHeadersHelper.CreateConsecutiveHeaders(5000, chainA[5000 - 1]); List <int> peerIdsClaimingA = peers.Take(5).Select(x => x.Connection.Id).ToList(); this.Shuffle(peers); foreach (INetworkPeer peer in peers) { this.puller.NewPeerTipClaimed(peer, peerIdsClaimingA.Contains(peer.Connection.Id) ? chainA.Last() : chainB.Last()); } var job = new DownloadJob() { Headers = new List <ChainedHeader>(chainA), Id = 1 }; var failedHashes = new List <uint256>(); List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, int.MaxValue); Assert.Empty(failedHashes); Assert.Equal(chainA.Count, assignedDownloads.Count); var peerIds = new HashSet <int>(); foreach (AssignedDownload assignedDownload in assignedDownloads.Skip(5000)) { peerIds.Add(assignedDownload.PeerId); } Assert.Equal(5, peerIds.Count); Assert.Equal(peerIdsClaimingA.Count, peerIds.Count); foreach (int id in peerIdsClaimingA) { Assert.Contains(id, peerIds); } }
/// <summary> /// Enqueues a job to the job queue. If there are more jobs than started threads, /// new threads will be started up to a number of MaxThreads threads. /// </summary> /// <param name="job"></param> public void AddJob(DownloadJob job) { _jobs.Enqueue(job); _threadCounterMutex.WaitOne(); if (_currentThreads < _jobs.Count) { for (int i = 0; i < (_jobs.Count - _currentThreads) && _currentThreads < MaxThreads; i++) { Thread t = new Thread(DownloadThread); t.Start(); _currentThreads++; } } _threadCounterMutex.ReleaseMutex(); }
public void MyTestInitialize() { _job = new DownloadJob(); _job.Dispatch(new IPEndPoint(IPAddress.Parse("10.0.0.1"), 1111), new List<int>() { 1,2,3,4,5 }); _job.Dispatch(new IPEndPoint(IPAddress.Parse("10.0.0.2"), 2222), new List<int>() { 6,7,8,9,10,11,12 }); _job.Dispatch(new IPEndPoint(IPAddress.Parse("10.0.0.3"), 3333), new List<int>() { 13,14,15 }); }
public static DownloadJob DownloadFile(string bucketId, string fileId, string localPath, io.storj.libstorj.Storj storj) { DownloadJob job = new DownloadJob(fileId); DownloadFileCallbackAsync callback = new DownloadFileCallbackAsync(job); try { var handle = storj.downloadFile(bucketId, fileId, localPath, callback); job.Id = handle; } catch (io.storj.libstorj.KeysNotFoundException) { throw new KeysNotFoundException(); } return(job); }
public void SongDoesExist() { var downloadManager = new DownloadManager(1); CancellationToken cancellationToken = CancellationToken.None; downloadManager.Start(cancellationToken); var existingSong = new PlaylistSong("d375405d047d6a2a4dd0f4d40d8da77554f1f677", "Does Exist", "5e20", "ejiejidayo"); var job = new DownloadJob(existingSong, DefaultSongsPath); Assert.IsTrue(downloadManager.TryPostJob(job, out var postedJob)); downloadManager.CompleteAsync().Wait(); Assert.AreEqual(JobStatus.Finished, postedJob.Status); Assert.AreEqual(DownloadResultStatus.Success, postedJob.Result.DownloadResult.Status); Assert.AreEqual(ZipExtractResultStatus.Success, postedJob.Result.ZipResult.ResultStatus); Assert.IsTrue(postedJob.Result.Successful); Assert.AreEqual(existingSong.Hash, postedJob.Result.HashAfterDownload); }
public void Canceled() { var cts = new CancellationTokenSource(100); var cancellationToken = cts.Token; var downloadManager = new DownloadManager(1); downloadManager.Start(cancellationToken); var existingSong = new PlaylistSong("d375405d047d6a2a4dd0f4d40d8da77554f1f677", "Does Exist", "5e20", "ejiejidayo"); var job = new DownloadJob(existingSong, DefaultSongsPath); Assert.IsTrue(downloadManager.TryPostJob(job, out var postedJob)); downloadManager.CompleteAsync().Wait(); Assert.AreEqual(JobStatus.Canceled, postedJob.Status); Assert.AreEqual(DownloadResultStatus.Canceled, postedJob.Result.DownloadResult.Status); Assert.IsNull(postedJob.Result.ZipResult); Assert.IsFalse(postedJob.Result.Successful); Assert.IsNull(postedJob.Result.HashAfterDownload); }
public void SongDoesntExist() { var downloadManager = new DownloadManager(1); CancellationToken cancellationToken = CancellationToken.None; downloadManager.Start(cancellationToken); var doesntExist = new PlaylistSong("196be1af64958d8b5375b328b0eafae2151d46f8", "Doesn't Exist", "ffff", "Who knows"); var job = new DownloadJob(doesntExist, DefaultSongsPath); Assert.IsTrue(downloadManager.TryPostJob(job, out var postedJob)); downloadManager.CompleteAsync().Wait(); Assert.AreEqual(DownloadResultStatus.NetNotFound, postedJob.Result.DownloadResult.Status); Assert.AreEqual(404, postedJob.Result.DownloadResult.HttpStatusCode); Assert.AreEqual(JobStatus.Finished, postedJob.Status); Assert.IsNull(postedJob.Result.ZipResult); Assert.IsFalse(postedJob.Result.Successful); Assert.AreEqual(null, postedJob.Result.HashAfterDownload); }
internal DownloadJob AddNotificationEventHandlers( DownloadJob job) { job.OnTotalBytesRecorded += async(_, args) => await HandleTotalBytesRecorded(args); job.OnFinished += async(_, args) => await HandleFinished(args); job.OnFailed += async(_, args) => await HandleFailed(args); /* This event is treated differently because we don't send messages immediately. Instead, they get * sent in bulk via ProgressNotificationsBackgroundService*/ job.OnProgress += (_, args) => _progressNotifications[args.Id] = args.TotalBytesRead; return(job); }
public ThreadScheduler() { _logList = new ConcurrentQueue <string>(); //调度线程日志 _Mode = MODE_EXIT_ON_FINISHED; _Parallelism = 2; _Processor = null; _evtStop = new ManualResetEvent(false); _Finished = 1; _evtTaskReady = new ManualResetEvent(false); _taskWaitQueue = new ConcurrentQueue <T>(); _threadWorkers = new List <ConsumingWorker <T> >(); _eventWorkers = new List <ManualResetEvent>(); }
public void DistributeHeaders_LimitedByEmptySlots() { INetworkPeer peer = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior); List <ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100); this.puller.NewPeerTipClaimed(peer, headers.Last()); var job = new DownloadJob() { Headers = new List <ChainedHeader>(headers), Id = 1 }; var failedHashes = new List <uint256>(); List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, 50); Assert.Equal(50, assignedDownloads.Count); Assert.Equal(50, job.Headers.Count); }
public void DistributeHeaders_PartOfTheJobFailed() { INetworkPeer peer = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior); List <ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100); this.puller.NewPeerTipClaimed(peer, headers[49]); var job = new DownloadJob() { Headers = new List <ChainedHeader>(headers), Id = 1 }; var failedHashes = new List <uint256>(); List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, int.MaxValue); Assert.Equal(50, assignedDownloads.Count); Assert.Equal(50, failedHashes.Count); }
protected override void OnActivityResult(int requestCode, Result resultCode, Intent data) { base.OnActivityResult(requestCode, resultCode, data); if (requestCode == requestOpenDocument && resultCode == Result.Ok && data != null) { var uri = data.Data; // Some URIs can be opened directly, including local filesystem, app assets, and content provider URIs. if (PSPDFKitGlobal.IsOpenableUri(this, uri)) { var intent = PdfActivityIntentBuilder.FromUri(this, uri) .Configuration(configuration) .Build(); StartActivity(intent); Finish(); } else { // The Uri cannot be directly opened. Download the PDF document from the uri, for local access. // Find the DownloadProgressFragment for showing download progress, or create a new one. var downloadFragment = SupportFragmentManager?.FindFragmentByTag(downloadProgressFragment)?.JavaCast <DownloadProgressFragment> (); if (downloadFragment == null) { var job = DownloadJob.StartDownload(new DownloadRequest.Builder(this).Uri(uri).Build()); downloadFragment = new DownloadProgressFragment(); downloadFragment.Show(SupportFragmentManager, downloadProgressFragment); downloadFragment.Job = job; } // Once the download is complete we launch the PdfActivity from the downloaded file. downloadFragment.Job.Complete += (sender, e) => { var intent = PdfActivityIntentBuilder.FromUri(this, Android.Net.Uri.FromFile(e.P0)) .Configuration(configuration) .Build(); StartActivity(intent); Finish(); }; } } }
public async Task PeerDisconnected_AllDownloadJobsAreReassignedAsync() { INetworkPeer peer1 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior1); INetworkPeer peer2 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior2); List <ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(10); this.puller.NewPeerTipClaimed(peer1, headers.Last()); this.puller.RequestBlocksDownload(headers); await this.puller.AssignDownloadJobsAsync(); // Make sure all jobs were assigned to 1. foreach (ChainedHeader chainedHeader in headers) { Assert.True(this.puller.AssignedDownloadsByHash.ContainsKey(chainedHeader.HashBlock)); } Assert.Equal(headers.Count, this.puller.AssignedDownloadsByHash.Count(x => x.Value.PeerId == peer1.Connection.Id)); this.puller.NewPeerTipClaimed(peer2, headers.Last()); this.puller.PeerDisconnected(peer1.Connection.Id); // Make sure all assignments went to reassign queue as a single job. Assert.Single(this.puller.ReassignedJobsQueue); DownloadJob reassignedJob = this.puller.ReassignedJobsQueue.Peek(); foreach (ChainedHeader chainedHeader in headers) { Assert.True(reassignedJob.Headers.Exists(x => x == chainedHeader)); } await this.puller.AssignDownloadJobsAsync(); // All should be assigned to peer 2. Assert.Equal(headers.Count, this.puller.AssignedDownloadsByHash.Count); Assert.Equal(headers.Count, this.puller.AssignedDownloadsByHash.Values.Count(x => x.PeerId == peer2.Connection.Id)); Assert.Empty(this.puller.ReassignedJobsQueue); }
static DownloadJob[] Read_Tree(string path) { var NewJobs = new List <DownloadJob>(); var tree = Tree.Load(path); foreach (var entry in tree.Entries) { var job = new DownloadJob(); job.Path = "objects/" + entry.SHA1.Insert(2, "/"); if (entry.IsDirectory) { job.Callback = Read_Tree; } else { job.Callback = null; } NewJobs.Add(job); } return(NewJobs.ToArray()); }
public void Invalid_DownloadUri_Null() { string songHash = "2C908DF9BB7AA93884AB9BFA8DDC598C3DE479E9"; Uri downloadUri = null; string tempDirectory = "TempDir"; try { var invalidJob = new DownloadJob(DefaultSong, downloadUri, tempDirectory); Assert.Fail("An ArgumentNullException should have been thrown."); } catch (AssertFailedException) { throw; } catch (ArgumentNullException expectedException) { Assert.AreEqual("downloadUri", expectedException?.ParamName); } catch (Exception unexpectedException) { Assert.Fail($"The wrong exception was thrown. Expected ArgumentNullException, caught {unexpectedException.GetType().ToString()}"); } }
/// <summary> /// DownloadFacade /// </summary> /// <param name="hash"></param> /// <param name="saveFileName"></param> public void Download(string hash, string saveFileName) { ServerFileInfoResponse response = _webHelper.GetServerResponse(HPPClient._serverIpEndPoint, hash); if (_downloadJobDict.ContainsKey(hash)) { return; } DownloadJob downloadJob = new DownloadJob(); downloadJob.Hash = hash; downloadJob.SaveFileName = saveFileName; downloadJob.Mine = new byte[response.FileSize.GetBitArrayLength()]; downloadJob.FileLen = response.FileSize; _downloadJobDict.Add(hash, downloadJob); PreAlloc(saveFileName, response.FileSize); downloadJob.Process(response.IpList); }
public void Invalid_SongHash_WhiteSpace() { string songHash = " "; Uri downloadUri = new Uri("https://beatsaver.com/api/download/hash/2c908df9bb7aa93884ab9bfa8ddc598c3de479e9"); string tempDirectory = "TempDir"; try { var invalidJob = new DownloadJob(DefaultSong, downloadUri, tempDirectory); Assert.Fail("An ArgumentNullException should have been thrown."); } catch (AssertFailedException) { throw; } catch (ArgumentNullException expectedException) { Assert.AreEqual("songHash", expectedException?.ParamName); } catch (Exception unexpectedException) { Assert.Fail($"The wrong exception was thrown. Expected ArgumentNullException, caught {unexpectedException.GetType().ToString()}"); } }
/// <summary> /// Downloads the songs in the provided Dictionary. /// </summary> /// <param name="queuedSongs"></param> /// <param name="skippedsongs"></param> /// <param name="useSongKeyAsOutputFolder"></param> /// <returns></returns> public List <SongInfo> DownloadSongs(Dictionary <string, SongInfo> queuedSongs, out int skippedsongs, bool useSongKeyAsOutputFolder) { //var existingSongs = Directory.GetDirectories(CustomSongsPath); string tempPath = ""; string outputPath = ""; List <SongInfo> matchedSongs = new List <SongInfo>(); skippedsongs = 0; DownloadBatch jobs = new DownloadBatch(); jobs.JobCompleted += OnJobFinished; foreach (var song in queuedSongs.Values) { tempPath = Path.Combine(Path.GetTempPath(), song.key + zipExtension); if (useSongKeyAsOutputFolder) { outputPath = Path.Combine(CustomSongsPath, song.key); } else { outputPath = CustomSongsPath; } bool songExists = Directory.Exists(outputPath); bool songInHistory = _songDownloadHistory.Contains(song.key); if ((songExists && songInHistory) || !songInHistory) { matchedSongs.Add(song); } if (songExists || songInHistory) { skippedsongs++; //Logger.Debug($"Skipping song - SongExists: {songExists}, SongInHistory: {songInHistory}"); continue; // We already have the song or don't want it, skip } DownloadJob job = new DownloadJob(song, tempPath, outputPath); jobs.AddJob(job); } jobs.RunJobs().Wait(); return(matchedSongs); }
private static void AddJob() { if (EpisodesToDownload.Count > 0) { Episode episode = EpisodesToDownload.Dequeue(); string downloadUrl = NineAnime.GetVideoUrl(SelectedServer.Id, episode.Id).Result; if (string.IsNullOrEmpty(downloadUrl)) { ErrorMessages.Add(string.Format("Episode {0}: Failed to Get Download URL!", episode.Name)); AddJob(); } else { DownloadJob job = new DownloadJob(episode.Name, downloadUrl, MainProgressBar); job.Start(); DownloadJobs.Add(job); job.DownloadTask.ContinueWith((Task task) => AddJob()); } } }
private static void Tests() { var thing = new ScrappedSong(); Web.HttpClientWrapper.Initialize(5); List <ScrappedSong> scrapedDict; using (StreamReader file = File.OpenText(@"C:\Users\Jared\source\repos\SyncSaberService\SyncSaberService\bin\Debug\ScrapedData\combinedScrappedData.json")) { JsonSerializer serializer = new JsonSerializer(); scrapedDict = (List <ScrappedSong>)serializer.Deserialize(file, typeof(List <ScrappedSong>)); } DownloadJob testJob = new DownloadJob(new SongInfo("111-111", "testName", "", "testAuthor"), "temp", "CustomSongs"); var testTask = testJob.RunJobAsync(); testTask.Wait(); var searchTest = BeatSaverReader.Search("6A097D39A5FA94F3B736E6EEF5A519A2", BeatSaverReader.SearchType.hash); var testReader = new ScoreSaberReader(); var sssongs = testReader.GetSSSongsFromPage(HttpClientWrapper.GetPageText("https://scoresaber.com/api.php?function=get-leaderboards&cat=3&limit=5&page=39&ranked=1")); foreach (var sssong in sssongs) { sssong.PopulateFields(); } var songs = testReader.GetSongsFromFeed(new ScoreSaberFeedSettings(0) { MaxPages = 10 }); SongInfo song = new SongInfo("18750-20381", "test", "testUrl", "testAuthor"); song.PopulateFields(); var test = song["key"]; var test2 = song["id"]; var test3 = song["uploaderId"]; }
private GetResponseDto AsDto( DownloadJob downloadJob) { if (downloadJob.Status is DownloadJob.DownloadStatus.NotStarted) { return(new GetResponseDto( downloadJob.Id, downloadJob.Link, downloadJob.SaveAsFile.Name, downloadJob.Status.ToString(), downloadJob.CreatedTicks)); } return(new GetResponseDto( downloadJob.Id, downloadJob.Link, downloadJob.SaveAsFile.Name, downloadJob.Status.ToString(), downloadJob.CreatedTicks, downloadJob.TotalBytes, downloadJob.BytesDownloaded, downloadJob.ReasonForFailure)); }
public void DistributeHeaders_NoPeerClaimTheChain() { INetworkPeer peer1 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior1); List <ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(10); ChainedHeader unclaimedHeader = this.helper.CreateChainedHeader(); this.puller.NewPeerTipClaimed(peer1, headers.Last()); var job = new DownloadJob() { Headers = new List <ChainedHeader>() { unclaimedHeader }, Id = 1 }; var failedHashes = new List <uint256>(); List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, int.MaxValue); Assert.Equal(failedHashes.First(), unclaimedHeader.HashBlock); Assert.Single(failedHashes); Assert.Empty(assignedDownloads); }
public void PostJob_Actual() { CancellationToken cancellationToken = CancellationToken.None; int concurrentDownloads = 3; var manager = new DownloadManager(concurrentDownloads); var songsDir = Path.GetFullPath(@"Output\DownloadManager_Tests\PostJob_Actual_Test"); if (Directory.Exists(songsDir)) { Directory.Delete(songsDir, true); } var song = new PlaylistSong("19f2879d11a91b51a5c090d63471c3e8d9b7aee3", "Believer", string.Empty, "rustic"); var job = new DownloadJob(song, songsDir); manager.Start(cancellationToken); manager.TryPostJob(job, out var postedJob); manager.CompleteAsync().Wait(); Assert.IsTrue(postedJob.Result.Successful); Assert.AreEqual(song.Hash, postedJob.Result.HashAfterDownload); foreach (var file in job.Result.ZipResult.ExtractedFiles) { Assert.IsTrue(File.Exists(file)); } }
public void DistributeHeaders_OnePeerForksAtSomePoint() { INetworkPeer peer1 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior1); INetworkPeer peer2 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior2); List <ChainedHeader> peer1Headers = ChainedHeadersHelper.CreateConsecutiveHeaders(1000); List <ChainedHeader> peer2Headers = ChainedHeadersHelper.CreateConsecutiveHeaders(500, peer1Headers[500 - 1]); this.puller.NewPeerTipClaimed(peer1, peer1Headers.Last()); this.puller.NewPeerTipClaimed(peer2, peer2Headers.Last()); var job = new DownloadJob() { Headers = new List <ChainedHeader>(peer1Headers), Id = 1 }; var failedHashes = new List <uint256>(); List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, int.MaxValue); Assert.Empty(failedHashes); Assert.Equal(peer1Headers.Count, assignedDownloads.Count); Assert.True(assignedDownloads.Take(500).Count(x => x.PeerId == peer1.Connection.Id) > 0); Assert.True(assignedDownloads.Take(500).Count(x => x.PeerId == peer2.Connection.Id) > 0); Assert.True(assignedDownloads.Skip(500).All(x => x.PeerId == peer1.Connection.Id)); }
private DownloadFileCallbackAsync(DownloadJob job) { _job = job; }
/// <summary> /// Cancels a download-job /// </summary> /// <param name="job">The job to cancel</param> /// <returns>True, if the job got cancelled; false if not</returns> public bool CancelDownload(DownloadJob job) { return(_storjJava.cancelDownload(job.Id)); }
/// <summary> /// Gets called when an Asset has finished downloading /// </summary> /// <param name="dl">The download that has finished</param> private static void AssetDownloaded(DownloadJob dl) { throw new NotImplementedException(); }
public async Task Post([FromBody] DownloadJob job) { await _fetchService.FetchAsync(job.Url, job.DatasetName); }
public List <AssignedDownload> DistributeHeadersLocked(DownloadJob downloadJob, List <uint256> failedHashes, int emptySlots) { return((List <AssignedDownload>) this.puller.InvokeMethod("DistributeHeadersLocked", downloadJob, failedHashes, emptySlots)); }
/// <summary> /// Stores the Downloaded data to the Asset /// </summary> /// <param name="d">The Download where the Asset was downloaded with</param> public void StoreAsset(DownloadJob d) { throw new NotImplementedException(); }