/// <summary>
/// Schedules a download to be downloaded
/// </summary>
/// <param name="dl">The download to be downloaded</param>
public static void ScheduleDownload(DownloadJob dl)
{
lock (downloads) {
downloads.Push (dl);
StartNextJob();
}
}
/// <summary>
/// Schedules a single file to be downloaded
/// </summary>
/// <param name="name">The name</param>
/// <param name="url">The URL to download from</param>
/// <param name="MCRequire">Wheter Minecraft requires this before starting</param>
public static DownloadJob ScheduleDownload(string Name, string Url, string Description, Action<DownloadJob,byte[]> finished = null)
{
DownloadJob dl = new DownloadJob(Url,Name,Description);
if(finished == null) dl.DownloadComplete += finished;
ScheduleDownload(dl);
return dl;
}
public void DistributeHeaders_WithALotOfPeersAndForkInTheMiddle()
{
var peers = new List <INetworkPeer>();
for (int i = 0; i < 10; i++)
{
INetworkPeer peer = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior);
peers.Add(peer);
}
List <ChainedHeader> chainA = ChainedHeadersHelper.CreateConsecutiveHeaders(10000);
List <ChainedHeader> chainB = ChainedHeadersHelper.CreateConsecutiveHeaders(5000, chainA[5000 - 1]);
List <int> peerIdsClaimingA = peers.Take(5).Select(x => x.Connection.Id).ToList();
this.Shuffle(peers);
foreach (INetworkPeer peer in peers)
{
this.puller.NewPeerTipClaimed(peer, peerIdsClaimingA.Contains(peer.Connection.Id) ? chainA.Last() : chainB.Last());
}
var job = new DownloadJob()
{
Headers = new List <ChainedHeader>(chainA), Id = 1
};
var failedHashes = new List <uint256>();
List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, int.MaxValue);
Assert.Empty(failedHashes);
Assert.Equal(chainA.Count, assignedDownloads.Count);
var peerIds = new HashSet <int>();
foreach (AssignedDownload assignedDownload in assignedDownloads.Skip(5000))
{
peerIds.Add(assignedDownload.PeerId);
}
Assert.Equal(5, peerIds.Count);
Assert.Equal(peerIdsClaimingA.Count, peerIds.Count);
foreach (int id in peerIdsClaimingA)
{
Assert.Contains(id, peerIds);
}
}
/// <summary>
/// Enqueues a job to the job queue. If there are more jobs than started threads,
/// new threads will be started up to a number of MaxThreads threads.
/// </summary>
/// <param name="job"></param>
public void AddJob(DownloadJob job)
{
_jobs.Enqueue(job);
_threadCounterMutex.WaitOne();
if (_currentThreads < _jobs.Count)
{
for (int i = 0; i < (_jobs.Count - _currentThreads) && _currentThreads < MaxThreads; i++)
{
Thread t = new Thread(DownloadThread);
t.Start();
_currentThreads++;
}
}
_threadCounterMutex.ReleaseMutex();
}
public void MyTestInitialize()
{
_job = new DownloadJob();
_job.Dispatch(new IPEndPoint(IPAddress.Parse("10.0.0.1"), 1111), new List<int>()
{
1,2,3,4,5
});
_job.Dispatch(new IPEndPoint(IPAddress.Parse("10.0.0.2"), 2222), new List<int>()
{
6,7,8,9,10,11,12
});
_job.Dispatch(new IPEndPoint(IPAddress.Parse("10.0.0.3"), 3333), new List<int>()
{
13,14,15
});
}
public static DownloadJob DownloadFile(string bucketId, string fileId, string localPath, io.storj.libstorj.Storj storj)
{
DownloadJob job = new DownloadJob(fileId);
DownloadFileCallbackAsync callback = new DownloadFileCallbackAsync(job);
try
{
var handle = storj.downloadFile(bucketId, fileId, localPath, callback);
job.Id = handle;
}
catch (io.storj.libstorj.KeysNotFoundException)
{
throw new KeysNotFoundException();
}
return(job);
}
public void SongDoesExist()
{
var downloadManager = new DownloadManager(1);
CancellationToken cancellationToken = CancellationToken.None;
downloadManager.Start(cancellationToken);
var existingSong = new PlaylistSong("d375405d047d6a2a4dd0f4d40d8da77554f1f677", "Does Exist", "5e20", "ejiejidayo");
var job = new DownloadJob(existingSong, DefaultSongsPath);
Assert.IsTrue(downloadManager.TryPostJob(job, out var postedJob));
downloadManager.CompleteAsync().Wait();
Assert.AreEqual(JobStatus.Finished, postedJob.Status);
Assert.AreEqual(DownloadResultStatus.Success, postedJob.Result.DownloadResult.Status);
Assert.AreEqual(ZipExtractResultStatus.Success, postedJob.Result.ZipResult.ResultStatus);
Assert.IsTrue(postedJob.Result.Successful);
Assert.AreEqual(existingSong.Hash, postedJob.Result.HashAfterDownload);
}
public void Canceled()
{
var cts = new CancellationTokenSource(100);
var cancellationToken = cts.Token;
var downloadManager = new DownloadManager(1);
downloadManager.Start(cancellationToken);
var existingSong = new PlaylistSong("d375405d047d6a2a4dd0f4d40d8da77554f1f677", "Does Exist", "5e20", "ejiejidayo");
var job = new DownloadJob(existingSong, DefaultSongsPath);
Assert.IsTrue(downloadManager.TryPostJob(job, out var postedJob));
downloadManager.CompleteAsync().Wait();
Assert.AreEqual(JobStatus.Canceled, postedJob.Status);
Assert.AreEqual(DownloadResultStatus.Canceled, postedJob.Result.DownloadResult.Status);
Assert.IsNull(postedJob.Result.ZipResult);
Assert.IsFalse(postedJob.Result.Successful);
Assert.IsNull(postedJob.Result.HashAfterDownload);
}
public void SongDoesntExist()
{
var downloadManager = new DownloadManager(1);
CancellationToken cancellationToken = CancellationToken.None;
downloadManager.Start(cancellationToken);
var doesntExist = new PlaylistSong("196be1af64958d8b5375b328b0eafae2151d46f8", "Doesn't Exist", "ffff", "Who knows");
var job = new DownloadJob(doesntExist, DefaultSongsPath);
Assert.IsTrue(downloadManager.TryPostJob(job, out var postedJob));
downloadManager.CompleteAsync().Wait();
Assert.AreEqual(DownloadResultStatus.NetNotFound, postedJob.Result.DownloadResult.Status);
Assert.AreEqual(404, postedJob.Result.DownloadResult.HttpStatusCode);
Assert.AreEqual(JobStatus.Finished, postedJob.Status);
Assert.IsNull(postedJob.Result.ZipResult);
Assert.IsFalse(postedJob.Result.Successful);
Assert.AreEqual(null, postedJob.Result.HashAfterDownload);
}
internal DownloadJob AddNotificationEventHandlers(
DownloadJob job)
{
job.OnTotalBytesRecorded +=
async(_, args) => await HandleTotalBytesRecorded(args);
job.OnFinished +=
async(_, args) => await HandleFinished(args);
job.OnFailed +=
async(_, args) => await HandleFailed(args);
/* This event is treated differently because we don't send messages immediately. Instead, they get
* sent in bulk via ProgressNotificationsBackgroundService*/
job.OnProgress +=
(_, args) => _progressNotifications[args.Id] = args.TotalBytesRead;
return(job);
}
public ThreadScheduler()
{
_logList = new ConcurrentQueue <string>(); //调度线程日志
_Mode = MODE_EXIT_ON_FINISHED;
_Parallelism = 2;
_Processor = null;
_evtStop = new ManualResetEvent(false);
_Finished = 1;
_evtTaskReady = new ManualResetEvent(false);
_taskWaitQueue = new ConcurrentQueue <T>();
_threadWorkers = new List <ConsumingWorker <T> >();
_eventWorkers = new List <ManualResetEvent>();
}
public void DistributeHeaders_LimitedByEmptySlots()
{
INetworkPeer peer = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior);
List <ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100);
this.puller.NewPeerTipClaimed(peer, headers.Last());
var job = new DownloadJob()
{
Headers = new List <ChainedHeader>(headers), Id = 1
};
var failedHashes = new List <uint256>();
List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, 50);
Assert.Equal(50, assignedDownloads.Count);
Assert.Equal(50, job.Headers.Count);
}
public void DistributeHeaders_PartOfTheJobFailed()
{
INetworkPeer peer = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior);
List <ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100);
this.puller.NewPeerTipClaimed(peer, headers[49]);
var job = new DownloadJob()
{
Headers = new List <ChainedHeader>(headers), Id = 1
};
var failedHashes = new List <uint256>();
List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, int.MaxValue);
Assert.Equal(50, assignedDownloads.Count);
Assert.Equal(50, failedHashes.Count);
}
protected override void OnActivityResult(int requestCode, Result resultCode, Intent data)
{
base.OnActivityResult(requestCode, resultCode, data);
if (requestCode == requestOpenDocument && resultCode == Result.Ok && data != null)
{
var uri = data.Data;
// Some URIs can be opened directly, including local filesystem, app assets, and content provider URIs.
if (PSPDFKitGlobal.IsOpenableUri(this, uri))
{
var intent = PdfActivityIntentBuilder.FromUri(this, uri)
.Configuration(configuration)
.Build();
StartActivity(intent);
Finish();
}
else
{
// The Uri cannot be directly opened. Download the PDF document from the uri, for local access.
// Find the DownloadProgressFragment for showing download progress, or create a new one.
var downloadFragment = SupportFragmentManager?.FindFragmentByTag(downloadProgressFragment)?.JavaCast <DownloadProgressFragment> ();
if (downloadFragment == null)
{
var job = DownloadJob.StartDownload(new DownloadRequest.Builder(this).Uri(uri).Build());
downloadFragment = new DownloadProgressFragment();
downloadFragment.Show(SupportFragmentManager, downloadProgressFragment);
downloadFragment.Job = job;
}
// Once the download is complete we launch the PdfActivity from the downloaded file.
downloadFragment.Job.Complete += (sender, e) => {
var intent = PdfActivityIntentBuilder.FromUri(this, Android.Net.Uri.FromFile(e.P0))
.Configuration(configuration)
.Build();
StartActivity(intent);
Finish();
};
}
}
}
public async Task PeerDisconnected_AllDownloadJobsAreReassignedAsync()
{
INetworkPeer peer1 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior1);
INetworkPeer peer2 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior2);
List <ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(10);
this.puller.NewPeerTipClaimed(peer1, headers.Last());
this.puller.RequestBlocksDownload(headers);
await this.puller.AssignDownloadJobsAsync();
// Make sure all jobs were assigned to 1.
foreach (ChainedHeader chainedHeader in headers)
{
Assert.True(this.puller.AssignedDownloadsByHash.ContainsKey(chainedHeader.HashBlock));
}
Assert.Equal(headers.Count, this.puller.AssignedDownloadsByHash.Count(x => x.Value.PeerId == peer1.Connection.Id));
this.puller.NewPeerTipClaimed(peer2, headers.Last());
this.puller.PeerDisconnected(peer1.Connection.Id);
// Make sure all assignments went to reassign queue as a single job.
Assert.Single(this.puller.ReassignedJobsQueue);
DownloadJob reassignedJob = this.puller.ReassignedJobsQueue.Peek();
foreach (ChainedHeader chainedHeader in headers)
{
Assert.True(reassignedJob.Headers.Exists(x => x == chainedHeader));
}
await this.puller.AssignDownloadJobsAsync();
// All should be assigned to peer 2.
Assert.Equal(headers.Count, this.puller.AssignedDownloadsByHash.Count);
Assert.Equal(headers.Count, this.puller.AssignedDownloadsByHash.Values.Count(x => x.PeerId == peer2.Connection.Id));
Assert.Empty(this.puller.ReassignedJobsQueue);
}
static DownloadJob[] Read_Tree(string path)
{
var NewJobs = new List <DownloadJob>();
var tree = Tree.Load(path);
foreach (var entry in tree.Entries)
{
var job = new DownloadJob();
job.Path = "objects/" + entry.SHA1.Insert(2, "/");
if (entry.IsDirectory)
{
job.Callback = Read_Tree;
}
else
{
job.Callback = null;
}
NewJobs.Add(job);
}
return(NewJobs.ToArray());
}
public void Invalid_DownloadUri_Null()
{
string songHash = "2C908DF9BB7AA93884AB9BFA8DDC598C3DE479E9";
Uri downloadUri = null;
string tempDirectory = "TempDir";
try
{
var invalidJob = new DownloadJob(DefaultSong, downloadUri, tempDirectory);
Assert.Fail("An ArgumentNullException should have been thrown.");
}
catch (AssertFailedException) { throw; }
catch (ArgumentNullException expectedException)
{
Assert.AreEqual("downloadUri", expectedException?.ParamName);
}
catch (Exception unexpectedException)
{
Assert.Fail($"The wrong exception was thrown. Expected ArgumentNullException, caught {unexpectedException.GetType().ToString()}");
}
}
/// <summary>
/// DownloadFacade
/// </summary>
/// <param name="hash"></param>
/// <param name="saveFileName"></param>
public void Download(string hash, string saveFileName)
{
ServerFileInfoResponse response = _webHelper.GetServerResponse(HPPClient._serverIpEndPoint, hash);
if (_downloadJobDict.ContainsKey(hash))
{
return;
}
DownloadJob downloadJob = new DownloadJob();
downloadJob.Hash = hash;
downloadJob.SaveFileName = saveFileName;
downloadJob.Mine = new byte[response.FileSize.GetBitArrayLength()];
downloadJob.FileLen = response.FileSize;
_downloadJobDict.Add(hash, downloadJob);
PreAlloc(saveFileName, response.FileSize);
downloadJob.Process(response.IpList);
}
public void Invalid_SongHash_WhiteSpace()
{
string songHash = " ";
Uri downloadUri = new Uri("https://beatsaver.com/api/download/hash/2c908df9bb7aa93884ab9bfa8ddc598c3de479e9");
string tempDirectory = "TempDir";
try
{
var invalidJob = new DownloadJob(DefaultSong, downloadUri, tempDirectory);
Assert.Fail("An ArgumentNullException should have been thrown.");
}
catch (AssertFailedException) { throw; }
catch (ArgumentNullException expectedException)
{
Assert.AreEqual("songHash", expectedException?.ParamName);
}
catch (Exception unexpectedException)
{
Assert.Fail($"The wrong exception was thrown. Expected ArgumentNullException, caught {unexpectedException.GetType().ToString()}");
}
}
/// <summary>
/// Downloads the songs in the provided Dictionary.
/// </summary>
/// <param name="queuedSongs"></param>
/// <param name="skippedsongs"></param>
/// <param name="useSongKeyAsOutputFolder"></param>
/// <returns></returns>
public List <SongInfo> DownloadSongs(Dictionary <string, SongInfo> queuedSongs, out int skippedsongs, bool useSongKeyAsOutputFolder)
{
//var existingSongs = Directory.GetDirectories(CustomSongsPath);
string tempPath = "";
string outputPath = "";
List <SongInfo> matchedSongs = new List <SongInfo>();
skippedsongs = 0;
DownloadBatch jobs = new DownloadBatch();
jobs.JobCompleted += OnJobFinished;
foreach (var song in queuedSongs.Values)
{
tempPath = Path.Combine(Path.GetTempPath(), song.key + zipExtension);
if (useSongKeyAsOutputFolder)
{
outputPath = Path.Combine(CustomSongsPath, song.key);
}
else
{
outputPath = CustomSongsPath;
}
bool songExists = Directory.Exists(outputPath);
bool songInHistory = _songDownloadHistory.Contains(song.key);
if ((songExists && songInHistory) || !songInHistory)
{
matchedSongs.Add(song);
}
if (songExists || songInHistory)
{
skippedsongs++;
//Logger.Debug($"Skipping song - SongExists: {songExists}, SongInHistory: {songInHistory}");
continue; // We already have the song or don't want it, skip
}
DownloadJob job = new DownloadJob(song, tempPath, outputPath);
jobs.AddJob(job);
}
jobs.RunJobs().Wait();
return(matchedSongs);
}
private static void AddJob()
{
if (EpisodesToDownload.Count > 0)
{
Episode episode = EpisodesToDownload.Dequeue();
string downloadUrl = NineAnime.GetVideoUrl(SelectedServer.Id, episode.Id).Result;
if (string.IsNullOrEmpty(downloadUrl))
{
ErrorMessages.Add(string.Format("Episode {0}: Failed to Get Download URL!", episode.Name));
AddJob();
}
else
{
DownloadJob job = new DownloadJob(episode.Name, downloadUrl, MainProgressBar);
job.Start();
DownloadJobs.Add(job);
job.DownloadTask.ContinueWith((Task task) => AddJob());
}
}
}
private static void Tests()
{
var thing = new ScrappedSong();
Web.HttpClientWrapper.Initialize(5);
List <ScrappedSong> scrapedDict;
using (StreamReader file = File.OpenText(@"C:\Users\Jared\source\repos\SyncSaberService\SyncSaberService\bin\Debug\ScrapedData\combinedScrappedData.json"))
{
JsonSerializer serializer = new JsonSerializer();
scrapedDict = (List <ScrappedSong>)serializer.Deserialize(file, typeof(List <ScrappedSong>));
}
DownloadJob testJob = new DownloadJob(new SongInfo("111-111", "testName", "", "testAuthor"), "temp", "CustomSongs");
var testTask = testJob.RunJobAsync();
testTask.Wait();
var searchTest = BeatSaverReader.Search("6A097D39A5FA94F3B736E6EEF5A519A2", BeatSaverReader.SearchType.hash);
var testReader = new ScoreSaberReader();
var sssongs = testReader.GetSSSongsFromPage(HttpClientWrapper.GetPageText("https://scoresaber.com/api.php?function=get-leaderboards&cat=3&limit=5&page=39&ranked=1"));
foreach (var sssong in sssongs)
{
sssong.PopulateFields();
}
var songs = testReader.GetSongsFromFeed(new ScoreSaberFeedSettings(0)
{
MaxPages = 10
});
SongInfo song = new SongInfo("18750-20381", "test", "testUrl", "testAuthor");
song.PopulateFields();
var test = song["key"];
var test2 = song["id"];
var test3 = song["uploaderId"];
}
private GetResponseDto AsDto(
DownloadJob downloadJob)
{
if (downloadJob.Status is DownloadJob.DownloadStatus.NotStarted)
{
return(new GetResponseDto(
downloadJob.Id,
downloadJob.Link,
downloadJob.SaveAsFile.Name,
downloadJob.Status.ToString(),
downloadJob.CreatedTicks));
}
return(new GetResponseDto(
downloadJob.Id,
downloadJob.Link,
downloadJob.SaveAsFile.Name,
downloadJob.Status.ToString(),
downloadJob.CreatedTicks,
downloadJob.TotalBytes,
downloadJob.BytesDownloaded,
downloadJob.ReasonForFailure));
}
public void DistributeHeaders_NoPeerClaimTheChain()
{
INetworkPeer peer1 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior1);
List <ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(10);
ChainedHeader unclaimedHeader = this.helper.CreateChainedHeader();
this.puller.NewPeerTipClaimed(peer1, headers.Last());
var job = new DownloadJob()
{
Headers = new List <ChainedHeader>()
{
unclaimedHeader
}, Id = 1
};
var failedHashes = new List <uint256>();
List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, int.MaxValue);
Assert.Equal(failedHashes.First(), unclaimedHeader.HashBlock);
Assert.Single(failedHashes);
Assert.Empty(assignedDownloads);
}
public void PostJob_Actual()
{
CancellationToken cancellationToken = CancellationToken.None;
int concurrentDownloads = 3;
var manager = new DownloadManager(concurrentDownloads);
var songsDir = Path.GetFullPath(@"Output\DownloadManager_Tests\PostJob_Actual_Test");
if (Directory.Exists(songsDir))
{
Directory.Delete(songsDir, true);
}
var song = new PlaylistSong("19f2879d11a91b51a5c090d63471c3e8d9b7aee3", "Believer", string.Empty, "rustic");
var job = new DownloadJob(song, songsDir);
manager.Start(cancellationToken);
manager.TryPostJob(job, out var postedJob);
manager.CompleteAsync().Wait();
Assert.IsTrue(postedJob.Result.Successful);
Assert.AreEqual(song.Hash, postedJob.Result.HashAfterDownload);
foreach (var file in job.Result.ZipResult.ExtractedFiles)
{
Assert.IsTrue(File.Exists(file));
}
}
public void DistributeHeaders_OnePeerForksAtSomePoint()
{
INetworkPeer peer1 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior1);
INetworkPeer peer2 = this.helper.CreatePeer(out ExtendedBlockPullerBehavior behavior2);
List <ChainedHeader> peer1Headers = ChainedHeadersHelper.CreateConsecutiveHeaders(1000);
List <ChainedHeader> peer2Headers = ChainedHeadersHelper.CreateConsecutiveHeaders(500, peer1Headers[500 - 1]);
this.puller.NewPeerTipClaimed(peer1, peer1Headers.Last());
this.puller.NewPeerTipClaimed(peer2, peer2Headers.Last());
var job = new DownloadJob()
{
Headers = new List <ChainedHeader>(peer1Headers), Id = 1
};
var failedHashes = new List <uint256>();
List <AssignedDownload> assignedDownloads = this.puller.DistributeHeadersLocked(job, failedHashes, int.MaxValue);
Assert.Empty(failedHashes);
Assert.Equal(peer1Headers.Count, assignedDownloads.Count);
Assert.True(assignedDownloads.Take(500).Count(x => x.PeerId == peer1.Connection.Id) > 0);
Assert.True(assignedDownloads.Take(500).Count(x => x.PeerId == peer2.Connection.Id) > 0);
Assert.True(assignedDownloads.Skip(500).All(x => x.PeerId == peer1.Connection.Id));
}
private DownloadFileCallbackAsync(DownloadJob job)
{
_job = job;
}
/// <summary>
/// Cancels a download-job
/// </summary>
/// <param name="job">The job to cancel</param>
/// <returns>True, if the job got cancelled; false if not</returns>
public bool CancelDownload(DownloadJob job)
{
return(_storjJava.cancelDownload(job.Id));
}
/// <summary>
/// Gets called when an Asset has finished downloading
/// </summary>
/// <param name="dl">The download that has finished</param>
private static void AssetDownloaded(DownloadJob dl)
{
throw new NotImplementedException();
}
public async Task Post([FromBody] DownloadJob job)
{
await _fetchService.FetchAsync(job.Url, job.DatasetName);
}
public List <AssignedDownload> DistributeHeadersLocked(DownloadJob downloadJob, List <uint256> failedHashes, int emptySlots)
{
return((List <AssignedDownload>) this.puller.InvokeMethod("DistributeHeadersLocked", downloadJob, failedHashes, emptySlots));
}
/// <summary>
/// Stores the Downloaded data to the Asset
/// </summary>
/// <param name="d">The Download where the Asset was downloaded with</param>
public void StoreAsset(DownloadJob d)
{
throw new NotImplementedException();
}