protected override void OnDetach(IDownload download)
{
download.DownloadStarted -= downloadStarted;
download.DownloadCancelled -= downloadCancelled;
download.DownloadCompleted -= downloadCompleted;
download.DataReceived -= downloadDataReceived;
}
/// <summary>
/// Removes the download from the active downloads.
/// </summary>
/// <param name="download">The download to remove.</param>
protected virtual void Remove(IDownload download)
{
lock (this)
{
_downloads.Remove(download);
}
}
private static IDownload Wrap(this IDownload download, IResponse response)
{
var cts = new CancellationTokenSource();
var task = Task.FromResult(response);
return(new Download(task, cts, download.Target, download.Source));
}
private static IDownload Wrap(this IDownload download, Func <IResponse, IDownload> callback)
{
var cts = new CancellationTokenSource();
var task = download.Task.Wrap(callback);
return(new Download(task, cts, download.Target, download.Source));
}
private void PerformTest(IAudioDownloadOutputSample sample)
{
const string DownloadDirectory = "fakepath";
MockForAudioDownloadTest processMock = this.CreateProcessFactory(sample, DownloadDirectory);
YoutubeDl downloader = new YoutubeDl(processMock);
Dictionary <double, bool> ExpectedPercentsToBeReported = new Dictionary <double, bool>(sample.ExpectedPercents.Distinct().Select(x => new KeyValuePair <double, bool>(x, false)));
Dictionary <DownloadState, bool> ExpectedStatusesToBeReported = new Dictionary <DownloadState, bool>(sample.ExpectedDownloadStatuses.Distinct().Select(x => new KeyValuePair <DownloadState, bool>(x, false)));
using (IDownload progress = downloader.PrepareDownload(new Uri(sample.MediaUri), MediaFormat.MP3Audio, DownloadDirectory)) {
RunDownload(processMock, ExpectedPercentsToBeReported, ExpectedStatusesToBeReported, progress);
}
foreach (var item in ExpectedStatusesToBeReported)
{
Assert.IsTrue(item.Value, $"Expected download status {item.Key} was not reported by event {nameof(IDownload.DownloadStatusChanged)}.");
}
foreach (var item in ExpectedPercentsToBeReported)
{
Assert.IsTrue(item.Value, $"Expected download percentage {item.Key} was not reported by event {nameof(IDownload.PercentageChanged)}.");
}
}
private static IDownload CheckIntegrity(this CorsRequest cors, IDownload download)
{
var response = download.Task.Result;
var value = cors.Request.Source?.GetAttribute(AttributeNames.Integrity);
var integrity = cors.Integrity;
if (value is { Length : > 0 } && integrity != null && response != null)
private void CreateDownloadHandler()
{
if (_downloadHandler != null)
{
return;
}
switch (Entry.FileDownload.Method.ToLower())
{
case "github":
_downloadHandler = new GitHubDownload(Entry.FileDownload.GithubUser,
Entry.FileDownload.GithubRepo, Entry.Name + ".zip",
Entry.RemoteVersionInfo.MatchPattern, Entry.RemoteVersionInfo.MatchGroup,
Entry.RemoteVersionInfo.MatchURL);
break;
case "deviantart":
_downloadHandler = new DeviantArtDownload(Entry.FileDownload.DeviantURL, Entry.Name + ".zip",
Entry.RemoteVersionInfo.MatchPattern,
Entry.RemoteVersionInfo.MatchGroup, Entry.RemoteVersionInfo.MatchURL,
Entry.FileDownload.FolderName ?? Entry.Name);
break;
case "direct":
// TODO
break;
default:
throw new Exception("Unknown download method " + Entry.FileDownload.Method + " for skin " + Entry.Name + ".");
}
}
private static IDownload CheckIntegrity(this CorsRequest cors, IDownload download)
{
var response = download.Task.Result;
var value = cors.Request.Source?.GetAttribute(AttributeNames.Integrity);
var integrity = cors.Integrity;
if (!String.IsNullOrEmpty(value) && integrity != null && response != null)
{
var content = new MemoryStream();
response.Content.CopyTo(content);
content.Position = 0;
if (!integrity.IsSatisfied(content.ToArray(), value))
{
response.Dispose();
throw new DomException(DomError.Security);
}
return(download.Wrap(new DefaultResponse
{
Address = response.Address,
Content = content,
Headers = response.Headers,
StatusCode = response.StatusCode
}));
}
return(download);
}
/// <summary>
/// Creates a task to load the resource of the resource type from the
/// request.
/// </summary>
/// <param name="element">The element to use.</param>
/// <param name="download">The issued download.</param>
/// <param name="callback">The callback handling the resource.</param>
/// <returns>The created task waiting for a response status.</returns>
public static async Task <Boolean> ProcessResponse(this Element element, IDownload download, Action <IResponse> callback)
{
var response = await download.Task.ConfigureAwait(false);
var completionStatus = new TaskCompletionSource <Boolean>();
element.Owner.QueueTask(() =>
{
if (response != null)
{
try
{
callback(response);
element.FireSimpleEvent(EventNames.Load);
completionStatus.SetResult(true);
}
catch
{
element.FireSimpleEvent(EventNames.Error);
completionStatus.SetResult(false);
}
finally
{
response.Dispose();
}
}
else
{
element.FireSimpleEvent(EventNames.Error);
completionStatus.SetResult(false);
}
});
return(await completionStatus.Task.ConfigureAwait(false));
}
public DownloadStartedEventArgs(IDownload download, DownloadCheckResult checkResult,
long alreadyDownloadedSize = 0)
{
this.Download = download;
this.CheckResult = checkResult;
this.AlreadyDownloadedSize = alreadyDownloadedSize;
}
/// <summary>
/// For more information, see:
/// http://www.w3.org/html/wg/drafts/html/master/embedded-content.html#update-the-image-data
/// </summary>
void GetImage(Url source)
{
if (source.IsInvalid)
{
source = null;
}
else if (_img != null && source.Equals(_img.Source))
{
return;
}
if (_download != null && !_download.IsCompleted)
{
_download.Cancel();
}
var document = Owner;
if (source != null && document != null)
{
var loader = document.Loader;
if (loader != null)
{
var request = this.CreateRequestFor(source);
var download = loader.DownloadAsync(request);
var task = this.ProcessResource <IImageInfo>(download, result => _img = result);
document.DelayLoad(task);
_download = download;
}
}
}
public DownloadDataReceivedEventArgs(IDownload download, byte[] data, int offset, int count)
{
this.Download = download;
this.Data = data;
this.Offset = offset;
this.Count = count;
}
protected override void OnDetach(IDownload download)
{
download.DownloadStarted -= downloadStarted;
download.DownloadCancelled -= downloadCancelled;
download.DownloadCompleted -= downloadCompleted;
download.DataReceived -= downloadDataReceived;
}
public IDownload Get(string id)
{
IDownload dl = null;
_cache.TryGetValue(id, out dl);
return(dl);
}
/// <summary>
/// Adds the download to the active downloads.
/// </summary>
/// <param name="download">The download to add.</param>
protected virtual void Add(IDownload download)
{
lock (this)
{
_downloads.Add(download);
}
}
public TestApp()
{
_download = Substitute.For <IDownload>();
_api = Substitute.For <IApi>();
_logger = Substitute.For <IConsoleLogger>();
_function = Substitute.For <IFunction>();
}
/// <summary>
/// Removes the download from the active downloads.
/// </summary>
/// <param name="download">The download to remove.</param>
protected virtual void Remove(IDownload download)
{
lock (this)
{
_downloads.Remove(download);
}
}
public DownloadDataReceivedEventArgs(IDownload download, byte[] data, int offset, int count)
{
this.Download = download;
this.Data = data;
this.Offset = offset;
this.Count = count;
}
/// <summary>
/// Adds the download to the active downloads.
/// </summary>
/// <param name="download">The download to add.</param>
protected virtual void Add(IDownload download)
{
lock (this)
{
_downloads.Add(download);
}
}
static void Main(string[] args)
{
ServicePointManager.DefaultConnectionLimit = 10;
ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => true;
IStorage storage = new LocalStorage();
IHttpService httpService = new HttpService();
var downloadManager = new Core.Managers.DownloadManager(httpService, storage);
downloadManager.DownloadUpdated.Subscribe(download =>
{
if (download.Status == DownloadStatus.Complete)
{
System.Console.WriteLine($"Completed: {download.Url}");
}
});
downloadManager.DownloadProgress.Subscribe(i =>
{
System.Console.WriteLine($"Count: {i}");
});
for (int i = 0; i < 100; ++i)
{
var fileUrl = $"http://localhost:59901/api/values?size={(1024 * 1024 * 8) + i}";
IDownload download = downloadManager.DownloadFile(fileUrl);
var x = i;
if (x == 0)
{
download.Progress.Subscribe(d => { System.Console.WriteLine($"{x} ==> {d}%"); });
}
}
System.Console.ReadKey();
}
public DownloadViewModel(IDownload download)
{
Name = download.Name;
var dp = download.Progress
.Sample(TimeSpan.FromMilliseconds(100))
.Buffer(2, 1)
.Select(progList => new DownloadProgressCombined(progList.Last(), progList.First()));
speed = dp.Select(p => p.Speed.Humanize("G03")).ToProperty(this, x => x.Speed);
downloaded = download.Progress
.Sample(TimeSpan.FromMilliseconds(250)).Select(p => p.Downloaded.Humanize("G03"))
.ToProperty(this, x => x.Downloaded);
var dpRealtime = download.Progress
.Sample(TimeSpan.FromSeconds(1.0 / 60));
progress = dpRealtime.Where(p => p.Size.Bits > 0).Select(p => 100d * p.Downloaded.Bits / p.Size.Bits)
.ToProperty(this, x => x.Progress);
size = dpRealtime.Select(p => p.Size.Humanize("G03")).ToProperty(this, x => x.Size);
state = dpRealtime.Select(p => p.State.Humanize()).ToProperty(this, x => x.State);
LogErrors(downloaded);
LogErrors(progress);
LogErrors(size);
LogErrors(speed);
LogErrors(state);
}
private void StartDownload(object sender, RoutedEventArgs e)
{
var lines = textBox.Text.Split(new [] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
foreach (var line in lines)
{
var url = line;
IDownload download = null;
if (url.StartsWith("rtmp://"))
{
download = new RtmpDownload();
download.Start(url, null);
}
else if (url.StartsWith("rtmfp://"))
{
download = new DownloadProtocol();
download.Start(url, null);
}
Settings.Default.DowloadHistory.Add(url);
MainWindow.DownloadList.Add(download);
}
Settings.Default.Save();
Close();
}
private void DownloadVideo(object sender, RoutedEventArgs e)
{
var url = textBox.Text;
Settings.Default.LastURL = url;
Settings.Default.Save();
IDownload download = null;
var dialog = new SaveFileDialog()
{
CheckFileExists = false, AddExtension = true, OverwritePrompt = true, CreatePrompt = false, CheckPathExists = false, DefaultExt = ".flv",
Filter = "Flash 视频|*.flv" // Filter files by extension
};
if (dialog.ShowDialog(this) != true)
{
return;
}
if (url.StartsWith("rtmp://"))
{
download = new RtmpDownload();
download.Start(url, dialog.FileName);
}
else if (url.StartsWith("rtmfp://"))
{
download = new DownloadProtocol();
download.Start(url, dialog.FileName);
}
Settings.Default.DowloadHistory.Add(url);
Settings.Default.Save();
DownloadList.Add(download);
listBox.SelectedItem = download;
}
public void TearDown()
{
_content = null;
if (File.Exists(imagePath))
{
File.Delete(imagePath);
}
}
protected void StartDownload(ResourceRequest request)
{
if (_download != null && !_download.IsCompleted)
{
_download.Cancel();
}
_download = _loader.DownloadAsync(request);
}
public void UpdateValues(IDownload download)
{
_download = download;
Id = _download.Id;
Name = _download.Name;
DownloadStatus = _download.Status;
PercentDownloaded = _download.PercentDownloaded > 0 ? _download.PercentDownloaded : 0;
PercentString = $"{(int)(PercentDownloaded * 100)} %";
}
public void Detach(IDownload download)
{
lock (this.monitor)
{
this.attachedDownloads.Remove(download);
}
this.OnDetach(download);
}
private void CloseDownload()
{
if (this.currentDownload != null)
{
this.currentDownload.DetachAllHandlers();
this.currentDownload.Stop();
this.currentDownload = null;
}
}
public void SetUp()
{
uri = new Uri("https://www.google.com/");
imageUri = new Uri("https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png");
imageName = "TestImage.jpg";
imageNameNoExtension = "TestImageNoExtension";
fileExtension = imageUri.GetFileExtension();
_content = new DownloadAsync();
}
public void Detach(IDownload download)
{
lock (this.monitor)
{
this.attachedDownloads.Remove(download);
}
this.OnDetach(download);
}
public Exception Download(DownloadInfo downloadInfo)
{
IDownload sourceFilter = null;
try
{
downloadThread = System.Threading.Thread.CurrentThread;
this.downloadResult = 0;
this.downloadFinished = false;
this.cancelled = false;
sourceFilter = (IDownload) new MPUrlSourceSplitter();
String url = UrlBuilder.GetFilterUrl(downloadInfo.Util, downloadInfo.Url);
IDownload downloadFilter = (IDownload)sourceFilter;
int result = downloadFilter.DownloadAsync(url, downloadInfo.LocalFile, this);
// throw exception if error occured while initializing download
Marshal.ThrowExceptionForHR(result);
while (!this.downloadFinished)
{
long total = 0;
long current = 0;
if (downloadFilter.QueryProgress(out total, out current) >= 0)
{
// succeeded or estimated value
downloadInfo.DownloadProgressCallback(total, current);
}
// sleep some time
System.Threading.Thread.Sleep(100);
if (this.cancelled)
{
downloadFilter.AbortOperation();
this.downloadFinished = true;
this.downloadResult = 0;
}
}
// throw exception if error occured while downloading
Marshal.ThrowExceptionForHR(this.downloadResult);
return(null);
}
catch (Exception ex)
{
return(ex);
}
finally
{
if (sourceFilter != null)
{
Marshal.ReleaseComObject(sourceFilter);
}
}
}
static void Main(string[] args)
{
// indicates if firstrun action is to be performed
Boolean firstRunner = false;
// set specified configuration file
ExeConfigurationFileMap configFileMap = new ExeConfigurationFileMap();
if (args.Length > 0)
{
configFileMap.ExeConfigFilename = args[0]; // full path to the config file
}
if (args.Length > 1)
{
firstRunner = Boolean.Parse(args[1]); // firstrun action, if not specified -> False
}
// Get the mapped configuration file
Configuration config = ConfigurationManager.OpenMappedExeConfiguration(configFileMap, ConfigurationUserLevel.None);
//now on use config object
AppSettingsSection section = (AppSettingsSection)config.GetSection("appSettings");
//// read custom configuration for download OIS Settings
OISServiceConfigurationSection oisSection = config.GetSection("oisdata") as OISServiceConfigurationSection;
if (oisSection != null)
{
OISServiceConfigurationElementCollection collection = oisSection.CustomConfigurations;
foreach (OISServiceConfigurationElement element in collection)
{
IDownload id = DownloadFactory.createDownloader(element.Type);
id.setImportSettings(element);
Console.WriteLine(config.ConnectionStrings.ConnectionStrings["database"].ConnectionString);
id.setExportSettings(new SqlConnection(config.ConnectionStrings.ConnectionStrings["database"].ConnectionString));
id.startDownload(firstRunner);
}
}
//// read custom configuration for scraping Web Settings
ScrapingServiceConfigurationSection scrapingSection = config.GetSection("scraping") as ScrapingServiceConfigurationSection;
if (scrapingSection != null)
{
ScrapingServiceConfigurationElementCollection collection = scrapingSection.CustomConfigurations;
foreach (ScrapingServiceConfigurationElement element in collection)
{
IDownload id = DownloadFactory.createDownloader(element.Type);
id.setImportSettings(element);
Console.WriteLine(config.ConnectionStrings.ConnectionStrings["database"].ConnectionString);
id.setExportSettings(new SqlConnection(config.ConnectionStrings.ConnectionStrings["database"].ConnectionString));
id.startDownload(firstRunner);
}
}
}
internal Scrapper(IParseHtml parseHtml, IDownload download)
{
_parseHtml = parseHtml;
_download = download;
Selected = DomainSettings.SelectedConfiguration;
if (Selected == null)
{
throw new MangaScrapperException("Error In Configuration Selection");
}
}
static DownloadFactory()
{
if (Instance == null)
{
lock (lockObj)
{
Instance = new Download.Implements.FdfsDownload();
}
}
}
public virtual Task ProcessAsync(ResourceRequest request)
{
if (IsDifferentToCurrentDownloadUrl(request.Target))
{
CancelDownload();
Download = _loader.DownloadAsync(request);
return FinishDownloadAsync();
}
return null;
}
/// <summary>
/// Processes the given request asynchronously.
/// </summary>
public virtual Task ProcessAsync(ResourceRequest request)
{
if (IsAvailable && IsDifferentToCurrentDownloadUrl(request.Target))
{
CancelDownload();
Download = _loader.FetchAsync(request);
return(FinishDownloadAsync());
}
return(null);
}
public void Attach(IDownload download)
{
if (download == null)
throw new ArgumentNullException("download");
lock (this.monitor)
{
this.attachedDownloads.Add(download);
}
this.OnAttach(download);
}
public void Detach(IDownload download)
{
if (download == null)
throw new ArgumentNullException("download");
download.DataReceived -= this.downloadDataReceived;
lock (this.monitor)
{
this.downloads.Remove(download);
}
}
private void BuildDownload()
{
lock (this.monitor)
{
if (this.DoStopIfNecessary())
{
return;
}
int? currentMaxReadBytes = this.maxReadBytes.HasValue ? (int?)this.maxReadBytes.Value - this.sumOfBytesRead : null;
this.currentDownload = this.downloadBuilder.Build(this.url, this.bufferSize, this.currentOffset, currentMaxReadBytes);
this.currentDownload.DownloadStarted += downloadStarted;
this.currentDownload.DownloadCancelled += downloadCancelled;
this.currentDownload.DownloadCompleted += downloadCompleted;
this.currentDownload.DataReceived += downloadDataReceived;
StartThread(this.currentDownload.Start, Thread.CurrentThread.Name + "-buildDownload");
}
}
void UpdateSource(String value)
{
if (_download != null)
{
_download.Cancel();
}
var document = Owner;
if (!String.IsNullOrEmpty(value) && document != null)
{
var loader = document.Loader;
if (loader != null)
{
var url = new Url(Source);
var request = this.CreateRequestFor(url);
var download = loader.DownloadAsync(request);
var task = this.ProcessResource<IObjectInfo>(download, result => _obj = result);
document.DelayLoad(task);
_download = download;
}
}
}
private static IDownload CheckIntegrity(this CorsRequest cors, IDownload download)
{
var response = download.Task.Result;
var value = cors.Request.Source?.GetAttribute(AttributeNames.Integrity);
var integrity = cors.Integrity;
if (!String.IsNullOrEmpty(value) && integrity != null && response != null)
{
var content = new MemoryStream();
response.Content.CopyTo(content);
content.Position = 0;
if (!integrity.IsSatisfied(content.ToArray(), value))
{
response.Dispose();
throw new DomException(DomError.Security);
}
return download.Wrap(new Response
{
Address = response.Address,
Content = content,
Headers = response.Headers,
StatusCode = response.StatusCode
});
}
return download;
}
public DownloadStartedEventArgs(IDownload download, DownloadCheckResult checkResult)
{
this.Download = download;
this.CheckResult = checkResult;
}
/// <summary>
/// Backend container of classes to make the programmer's life a little easier.
/// </summary>
public ClassContainer ()
{
this.IOCode = new Storage();
this.DownloadingCode = new Download (this.IOCode);
this.BakedExceptionCode = new PrebakedError ();
}
/// <summary>
/// For more information, see:
/// http://www.w3.org/html/wg/drafts/html/master/embedded-content.html#update-the-image-data
/// </summary>
void GetImage(Url source)
{
if (source.IsInvalid)
{
source = null;
}
else if (_img != null && source.Equals(_img.Source))
{
return;
}
if (_download != null && !_download.IsCompleted)
{
_download.Cancel();
}
var document = Owner;
if (source != null && document != null)
{
var loader = document.Loader;
if (loader != null)
{
var request = this.CreateRequestFor(source);
var download = loader.DownloadAsync(request);
var task = this.ProcessResource<IImageInfo>(download, result => _img = result);
document.DelayLoad(task);
_download = download;
}
}
}
async Task<IDocument> GetDocumentAsync()
{
var referer = _document.DocumentUri;
var loader = _document.Loader;
if (_htmlContent == null && !String.IsNullOrEmpty(_requestUrl) && !_requestUrl.Is(_element.BaseUri) && loader != null)
{
var cancel = _cts.Token;
var url = _element.HyperReference(_requestUrl);
var request = _element.CreateRequestFor(url);
_download = loader.DownloadAsync(request);
cancel.Register(_download.Cancel);
var response = await _download.Task.ConfigureAwait(false);
return await _context.OpenAsync(response, cancel).ConfigureAwait(false);
}
return await _context.OpenAsync(m => m.Content(_htmlContent).Address(referer), _cts.Token).ConfigureAwait(false);
}
private void InitiateStartParams(bool downloadImmediately)
{
this.Icon = formIcon;
this.changeSaveLocation.BackgroundImage = imageList[0];
this.changeTemporaryLocation.BackgroundImage = imageList[0];
this.moveQueuedItemUp.BackgroundImage = imageList[1];
this.moveQueuedItemDown.BackgroundImage = imageList[2];
this.iMainForm = (this as IMainForm);
this.mainForm = this;
this.Validation = (new Validation() as IValidation);
this.Storage = (new Storage(this.iMainForm, this.Validation) as IStorage);
this.Download = (new Download(this.iMainForm, this.Storage) as IDownload);
Storage.ReadFromRegistry();
VideoQueue.Items = Storage.ReadUrls();
this.RefreshQueue(0, false);
writeFileVersionToStatBar();
if (this.queuedBox.Items.Count >= 1)
{
this.urlToModify.Text = VideoQueue.Items [0].Location;
this.resolutionToModify.Value = VideoQueue.Items [0].Resolution;
this.formatToModify.SelectedItem = VideoQueue.Items [0].Format.ToString();
if (downloadImmediately)
{
StartDownloadButtonClick(null, null);
}
}
SchedulingEnabledCheckedChanged(null, null);
}
protected override void OnAttach(IDownload download)
{
download.DataReceived += downloadDataReceived;
}
protected void SetDownload(IDownload download)
{
_download = download;
}
public DownloadCancelledEventArgs(IDownload download, Exception exception)
{
this.Download = download;
this.Exception = exception;
}
private void downloadCancelled(DownloadCancelledEventArgs args)
{
var download = args.Download;
lock (this.monitor)
{
if (download == this.currentDownload)
{
CountRetryAndCancelIfMaxRetriesReached();
if (this.currentDownload != null)
{
this.currentDownload = null;
StartThread(this.SleepThenBuildDownload, Thread.CurrentThread.Name + "-afterCancel");
}
}
}
}
private void CloseDownload()
{
if (this.currentDownload != null)
{
this.currentDownload.DetachAllHandlers();
this.currentDownload.Stop();
this.currentDownload = null;
}
}
protected virtual void OnDetach(IDownload download)
{
}
public DownloadEventArgs(IDownload download)
{
this.Download = download;
}
