public async Task <AsyncRequestState <T> > DownloadAsync <T>(
CrawlStep crawlStep,
CrawlStep referrer,
DownloadMethod method,
Action <AsyncRequestState <T> > completed,
Action <DownloadProgressEventArgs> progress, T state)
{
AspectF.Define.
NotNull(crawlStep, "crawlStep");
if (UserAgent.IsNullOrEmpty())
{
UserAgent = "Mozilla/5.0";
}
AsyncRequestState <T> requestState = new AsyncRequestState <T>
{
CrawlStep = crawlStep,
Referrer = referrer,
State = state,
DownloadProgress = progress,
Retry = RetryCount.HasValue ? RetryCount.Value + 1 : 1,
Method = method,
};
return(await DownloadAsync(requestState));
}
private string GetCacheFileName(CrawlStep step, DownloadMethod method)
{
string fileName = FileSystemHelpers.ToValidFileName(string.Format("{0}_{1}", step.Uri, method));
fileName = Path.Combine(m_CacheFolder, fileName);
return(fileName.Max(248));
}
public void DownloadAsync <T>(CrawlStep crawlStep, CrawlStep referrer, DownloadMethod method,
Action <RequestState <T> > completed, Action <DownloadProgressEventArgs> progress,
T state)
{
AspectF.Define.
NotNull(crawlStep, "crawlStep").
NotNull(completed, "completed");
if (UserAgent.IsNullOrEmpty())
{
UserAgent = "Mozilla/5.0";
}
RequestState <T> requestState = new RequestState <T>
{
DownloadTimer = Stopwatch.StartNew(),
Complete = completed,
CrawlStep = crawlStep,
Referrer = referrer,
State = state,
DownloadProgress = progress,
Retry = RetryCount.HasValue ? RetryCount.Value + 1 : 1,
Method = method,
};
DownloadAsync(requestState, null);
}
public DownloadObject(string eUrl, string sUrl, Country countryO, DownloadMethod downlMethod)
{
EpisodeUrl = eUrl;
SubtitleUrl = sUrl;
CountryOfOrigin = countryO;
DlMethod = downlMethod;
}
public bool DownloadFile(DownloadMethod method, string remoteFileLocation, string localPath)
{
try
{
remoteFileLocation = WebHelper.RemoveTrailingBackslash(remoteFileLocation);
var localFileFullPath = FileHelper.AddJpgExt(FileHelper.GenerateLocalPathToFile(remoteFileLocation, localPath));
switch (method)
{
case DownloadMethod.WebClientAsync:
return(DownloadFileWebClientAsync(remoteFileLocation, localFileFullPath));
case DownloadMethod.PostSharp:
return(DownloadFileRestSharp(remoteFileLocation, localFileFullPath));
case DownloadMethod.WebClient:
return(DownloadFileWebClient(remoteFileLocation, localFileFullPath));
default:
return(false);
}
}
catch
{
return(false);
}
}
/// <summary>
/// Default constructor
/// </summary>
/// <param name="passedData">A download object containing information about the show to download</param>
/// <param name="fName">The filename</param>
public DownloadWindow(DownloadObject passedData, string fName)
{
InitializeComponent();
DataContext = dwBinding;
this.Show();
this.Closed += DownloadWindow_Closed;
url = passedData.EpisodeUrl;
subtitle = passedData.SubtitleUrl;
dlMethod = passedData.DlMethod;
fileName = fName;
cnt = passedData.CountryOfOrigin;
//Clean the name for windows
foreach (var c in Path.GetInvalidFileNameChars())
{
fileName = fileName.Replace(c, '-');
}
//Replace spaces if required
if (Properties.Settings.Default.SpaceReplace)
{
fileName = fileName.Replace(" ", ".");
}
if (!subtitle.Equals("") && Properties.Settings.Default.DownloadSubtitlesSetting)
{
Task.Factory.StartNew(() => DownloadSubtitle(subtitle, fileName, Properties.Settings.Default.SubtitleFormat)); //Run the downloadsub in a background thread
}
DownloadShow();
}
public static string GetHtmlAsync(string url, string postdata, DownloadMethod method)
{
WebDownLoader info = new WebDownLoader();
info.Url = url;
info.Method = method;
info.PostData = postdata;
return(GetHtmlAsync(info));
}
/// <summary>
/// 下载文件
/// </summary>
/// <param name="files"></param>
/// <param name="method"></param>
public async Task <DownloadResult> DownloadFiles(NetDiskFile[] files, DownloadMethod method)
{
return(JsonConvert.DeserializeObject <DownloadResult>(await DataServer.SendPacketAsync(new DownloadPacket()
{
Token = Token,
Info = files,
Method = (int)method
})));
}
public static string GetHtml(string url, string postdata, DownloadMethod method)
{
WebDownLoader info = new WebDownLoader();
info.Url = url;
info.Method = method;
info.PostData = postdata;
info.ConnectionTimeout = new TimeSpan(0, 10, 0);
return(GetHtml(info));
}
public Boolean Download(DownloadMethod method)
{
switch (method)
{
case DownloadMethod.ByUrl:
return(DownloadByUrl(DownloadUrl));
default:
return(false);
}
}
public void DownloadAsync <T>(CrawlStep crawlStep, CrawlStep referrer, DownloadMethod method, Action <RequestState <T> > completed,
Action <DownloadProgressEventArgs> progress, T state)
{
completed(new RequestState <T>
{
DownloadTimer = Stopwatch.StartNew(),
Complete = completed,
CrawlStep = crawlStep,
Referrer = referrer,
State = state,
DownloadProgress = progress,
Retry = RetryCount.HasValue ? RetryCount.Value + 1 : 1,
Method = method,
});
}
private HttpMethod ConvertToHttpMethod(DownloadMethod method)
{
switch (method)
{
case DownloadMethod.GET:
return(HttpMethod.Get);
case DownloadMethod.HEAD:
return(HttpMethod.Head);
case DownloadMethod.POST:
return(HttpMethod.Post);
default:
throw new NotSupportedException();
}
}
public PropertyBag Download(CrawlStep crawlStep, DownloadMethod method)
{
PropertyBag result = new PropertyBag
{
Step = crawlStep,
CharacterSet = string.Empty,
ContentEncoding = string.Empty,
ContentType = "text/html",
Headers = null,
IsMutuallyAuthenticated = false,
IsFromCache = false,
LastModified = DateTime.UtcNow,
Method = "GET",
ProtocolVersion = new Version(3, 0),
ResponseUri = crawlStep.Uri,
Server = "N/A",
StatusCode = HttpStatusCode.OK,
StatusDescription = "OK",
Response = Encoding.UTF8.GetBytes(Resources.ncrawler_codeplex_com),
DownloadTime = TimeSpan.FromSeconds(1),
};
return(result);
}
public PropertyBag Download(CrawlStep crawlStep, DownloadMethod method)
{
PropertyBag result = new PropertyBag
{
Step = crawlStep,
CharacterSet = string.Empty,
ContentEncoding = string.Empty,
ContentType = "text/html",
Headers = null,
IsMutuallyAuthenticated = false,
IsFromCache = false,
LastModified = DateTime.UtcNow,
Method = "GET",
ProtocolVersion = new Version(3, 0),
ResponseUri = crawlStep.Uri,
Server = "N/A",
StatusCode = HttpStatusCode.OK,
StatusDescription = "OK",
Response = Encoding.UTF8.GetBytes(Resources.ncrawler_codeplex_com),
DownloadTime = TimeSpan.FromSeconds(1),
};
return result;
}
public Task <RequestState <T> > DownloadAsync <T>(CrawlStep crawlStep, CrawlStep referrer, DownloadMethod method, Func <RequestState <T>, Task> completed, Action <DownloadProgressEventArgs> progress, T state)
{
var result = new RequestState <T>
{
StartTime = DateTime.UtcNow,
Complete = completed,
CrawlStep = crawlStep,
Referrer = referrer,
State = state,
DownloadProgress = progress,
Retry = this.RetryCount.HasValue ? this.RetryCount.Value + 1 : 1,
Method = ConvertToHttpMethod(method),
};
return(Task.Factory.StartNew(() =>
{
completed(result);
return result;
}));
}
public Task <PropertyBag> DownloadAsync(CrawlStep crawlStep, CrawlStep referrer = null, DownloadMethod method = DownloadMethod.GET)
{
var result = new PropertyBag
{
Step = crawlStep,
CharacterSet = string.Empty,
ContentEncoding = string.Empty,
ContentType = "text/html",
Headers = null,
IsMutuallyAuthenticated = false,
IsFromCache = false,
LastModified = DateTime.UtcNow,
Method = "GET",
ProtocolVersion = new Version(3, 0),
ResponseUri = crawlStep.Uri,
Server = "N/A",
StatusCode = HttpStatusCode.OK,
StatusDescription = "OK",
GetResponse = () => new MemoryStream(Encoding.UTF8.GetBytes(Resources.ncrawler_codeplex_com)),
DownloadTime = TimeSpan.FromSeconds(1),
};
return(Task.FromResult(result));
}
private string GetCacheFileName(CrawlStep step, DownloadMethod method)
{
string fileName = FileSystemHelpers.ToValidFileName(string.Format("{0}_{1}", step.Uri, method));
fileName = Path.Combine(m_CacheFolder, fileName);
return fileName.Max(248);
}
/// <summary>
/// Gets or Sets a value indicating if cookies will be stored.
/// </summary>
public PropertyBag Download(CrawlStep crawlStep, DownloadMethod method)
{
AspectF.Define.
NotNull(crawlStep, "crawlStep");
if (UserAgent.IsNullOrEmpty())
{
UserAgent = "Mozilla/5.0";
}
if (m_CacheEnabled)
{
if (CacheEntryExists(crawlStep, method))
{
return GetCacheEntry(crawlStep, method);
}
}
HttpWebRequest req = (HttpWebRequest) WebRequest.Create(crawlStep.Uri);
req.Method = method.ToString();
req.AllowAutoRedirect = true;
req.UserAgent = UserAgent;
req.Accept = "*/*";
req.KeepAlive = true;
if (ConnectionTimeout.HasValue)
{
req.Timeout = Convert.ToInt32(ConnectionTimeout.Value.TotalMilliseconds);
}
if (ReadTimeout.HasValue)
{
req.ReadWriteTimeout = Convert.ToInt32(ReadTimeout.Value.TotalMilliseconds);
}
if (UseCookies)
{
req.CookieContainer = CookieContainer;
}
Stopwatch downloadTimer = Stopwatch.StartNew();
HttpWebResponse resp;
try
{
resp = (HttpWebResponse) req.GetResponse();
}
catch (WebException we)
{
resp = we.Response as HttpWebResponse;
if (resp.IsNull())
{
throw;
}
}
using (resp)
using (Stream responseStream = resp.GetResponseStream())
{
downloadTimer.Stop();
PropertyBag result = new PropertyBag
{
Step = crawlStep,
CharacterSet = resp.CharacterSet,
ContentEncoding = resp.ContentEncoding,
ContentType = resp.ContentType,
Headers = resp.Headers,
IsMutuallyAuthenticated = resp.IsMutuallyAuthenticated,
IsFromCache = resp.IsFromCache,
LastModified = resp.LastModified,
Method = resp.Method,
ProtocolVersion = resp.ProtocolVersion,
ResponseUri = resp.ResponseUri,
Server = resp.Server,
StatusCode = resp.StatusCode,
StatusDescription = resp.StatusDescription,
Response = CopyStreamToMemory(responseStream, MaximumContentSize),
DownloadTime = downloadTimer.Elapsed,
};
if (m_CacheEnabled)
{
WriteCacheEntry(crawlStep, method, result);
}
return result;
}
}
private void WriteCacheEntry(CrawlStep step, DownloadMethod method, PropertyBag result)
{
File.WriteAllBytes(GetCacheFileName(step, method), result.ToBinary());
}
public async Task <PropertyBag> DownloadAsync(CrawlStep crawlStep, CrawlStep referrer, DownloadMethod method)
{
return(await DownloadInternalSync(crawlStep, referrer, method));
}
private bool CacheEntryExists(CrawlStep step, DownloadMethod method)
{
return(FileSystemHelpers.FileExists(GetCacheFileName(step, method)));
}
/// <summary>
/// Gets or Sets a value indicating if cookies will be stored.
/// </summary>
public PropertyBag Download(CrawlStep crawlStep, DownloadMethod method)
{
AspectF.Define.
NotNull(crawlStep, "crawlStep");
if (UserAgent.IsNullOrEmpty())
{
UserAgent = "Mozilla/5.0";
}
if (m_CacheEnabled)
{
if (CacheEntryExists(crawlStep, method))
{
return(GetCacheEntry(crawlStep, method));
}
}
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(crawlStep.Uri);
req.Method = method.ToString();
req.AllowAutoRedirect = true;
req.UserAgent = UserAgent;
req.Accept = "*/*";
req.KeepAlive = true;
if (ConnectionTimeout.HasValue)
{
req.Timeout = Convert.ToInt32(ConnectionTimeout.Value.TotalMilliseconds);
}
if (ReadTimeout.HasValue)
{
req.ReadWriteTimeout = Convert.ToInt32(ReadTimeout.Value.TotalMilliseconds);
}
if (UseCookies)
{
req.CookieContainer = CookieContainer;
}
Stopwatch downloadTimer = Stopwatch.StartNew();
HttpWebResponse resp;
try
{
resp = (HttpWebResponse)req.GetResponse();
}
catch (WebException we)
{
resp = we.Response as HttpWebResponse;
if (resp.IsNull())
{
throw;
}
}
using (resp)
using (Stream responseStream = resp.GetResponseStream())
{
downloadTimer.Stop();
PropertyBag result = new PropertyBag
{
Step = crawlStep,
CharacterSet = resp.CharacterSet,
ContentEncoding = resp.ContentEncoding,
ContentType = resp.ContentType,
Headers = resp.Headers,
IsMutuallyAuthenticated = resp.IsMutuallyAuthenticated,
IsFromCache = resp.IsFromCache,
LastModified = resp.LastModified,
Method = resp.Method,
ProtocolVersion = resp.ProtocolVersion,
ResponseUri = resp.ResponseUri,
Server = resp.Server,
StatusCode = resp.StatusCode,
StatusDescription = resp.StatusDescription,
Response = CopyStreamToMemory(responseStream, MaximumContentSize),
DownloadTime = downloadTimer.Elapsed,
};
if (m_CacheEnabled)
{
WriteCacheEntry(crawlStep, method, result);
}
return(result);
}
}
/// <summary>
/// Gets or Sets a value indicating if cookies will be stored.
/// </summary>
private async Task <PropertyBag> DownloadInternalSync(CrawlStep crawlStep, CrawlStep referrer, DownloadMethod method)
{
PropertyBag result = null;
Exception ex = null;
using (var resetEvent = new ManualResetEvent(false))
{
await DownloadAsync <object>(crawlStep, referrer, method,
(RequestState <object> state) =>
{
if (state.Exception.IsNull())
{
result = state.PropertyBag;
if (!result.GetResponse.IsNull())
{
using (var response = result.GetResponse())
{
byte[] data;
if (response is MemoryStream)
{
data = ((MemoryStream)response).ToArray();
}
else
{
using (var copy = response.CopyToMemory())
{
data = copy.ToArray();
}
}
result.GetResponse = () => new MemoryStream(data);
}
}
}
else
{
ex = state.Exception;
}
resetEvent.Set();
return(Task.FromResult(0));
}, null, null);
resetEvent.WaitOne();
}
if (!ex.IsNull())
{
throw new Exception("Error write downloading {0}".FormatWith(crawlStep.Uri), ex);
}
return(result);
}
private void WriteCacheEntry(CrawlStep step, DownloadMethod method, PropertyBag result)
{
File.WriteAllBytes(GetCacheFileName(step, method), result.ToBinary());
}
public PropertyBag Download(CrawlStep crawlStep, CrawlStep referrer, DownloadMethod method)
{
return(DownloadInternalSync(crawlStep, referrer, method));
}
private bool CacheEntryExists(CrawlStep step, DownloadMethod method)
{
return FileSystemHelpers.FileExists(GetCacheFileName(step, method));
}
private PropertyBag GetCacheEntry(CrawlStep step, DownloadMethod method)
{
return File.ReadAllBytes(GetCacheFileName(step, method)).FromBinary<PropertyBag>();
}
public async Task <RequestState <T> > DownloadAsync <T>(CrawlStep crawlStep, CrawlStep referrer, DownloadMethod method,
Func <RequestState <T>, Task> completed, Action <DownloadProgressEventArgs> progress,
T state)
{
AspectF.Define.
NotNull(crawlStep, "crawlStep").
NotNull(completed, "completed");
if (this.UserAgent.IsNullOrEmpty())
{
this.UserAgent = "Mozilla/5.0";
}
var requestState = new RequestState <T>
{
DownloadTimer = Stopwatch.StartNew(),
Complete = completed,
CrawlStep = crawlStep,
Referrer = referrer,
State = state,
DownloadProgress = progress,
Retry = this.RetryCount.HasValue ? this.RetryCount.Value + 1 : 1,
Method = ConvertToHttpMethod(method),
};
return(await this.DownloadAsync(requestState, null));
}
private PropertyBag GetCacheEntry(CrawlStep step, DownloadMethod method)
{
return(File.ReadAllBytes(GetCacheFileName(step, method)).FromBinary <PropertyBag>());
}
public async Task <PropertyBag> DownloadAsync(CrawlStep crawlStep, CrawlStep referrer, DownloadMethod method)
{
return(await this.DownloadInternalSync(crawlStep, referrer, method).ConfigureAwait(false));
}
