protected WebRequest GetRequest(ResourceLocation location)
{
WebRequest request = WebRequest.Create(location.URL);
request.Timeout = 30000;
SetProxy(request);
return request;
}
public override RemoteFileInfo GetFileInfo(ResourceLocation rl, out Stream stream)
{
stream = null;
mappedUrl = null;
String title;
String pageData = GetPageData(rl);
mappedUrl = ResolveVideoURL(rl.URL, pageData, out title);
WebRequest request = this.GetRequest(mappedUrl);
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
RemoteFileInfo result = new RemoteFileInfo();
result.FileSize = response.ContentLength;
result.AcceptRanges = String.Compare(response.Headers["Accept-Ranges"], "bytes", true) == 0;
if (response.ResponseUri != null)
{
mappedUrl.URL = response.ResponseUri.OriginalString;
}
else
{
stream = response.GetResponseStream();
}
return result;
}
}
private Downloader(
int id,
ResourceLocation rl,
ResourceLocation[] mirrors,
string localFile)
{
this.threads = new List<Thread>();
this.id = id;
this.resourceLocation = rl;
if (mirrors == null)
{
this.mirrors = new List<ResourceLocation>();
}
else
{
this.mirrors = new List<ResourceLocation>(mirrors);
}
this.localFile = localFile;
extentedProperties = new Dictionary<string, object>();
defaultDownloadProvider = rl.BindProtocolProviderInstance(this);
segmentCalculator = new MinSizeSegmentCalculator();
this.MirrorSelector = new SequentialMirrorSelector();
}
public Downloader(
int id,
ResourceLocation rl,
ResourceLocation[] mirrors,
string localFile,
List<Segment> segments,
RemoteFileInfo remoteInfo,
int requestedSegmentCount,
DateTime createdDateTime):
this(id, rl, mirrors, localFile)
{
if (segments.Count > 0)
{
SetState(DownloaderState.Prepared);
}
else
{
SetState(DownloaderState.NeedToPrepare);
}
this.createdDateTime = createdDateTime;
this.remoteFileInfo = remoteInfo;
this.requestedSegmentCount = requestedSegmentCount;
this.segments = segments;
}
public RemoteFileInfo GetFileInfo(ResourceLocation rl, out Stream stream)
{
FtpWebRequest request;
RemoteFileInfo result = new RemoteFileInfo();
result.AcceptRanges = true;
stream = null;
request = (FtpWebRequest)GetRequest(rl);
request.Method = WebRequestMethods.Ftp.GetFileSize;
FillCredentials(request, rl);
using (FtpWebResponse response = (FtpWebResponse)request.GetResponse())
{
result.FileSize = response.ContentLength;
}
request = (FtpWebRequest)GetRequest(rl);
request.Method = WebRequestMethods.Ftp.GetDateTimestamp;
FillCredentials(request, rl);
using (FtpWebResponse response = (FtpWebResponse)request.GetResponse())
{
result.LastModified = response.LastModified;
}
return result;
}
public string GetTitle(ResourceLocation rl)
{
String pageData = GetPageData(rl);
string title;
mappedUrl = ResolveVideoURL(rl.URL, pageData, out title);
return title;
}
public ZipRemoteFile(ResourceLocation url, IProtocolProvider protocolProvider)
{
if (protocolProvider == null)
{
throw new ArgumentNullException("protocolProvider");
}
this.url = url;
this.protocolProvider = protocolProvider;
}
private String GetPageData(ResourceLocation rl)
{
String pageData;
using (StreamReader sr = new StreamReader(this.CreateStream(rl, 0, 0)))
{
pageData = sr.ReadToEnd();
}
return pageData;
}
public RemoteFileInfo GetFileInfo(ResourceLocation rl, out System.IO.Stream stream)
{
RemoteFileInfo result = proxy.GetFileInfo(rl, out stream);
if (stream != null)
{
stream = new LimitedRateStreamProxy(stream, speedLimit);
}
return result;
}
public Stream CreateStream(ResourceLocation rl, long initialPosition, long endPosition)
{
/*
* a chamada desse metodo nao sera apos o GetFileInfo se o download for resumeable
* sendo assim, preciso ter algum repositorio comum para guardas os "remoteZipFile"
* assim evita-se de ficar carregando as entries a cada segmento/arquivo.
*
* outro, pronto, apos o uso do segmento, deve-se liberar o remoteZipFile se não houver
* nenhum outro download vinculado a este "remoteZipFile"
*/
return remoteZipFile.GetInputStream(entry);
}
public Downloader(
ResourceLocation rl,
ResourceLocation[] mirrors,
string localFile,
int segmentCount)
: this(rl, mirrors, localFile)
{
SetState(DownloaderState.NeedToPrepare);
this.createdDateTime = DateTime.Now;
this.requestedSegmentCount = segmentCount;
this.segments = new List<Segment>();
}
public static ResourceLocation FromURL(
string url,
bool authenticate,
string login,
string password)
{
ResourceLocation rl = new ResourceLocation();
rl.URL = url;
rl.Authenticate = authenticate;
rl.Login = login;
rl.Password = password;
return rl;
}
public Stream CreateStream(ResourceLocation rl, long initialPosition, long endPosition)
{
FtpWebRequest request = (FtpWebRequest)GetRequest(rl);
FillCredentials(request, rl);
request.Method = WebRequestMethods.Ftp.DownloadFile;
request.ContentOffset = initialPosition;
WebResponse response = request.GetResponse();
return response.GetResponseStream();
}
public virtual RemoteFileInfo GetFileInfo(ResourceLocation rl, out Stream stream)
{
HttpWebRequest request = (HttpWebRequest)GetRequest(rl);
FillCredentials(request, rl);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
RemoteFileInfo result = new RemoteFileInfo();
result.MimeType = response.ContentType;
result.LastModified = response.LastModified;
result.FileSize = response.ContentLength;
result.AcceptRanges = String.Compare(response.Headers["Accept-Ranges"], "bytes", true) == 0;
stream = response.GetResponseStream();
return result;
}
public RemoteFileInfo GetFileInfo(ResourceLocation rl, out System.IO.Stream stream)
{
stream = null;
remoteZipFile = new ZipRemoteFile(rl);
remoteZipFile.Load();
string entryName = GetZipEntryNameProperty(this.downloader);
entry = remoteZipFile[entryName];
RemoteFileInfo result = new RemoteFileInfo();
result.AcceptRanges = false; // TODO make resumeable
result.FileSize = entry.Size;
result.LastModified = entry.DateTime;
result.MimeType = "application/zip";
return result;
}
private void FillCredentials(HttpWebRequest request, ResourceLocation rl)
{
if (rl.Authenticate)
{
string login = rl.Login;
string domain = string.Empty;
int slashIndex = login.IndexOf('\\');
if (slashIndex >= 0)
{
domain = login.Substring(0, slashIndex );
login = login.Substring(slashIndex + 1);
}
NetworkCredential myCred = new NetworkCredential(login, rl.Password);
myCred.Domain = domain;
request.Credentials = myCred;
}
}
public virtual Stream CreateStream(ResourceLocation rl, long initialPosition, long endPosition)
{
HttpWebRequest request = (HttpWebRequest)GetRequest(rl);
FillCredentials(request, rl);
if (initialPosition != 0)
{
if (endPosition == 0)
{
request.AddRange((int)initialPosition);
}
else
{
request.AddRange((int)initialPosition, (int)endPosition);
}
}
WebResponse response = request.GetResponse();
return response.GetResponseStream();
}
private void SegmentThreadProc(object objSegment)
{
Segment segment = (Segment)objSegment;
segment.LastError = null;
try
{
if (segment.EndPosition > 0 && segment.StartPosition >= segment.EndPosition)
{
segment.State = SegmentState.Finished;
// raise the event
OnSegmentStoped(segment);
return;
}
int buffSize = 8192;
byte[] buffer = new byte[buffSize];
segment.State = SegmentState.Connecting;
// raise the event
OnSegmentStarting(segment);
if (segment.InputStream == null)
{
// get the next URL (It can the the main url or some mirror)
ResourceLocation location = this.MirrorSelector.GetNextResourceLocation();
// get the protocol provider for that mirror
IProtocolProvider provider = location.BindProtocolProviderInstance(this);
while (location != this.ResourceLocation)
{
Stream tempStream;
// get the remote file info on mirror
RemoteFileInfo tempRemoteInfo = provider.GetFileInfo(location, out tempStream);
if (tempStream != null)
{
tempStream.Dispose();
}
// check if the file on mirror is the same
if (tempRemoteInfo.FileSize == remoteFileInfo.FileSize &&
tempRemoteInfo.AcceptRanges == remoteFileInfo.AcceptRanges)
{
// if yes, stop looking for the mirror
break;
}
lock (mirrors)
{
// the file on the mirror is not the same, so remove from the mirror list
mirrors.Remove(location);
}
// the file on the mirror is different
// so get other mirror to use in the segment
location = this.MirrorSelector.GetNextResourceLocation();
provider = location.BindProtocolProviderInstance(this);
}
// get the input stream from start position
segment.InputStream = provider.CreateStream(location, segment.StartPosition, segment.EndPosition);
// change the segment URL to the mirror URL
segment.CurrentURL = location.URL;
}
else
{
// change the segment URL to the main URL
segment.CurrentURL = this.resourceLocation.URL;
}
using (segment.InputStream)
{
// raise the event
OnSegmentStarted(segment);
// change the segment state
segment.State = SegmentState.Downloading;
segment.CurrentTry = 0;
long readSize;
do
{
// reads the buffer from input stream
segment.InputStream.ReadTimeout = streamTimeout;
readSize = segment.InputStream.Read(buffer, 0, buffSize);
// check if the segment has reached the end
if (segment.EndPosition > 0 &&
segment.StartPosition + readSize > segment.EndPosition)
{
// adjust the 'readSize' to write only necessary bytes
readSize = (segment.EndPosition - segment.StartPosition);
if (readSize <= 0)
{
segment.StartPosition = segment.EndPosition;
break;
}
}
// locks the stream to avoid that other threads changes
// the position of stream while this thread is writing into the stream
lock (segment.OutputStream)
{
segment.OutputStream.Position = segment.StartPosition;
segment.OutputStream.Write(buffer, 0, (int)readSize);
}
// increse the start position of the segment and also calculates the rate
segment.IncreaseStartPosition(readSize);
// check if the stream has reached its end
if (segment.EndPosition > 0 && segment.StartPosition >= segment.EndPosition)
{
segment.StartPosition = segment.EndPosition;
break;
}
// check if the user have requested to pause the download
if (state == DownloaderState.Pausing)
{
segment.State = SegmentState.Paused;
break;
}
//Thread.Sleep(1500);
}while (readSize > 0);
if (segment.State == SegmentState.Downloading)
{
segment.State = SegmentState.Finished;
// try to create other segment,
// spliting the missing bytes from one existing segment
AddNewSegmentIfNeeded();
}
}
// raise the event
OnSegmentStoped(segment);
}
catch (Exception ex)
{
// store the error information
segment.State = SegmentState.Error;
segment.LastError = ex;
Debug.WriteLine(ex.Message);
// raise the event
OnSegmentFailed(segment);
}
finally
{
// clean up the segment
segment.InputStream = null;
}
}
public ZipRemoteFile(ResourceLocation url) :
this(url, ProtocolProviderFactory.GetProvider(url.URL))
{
}
public System.IO.Stream CreateStream(ResourceLocation rl, long initialPosition, long endPosition)
{
return new LimitedRateStreamProxy(proxy.CreateStream(rl, initialPosition, endPosition), speedLimit);
}
private static ResourceLocation GetNumberURL(ResourceLocation locationBase, int i, int wildcard)
{
ResourceLocation r = locationBase.Clone();
r.URL = locationBase.URL.Replace("(*)", i.ToString().PadLeft(wildcard, '0'));
return r;
}
public void AddDownloadURLs(ResourceLocation[] args, int segments, string path, int nrOfSubfolders)
{
if (args == null) return;
if (path == null)
{
//path = PathHelper.GetWithBackslash(Core.Settings.Default.DownloadFolder);
path = PathHelper.GetWithBackslash(DownloadManager.Instance.DefaultDownloadDirectory);
}
else
{
path = PathHelper.GetWithBackslash(path);
}
try
{
DownloadManager.Instance.OnBeginAddBatchDownloads();
foreach (ResourceLocation rl in args)
{
Uri uri = new Uri(rl.URL);
string fileName = uri.Segments[uri.Segments.Length - 1];
fileName = HttpUtility.UrlDecode(fileName).Replace("/", "\\");
DownloadManager.Instance.Add(rl, null, path + fileName, segments, false);
}
}
finally
{
DownloadManager.Instance.OnEndAddBatchDownloads();
}
}
private ResourceLocation GetCharURL(ResourceLocation locationBase, string c)
{
ResourceLocation r = locationBase.Clone();
r.URL = locationBase.URL.Replace("(*)", c);
return r;
}
public Downloader Add(ResourceLocation rl, ResourceLocation[] mirrors, string localFile, int segments, bool autoStart)
{
Downloader d = new Downloader(rl, mirrors, localFile, segments);
Add(d, autoStart);
return d;
}
private void AddDownloadsFromZip(TreeNodeCollection nodes, ResourceLocation[] mirrors)
{
for (int i = 0; i < nodes.Count; i++)
{
if (nodes[i].Checked)
{
if (nodes[i].Nodes.Count > 0)
{
AddDownloadsFromZip(nodes[i].Nodes, mirrors);
}
else
{
ResourceLocation newLocation = this.DownloadLocation;
newLocation.ProtocolProviderType = typeof(ZipProtocolProvider).AssemblyQualifiedName;
string entryName = ((ZipEntry)nodes[i].Tag).Name;
Downloader download = DownloadManager.Instance.Add(
newLocation,
mirrors,
this.folderBrowser1.Folder + entryName,
1,
false);
ZipProtocolProvider.SetZipEntryNameProperty(download, entryName);
if (this.StartNow)
{
download.Start();
}
}
}
}
}
public Downloader Add(ResourceLocation rl, ResourceLocation[] mirrors, string localFile, List<Segment> segments, RemoteFileInfo remoteInfo, int requestedSegmentCount, bool autoStart, DateTime createdDateTime)
{
Downloader d = new Downloader(rl, mirrors, localFile, segments, remoteInfo, requestedSegmentCount, createdDateTime);
Add(d, autoStart);
return d;
}
public override Stream CreateStream(ResourceLocation rl, long initialPosition, long endPosition)
{
return base.CreateStream(mappedUrl ?? rl, initialPosition, endPosition);
}
public ImportFromFilePreviewForm(ResourceLocation[] locations)
{
InitializeComponent();
lslUrls.DataSource = locations;
}
public int AddDownload(string url, string file = null, int segmentNb = 1, bool startNow = false)
{
int id = 0;
Execute(() =>
{
if (file == null)
{
Uri uri = new Uri(url);
file = uri.Segments[uri.Segments.Length - 1];
}
if (!Path.IsPathRooted(file))
file = Path.Combine(DownloadManager.Instance.DefaultDownloadDirectory, file);
ResourceLocation rl = new ResourceLocation();
rl.URL = url;
ResourceLocation[] mirrors = new ResourceLocation[0];
Downloader download = DownloadManager.Instance.Add(rl, mirrors, file, segmentNb, startNow);
Log.Write(string.Format("service : add download = url \"{0}\" file \"{1}\"", download.ResourceLocation.URL, download.LocalFile), LogMode.Information);
id = download.Id;
}
);
return id;
}
public static ResourceLocation FromURL(string url)
{
ResourceLocation rl = new ResourceLocation();
rl.URL = url;
return rl;
}
