/// Execute action
public override object Execute()
{
string fromExpanded = Context.TransformStr(From, Transform);
string toExpanded = Context.TransformStr(To, Transform);
string outToExpanded = Context.TransformStr(OutTo, Transform);
if (string.IsNullOrEmpty(outToExpanded))
{
if (!string.IsNullOrEmpty(toExpanded))
{
toExpanded = UrlToLocalFileName(fromExpanded, toExpanded);
}
}
var enc = Utils.GetEncoding(Context.TransformStr(Encoding, Transform));
Uri uri = new Uri(fromExpanded);
VerboseMessage("Downloading {0} => {1}...", Utils.SecureUri(fromExpanded), toExpanded);
bool passive = PassiveFtp;
bool ftpssl = false;
bool ftp = false;
var scheme = uri.Scheme;
if (scheme == "ftpa" || scheme == "ftps" || scheme == "ftpas" || scheme == "ftpsa")
{
ftp = true;
UriBuilder ub = new UriBuilder(uri);
ub.Scheme = "ftp";
uri = ub.Uri;
passive = !(scheme == "ftpa" || scheme == "ftpas" || scheme == "ftpsa");
ftpssl = (scheme == "ftps" || scheme == "ftpas" || scheme == "ftpsa");
}
var timeout = Utils.ToTimeSpan(Context.TransformStr(Timeout, Transform));
if (uri.IsFile || uri.Scheme == "embed")
{
var fname = (uri.Scheme == "embed") ? uri.ToString() : uri.LocalPath;
VerboseMessage("Local filename '{0}' detected. Copying instead", fname);
try
{
if (Binary && toExpanded != null)
{
if (File.Exists(toExpanded))
{
File.Delete(toExpanded);
}
using (var toStr = Context.CreateStream(toExpanded))
copyFile(fname, toStr, toExpanded, true);
}
else
{
using (var ms = new MemoryStream())
{
copyFile(fname, ms, "memory:///", true);
if (Binary)
{
Context.OutTo(outToExpanded, ms.ToArray());
}
else
{
Context.OutTo(outToExpanded, (enc == null ? new StreamReader(ms) : new StreamReader(ms, enc)).ReadToEnd());
}
}
}
}
catch
{
File.Delete(toExpanded);
throw;
}
return(null);
}
using (DownloadState state = new DownloadState(Context))
{
using (WebClientEx webClient = new WebClientEx(passive, Binary))
{
webClient.KeepAlive = (ftp && !passive);
webClient.FtpSsl = ftpssl;
webClient.CachePolicy = new RequestCachePolicy(CacheLevel);
string user = Context.TransformStr(User, Transform);
string password = Context.TransformStr(Password, Transform);
uri = webClient.SetCredentials(uri, user, password);
if (!string.IsNullOrEmpty(Post))
{
webClient.HttpPost = Context.Transform(Post, Transform);
if (!string.IsNullOrEmpty(PostContentType))
{
webClient.HttpPostContentType = Context.TransformStr(PostContentType, Transform);
}
}
webClient.HttpUserAgent = Context.TransformStr(UserAgent, Transform);
webClient.Timeout = timeout;
int oldPercentage = -1;
long bytesReceived = -1;
// We must ensure that all script components are executed in a single thread
webClient.DownloadProgressChanged += state.ProgressChanged;
webClient.DownloadFileCompleted += state.FileCompleted;
webClient.DownloadStringCompleted += state.StringCompleted;
webClient.DownloadDataCompleted += state.DataCompleted;
if (enc != null)
{
webClient.Encoding = enc;
}
string tmp = null;
if (string.IsNullOrEmpty(outToExpanded))
{
tmp = Direct ? toExpanded : Path.GetTempFileName();
}
var lastUpdate = System.Diagnostics.Stopwatch.StartNew();
WaitHandle[] wh = new WaitHandle[] { state.Completed, state.ProgressAvailable };
try
{
if (tmp == null)
{
if (Binary)
{
webClient.DownloadDataAsync(uri);
}
else
{
webClient.DownloadStringAsync(uri);
}
}
else
{
webClient.DownloadFileAsync(uri, tmp);
}
string pref = Context.TransformStr(Name, Transform);
while (true)
{
int n = WaitHandle.WaitAny(wh, 300, true);
if (n == 0 || n == 1)
{
lastUpdate = System.Diagnostics.Stopwatch.StartNew();
DownloadProgress ps = state.Progress;
if (n == 0)
{
ps = state.Progress;
if (Binary && state.Result != null)
{
ps.BytesReceived = ((byte[])state.Result).LongLength;
}
else if (tmp != null)
{
ps.BytesReceived = new FileInfo(tmp).Length;
}
}
if (ps.BytesReceived > 0 && ps.BytesReceived > bytesReceived)
{
VerboseMessage("Received: {0}", ps);
Context.OnProgress(ps.ProgressPercentage, uri.ToString());
oldPercentage = ps.ProgressPercentage;
if (base.Items.Count != 0)
{
Vars sv = new Vars();
sv.Set("", ps);
Context.ExecuteWithVars(baseExecute, sv, pref);
}
bytesReceived = ps.BytesReceived;
}
}
else
{
// Sometimes FTP hangs, seen with FileZilla 0.9.31 + VMWare a few times
if (timeout.HasValue && lastUpdate.Elapsed > timeout.Value)
{
throw new TimeoutException();
}
}
if (n == 0)
{
break;
}
Context.OnProgress(Math.Max(oldPercentage, 0), uri.ToString());
}
if (state.Error != null)
{
if (state.Error is TargetInvocationException)
{
Utils.Rethrow(state.Error.InnerException);
}
else
{
Utils.Rethrow(state.Error);
}
}
if (tmp != null && toExpanded != tmp)
{
if (File.Exists(toExpanded))
{
File.Delete(toExpanded);
}
using (var toStr = Context.CreateStream(toExpanded))
copyFile(tmp, toStr, toExpanded, false);
VerboseMessage("Copying completed. Deleting '{0}'", tmp);
File.Delete(tmp);
}
}
catch (Exception e)
{
VerboseMessage("Caught exception: {0}", e.Message);
webClient.CancelAsync();
state.SetCompleted();
throw;
}
finally
{
VerboseMessage("Waiting for download completion");
state.Completed.WaitOne(timeout ?? TimeSpan.FromSeconds(30), false);
VerboseMessage("Waiting completed");
webClient.DownloadProgressChanged -= state.ProgressChanged;
webClient.DownloadFileCompleted -= state.FileCompleted;
webClient.DownloadStringCompleted -= state.StringCompleted;
webClient.DownloadDataCompleted -= state.DataCompleted;
try
{
if (webClient.IsBusy)
{
webClient.CancelAsync();
}
}
catch
{
}
if (tmp == null)
{
Context.OutTo(outToExpanded, Binary ? state.Result : state.ResultStr);
}
else if (tmp != toExpanded)
{
try
{
File.Delete(tmp);
}
catch (IOException)
{
Thread.Sleep(500);
File.Delete(tmp);
}
}
}
VerboseMessage("Download completed.");
}
}
return(null);
}
/// Open file in specified mode for writing.
public virtual Stream OpenStream(string fileName, FileMode mode, bool shared)
{
WriteVerbose("OpenStream> Opening '" + fileName + "' for " + mode + (shared?"(shared)":string.Empty));
Uri u;
if (Uri.TryCreate(fileName, UriKind.Absolute, out u) && !u.IsFile)
{
if (mode != FileMode.Open)
{
throw new ArgumentOutOfRangeException("mode", "Only FileMode.Open mode is allowed for URIs");
}
WriteVerbose("OpenStream> Reading from " + Utils.SecureUri(u));
if (u.Scheme == "embed")
{
fileName = u.GetComponents(UriComponents.Path, UriFormat.Unescaped);
Stream str = FindResourceStream(fileName);
if (str == null)
{
throw new FileNotFoundException("Embedded resource file not found", fileName);
}
return(str);
}
bool active = false;
if (u.Scheme == "ftpa")
{
u = new UriBuilder(u)
{
Scheme = "ftp"
}.Uri;
active = true;
}
WriteVerbose("OpenStream> Starting download");
using (var w = new WebClientEx(!active, true))
{
w.CachePolicy = new RequestCachePolicy(RequestCacheLevel.Revalidate);
u = w.SetCredentials(u, null, null);
var ms = new MemoryStream(w.DownloadData(u));
WriteVerbose("OpenStream> Download completed");
return(ms);
}
}
FileShare share = shared ? FileShare.Read : FileShare.None;
if (mode == FileMode.Open)
{
// Uri constructor does not work well with C:\Windows\..\X.txt producing C:\Windows\X.txt
if (u != null && fileName.StartsWith(Uri.UriSchemeFile + ":", StringComparison.OrdinalIgnoreCase))
{
fileName = u.LocalPath;
}
return(new FileStream(fileName, mode, FileAccess.Read, share, 16384));
}
if (mode == FileMode.Append)
{
return(new FileStream(fileName, mode, FileAccess.Write, share, 16384));
}
return(new FileStream(fileName, mode, FileAccess.ReadWrite, share, 16384));
}
/// Execute action
public override object Execute()
{
string fromExpanded = Context.TransformStr(From, Transform);
string toExpanded = Context.TransformStr(To, Transform);
string outToExpanded = Context.TransformStr(OutTo, Transform);
if (string.IsNullOrEmpty(outToExpanded))
{
if (!string.IsNullOrEmpty(toExpanded))
toExpanded = UrlToLocalFileName(fromExpanded, toExpanded);
}
var enc = Utils.GetEncoding(Context.TransformStr(Encoding, Transform));
Uri uri = new Uri(fromExpanded);
VerboseMessage("Downloading {0} => {1}...", Utils.SecureUri(fromExpanded), toExpanded);
bool passive = PassiveFtp;
bool ftpssl = false;
bool ftp = false;
var scheme = uri.Scheme;
if (scheme == "ftpa" || scheme == "ftps" || scheme == "ftpas" || scheme == "ftpsa")
{
ftp = true;
UriBuilder ub = new UriBuilder(uri);
ub.Scheme = "ftp";
uri = ub.Uri;
passive = !(scheme == "ftpa" || scheme == "ftpas" || scheme == "ftpsa");
ftpssl = (scheme == "ftps" || scheme == "ftpas" || scheme == "ftpsa");
}
var timeout = Utils.ToTimeSpan(Context.TransformStr(Timeout, Transform));
if (uri.IsFile || uri.Scheme == "embed")
{
var fname = (uri.Scheme == "embed") ? uri.ToString() : uri.LocalPath;
VerboseMessage("Local filename '{0}' detected. Copying instead", fname);
try
{
if (Binary && toExpanded!=null)
{
if (File.Exists(toExpanded))
File.Delete(toExpanded);
using (var toStr = Context.CreateStream(toExpanded))
copyFile(fname, toStr, toExpanded, true);
}
else
{
using (var ms = new MemoryStream())
{
copyFile(fname, ms, "memory:///", true);
if (Binary)
Context.OutTo(outToExpanded, ms.ToArray());
else
Context.OutTo(outToExpanded, (enc == null ? new StreamReader(ms) : new StreamReader(ms, enc)).ReadToEnd());
}
}
}
catch
{
File.Delete(toExpanded);
throw;
}
return null;
}
using (DownloadState state = new DownloadState(Context))
{
using (WebClientEx webClient = new WebClientEx(passive, Binary))
{
webClient.KeepAlive = (ftp && !passive);
webClient.FtpSsl = ftpssl;
webClient.CachePolicy = new RequestCachePolicy(CacheLevel);
string user = Context.TransformStr(User, Transform);
string password = Context.TransformStr(Password, Transform);
uri = webClient.SetCredentials(uri, user, password);
if (!string.IsNullOrEmpty(Post))
{
webClient.HttpPost = Context.Transform(Post, Transform);
if (!string.IsNullOrEmpty(PostContentType))
webClient.HttpPostContentType = Context.TransformStr(PostContentType, Transform);
}
webClient.HttpUserAgent = Context.TransformStr(UserAgent, Transform);
webClient.Timeout = timeout;
int oldPercentage = -1;
long bytesReceived = -1;
// We must ensure that all script components are executed in a single thread
webClient.DownloadProgressChanged += state.ProgressChanged;
webClient.DownloadFileCompleted += state.FileCompleted;
webClient.DownloadStringCompleted += state.StringCompleted;
webClient.DownloadDataCompleted += state.DataCompleted;
if (enc != null)
webClient.Encoding = enc;
string tmp = null;
if (string.IsNullOrEmpty(outToExpanded))
tmp = Direct ? toExpanded : Path.GetTempFileName();
var lastUpdate = System.Diagnostics.Stopwatch.StartNew();
WaitHandle[] wh = new WaitHandle[] { state.Completed, state.ProgressAvailable };
try
{
if (tmp == null)
{
if (Binary)
webClient.DownloadDataAsync(uri);
else
webClient.DownloadStringAsync(uri);
}
else
webClient.DownloadFileAsync(uri, tmp);
string pref = Context.TransformStr(Name, Transform);
while (true)
{
int n = WaitHandle.WaitAny(wh, 300, true);
if (n == 0 || n == 1)
{
lastUpdate = System.Diagnostics.Stopwatch.StartNew();
DownloadProgress ps = state.Progress;
if (n == 0)
{
ps = state.Progress;
if (Binary && state.Result != null)
ps.BytesReceived = ((byte[])state.Result).LongLength;
else if (tmp != null)
ps.BytesReceived = new FileInfo(tmp).Length;
}
if (ps.BytesReceived > 0 && ps.BytesReceived > bytesReceived)
{
VerboseMessage("Received: {0}", ps);
Context.OnProgress(ps.ProgressPercentage, uri.ToString());
oldPercentage = ps.ProgressPercentage;
if (base.Items.Count != 0)
{
Vars sv = new Vars();
sv.Set("", ps);
Context.ExecuteWithVars(baseExecute, sv, pref);
}
bytesReceived = ps.BytesReceived;
}
}
else
{
// Sometimes FTP hangs, seen with FileZilla 0.9.31 + VMWare a few times
if (timeout.HasValue && lastUpdate.Elapsed > timeout.Value)
throw new TimeoutException();
}
if (n == 0)
{
break;
}
Context.OnProgress(Math.Max(oldPercentage, 0), uri.ToString());
}
if (state.Error != null)
{
if (state.Error is TargetInvocationException)
Utils.Rethrow(state.Error.InnerException);
else
Utils.Rethrow(state.Error);
}
if (tmp != null && toExpanded != tmp)
{
if (File.Exists(toExpanded))
File.Delete(toExpanded);
using (var toStr = Context.CreateStream(toExpanded))
copyFile(tmp, toStr, toExpanded, false);
VerboseMessage("Copying completed. Deleting '{0}'", tmp);
File.Delete(tmp);
}
}
catch (Exception e)
{
VerboseMessage("Caught exception: {0}", e.Message);
webClient.CancelAsync();
state.SetCompleted();
throw;
}
finally
{
VerboseMessage("Waiting for download completion");
state.Completed.WaitOne(timeout ?? TimeSpan.FromSeconds(30), false);
VerboseMessage("Waiting completed");
webClient.DownloadProgressChanged -= state.ProgressChanged;
webClient.DownloadFileCompleted -= state.FileCompleted;
webClient.DownloadStringCompleted -= state.StringCompleted;
webClient.DownloadDataCompleted -= state.DataCompleted;
try
{
if (webClient.IsBusy)
webClient.CancelAsync();
}
catch
{
}
if (tmp == null)
Context.OutTo(outToExpanded, Binary ? state.Result : state.ResultStr);
else if (tmp != toExpanded)
{
try
{
File.Delete(tmp);
}
catch (IOException)
{
Thread.Sleep(500);
File.Delete(tmp);
}
}
}
VerboseMessage("Download completed.");
}
}
return null;
}
/// Open file in specified mode for writing.
public virtual Stream OpenStream(string fileName, FileMode mode, bool shared)
{
WriteVerbose("OpenStream> Opening '"+fileName+"' for "+mode+(shared?"(shared)":string.Empty));
Uri u;
if (Uri.TryCreate(fileName, UriKind.Absolute, out u) && !u.IsFile)
{
if (mode!=FileMode.Open)
throw new ArgumentOutOfRangeException("mode","Only FileMode.Open mode is allowed for URIs");
WriteVerbose("OpenStream> Reading from " + Utils.SecureUri(u));
if (u.Scheme == "embed")
{
fileName = u.GetComponents(UriComponents.Path, UriFormat.Unescaped);
Stream str = FindResourceStream(fileName);
if (str == null)
throw new FileNotFoundException("Embedded resource file not found", fileName);
return str;
}
bool active = false;
if (u.Scheme == "ftpa")
{
u = new UriBuilder(u) { Scheme = "ftp" }.Uri;
active = true;
}
WriteVerbose("OpenStream> Starting download");
using (var w = new WebClientEx(!active, true))
{
w.CachePolicy = new RequestCachePolicy(RequestCacheLevel.Revalidate);
u = w.SetCredentials(u, null, null);
var ms = new MemoryStream(w.DownloadData(u));
WriteVerbose("OpenStream> Download completed");
return ms;
}
}
FileShare share = shared ? FileShare.Read : FileShare.None;
if (mode == FileMode.Open)
{
// Uri constructor does not work well with C:\Windows\..\X.txt producing C:\Windows\X.txt
if (u != null && fileName.StartsWith(Uri.UriSchemeFile + ":", StringComparison.OrdinalIgnoreCase))
fileName = u.LocalPath;
return new FileStream(fileName, mode, FileAccess.Read, share, 16384);
}
if (mode == FileMode.Append)
return new FileStream(fileName, mode, FileAccess.Write, share, 16384);
return new FileStream(fileName, mode, FileAccess.ReadWrite, share, 16384);
}