/// <summary>
/// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class.
/// </summary>
/// <param name="token">The upload token that identifies the transfer.</param>
/// <param name="lastTransmittedBlockNumber">The last transferred block number, if this stream
/// is created for an upload that is already in progress. Can be set along with the <paramref name="startOffset"/> in
/// order to resume an upload. Defaults to null in case of an initial upload.</param>
/// <param name="startOffset">The initial offset of the upload. Can be set along with the
/// <paramref name="lastTransmittedBlockNumber"/> in order to resume an upload. Defaults to
/// zero in case of an initial upload.</param>
/// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush
/// its internal buffer once it reaches the defined size.<br/>
/// If the threshold is 0, every write is immediately flushed.</param>
/// <param name="outputAction">An action that is being invoked with created
/// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param>
/// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction"
/// is a null reference.</exception>
/// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/>
/// is negative.</exception>
public StreamedBlockOutputStream(UploadToken token, int autoFlushThreshold, long startOffset,
long? lastTransmittedBlockNumber, Action<StreamedDataBlock> outputAction)
: base(token, autoFlushThreshold, startOffset, lastTransmittedBlockNumber, b => { })
{
OutputAction = outputAction;
base.OutputAction = ParseBuffer;
}
/// <summary>
/// Loads the dataset into the dataset application.
/// </summary>
/// <param name="ownerId">The ID of the endpoint which requested the load of the dataset.</param>
/// <param name="token">The token that indicates which file should be uploaded.</param>
/// <returns>A task that will finish once the dataset is loaded.</returns>
public Task Load(EndpointId ownerId, UploadToken token)
{
// This one is more tricky than you think. We need to return a Task (not a Task<T>), however
// if we do Task<T>.ContinueWith we get a System.Threading.Tasks.ContinuationTaskFromResultTask<System.IO.Stream>
// object. When we give that back to the command system it tries to push the stream across the network ... not
// really what we want.
//
// So to fix this we create a task and then inside that task we load the file (via another task) and continue
// of the nested task. The continuation task is a child of the global task and all is well because the
// global task doesn't finish until the nested child task is done, and we get a Task object back
var globalTask = Task.Factory.StartNew(
() =>
{
var filePath = Path.GetTempFileName();
var task = m_DataDownload(ownerId, token, filePath);
task.ContinueWith(
t =>
{
using (m_Diagnostics.Profiler.Measure(BaseConstants.TimingGroup, "Loading dataset from file."))
{
m_LoadAction(new FileInfo(filePath));
}
},
TaskContinuationOptions.AttachedToParent | TaskContinuationOptions.ExecuteSynchronously);
});
return(globalTask);
}
/// <summary>
/// Creates a token for the currently processed resource.
/// </summary>
/// <param name="submittedResourceFilePath">The identifier (file path)
/// for the requested resource, as submitted by the requesting party.</param>
/// <param name="fileItem">Represents the file resource to be uploaded.</param>
/// <param name="resourceLength">The length of the resource to be uploaded in bytes.</param>
/// <param name="contentType">The content type of the uploaded resource.</param>
/// <returns>A token for the request.</returns>
protected override UploadToken CreateUploadToken(string submittedResourceFilePath, ZipFileItem fileItem, long resourceLength, string contentType)
{
var fileInfo = fileItem.ResourceInfo;
string transferId = Guid.NewGuid().ToString();
var token = new UploadToken
{
TransferId = transferId,
ResourceIdentifier = submittedResourceFilePath,
CreationTime = SystemTime.Now(),
ContentType = fileInfo.ContentType,
MaxResourceSize = resourceLength,
MaxBlockSize = MaxBlockSize,
ResourceName = fileInfo.Name,
ResourceLength = resourceLength
};
TimeSpan?expiration = Config.FileSystemConfiguration.UploadTokenExpirationTime;
if (expiration.HasValue)
{
token.ExpirationTime = SystemTime.Now().Add(expiration.Value);
}
return(token);
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class.
/// </summary>
/// <param name="token">The upload token that identifies the transfer.</param>
/// <param name="lastTransmittedBlockNumber">The last transferred block number, if this stream
/// is created for an upload that is already in progress. Can be set along with the <paramref name="startOffset"/> in
/// order to resume an upload. Defaults to null in case of an initial upload.</param>
/// <param name="startOffset">The initial offset of the upload. Can be set along with the
/// <paramref name="lastTransmittedBlockNumber"/> in order to resume an upload. Defaults to
/// zero in case of an initial upload.</param>
/// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush
/// its internal buffer once it reaches the defined size.<br/>
/// If the threshold is 0, every write is immediately flushed.</param>
/// <param name="outputAction">An action that is being invoked with created
/// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param>
/// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction"
/// is a null reference.</exception>
/// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/>
/// is negative.</exception>
public StreamedBlockOutputStream(UploadToken token, int autoFlushThreshold, long startOffset,
long?lastTransmittedBlockNumber, Action <StreamedDataBlock> outputAction)
: base(token, autoFlushThreshold, startOffset, lastTransmittedBlockNumber, b => { })
{
OutputAction = outputAction;
base.OutputAction = ParseBuffer;
}
/// <summary>
/// 获取令牌
/// </summary>
public void tk()
{
UploadToken uploadToken = new UploadToken();
string name = _request.QueryString["name"];
string size = _request.QueryString["size"];
string ext = name.Substring(name.LastIndexOf('.'));
string token = SimpleEncryptor.MD5(name + size);
uploadToken.name = name;
uploadToken.size = string.IsNullOrEmpty(size) ? 0 : int.Parse(size);
uploadToken.token = token;
if (!File.Exists(_server.MapPath(_tokenPath + token + ".token")))
{
string modified = _request.QueryString["modified"];
uploadToken.filePath = "";
uploadToken.modified = modified;
SetTokenInfo(token, uploadToken);
}
TokenResult tokenResult = new TokenResult();
tokenResult.message = "";
tokenResult.token = token;
tokenResult.success = true;
string result = JSONHelper.SerializeObject(tokenResult);;
_response.Write(result);
}
private string UploadChunck(UploadToken token, string fileName, long fileSize, int offset, int bytesToUpload, Action <long> cb)
{
HttpWebRequest r = (HttpWebRequest)WebRequest.Create(token.upload_link_secure);
r.Method = "PUT";
r.ContentLength = fileSize;
r.ContentType = "video/x-ms-wmv";
r.KeepAlive = true;
r.Timeout = System.Threading.Timeout.Infinite;
r.ReadWriteTimeout = System.Threading.Timeout.Infinite;
r.Headers[HttpRequestHeader.Authorization] = authorization;
if (offset > 0)
{
r.Headers[HttpRequestHeader.ContentRange] = String.Format("bytes {0}-{1}/{2}", offset, fileSize, fileSize);
}
var file = File.OpenRead(fileName);
file.Seek(offset, SeekOrigin.Begin);
const int BUFSIZE = 1048576;
byte[] buffer = new byte[BUFSIZE];
string s = "";
try
{
using (Stream str = r.GetRequestStream())
{
long bytesRead = 0;
while (bytesRead < fileSize)
{
int n = file.Read(buffer, 0, BUFSIZE);
if (n == 0)
{
break;
}
str.Write(buffer, 0, n);
bytesRead += n;
cb(bytesRead);
}
}
file.Close();
file.Dispose();
file = null;
}
catch (WebException e)
{
s = e.Message + " - " + e.Status.ToString();
}
catch (IOException e)
{
s = e.Message;
}
return(s);
}
/// <summary>
/// Initializes a new instance of the <see cref="DataDownloadRequestMessage"/> class.
/// </summary>
/// <param name="origin">The ID of the endpoint that send the message.</param>
/// <param name="id">The ID of the current message.</param>
/// <param name="token">The token that indicates which file should be uploaded.</param>
/// <exception cref="ArgumentNullException">
/// Thrown if <paramref name="origin"/> is <see langword="null" />.
/// </exception>
/// <exception cref="ArgumentNullException">
/// Thrown if <paramref name="id"/> is <see langword="null" />.
/// </exception>
/// <exception cref="ArgumentNullException">
/// Thrown if <paramref name="token"/> is <see langword="null" />.
/// </exception>
public DataDownloadRequestMessage(EndpointId origin, MessageId id, UploadToken token)
: base(origin, id)
{
{
Lokad.Enforce.Argument(() => token);
}
Token = token;
}
private void OnUploadComplete(UploadToken response, Exception error)
{
if (error != null)
{
throw error;
}
_FileStream.Close();
openTasks--;
}
public void Create()
{
var sender = new EndpointId("sendingEndpoint");
var token = new UploadToken();
var msg = new DataDownloadRequestMessage(sender, token);
Assert.IsNotNull(msg.Id);
Assert.AreSame(sender, msg.Sender);
Assert.AreSame(token, msg.Token);
}
public void InvokeWithoutFileRegistration()
{
var uploads = new WaitingUploads();
var layer = new Mock <IProtocolLayer>();
{
layer.Setup(l => l.Id)
.Returns(new EndpointId("other"));
layer.Setup(l => l.SendMessageTo(It.IsAny <EndpointId>(), It.IsAny <ICommunicationMessage>(), It.IsAny <int>()))
.Callback <EndpointId, ICommunicationMessage, int>(
(e, m, r) =>
{
Assert.IsInstanceOf <FailureMessage>(m);
})
.Verifiable();
layer.Setup(l => l.UploadData(
It.IsAny <EndpointId>(),
It.IsAny <string>(),
It.IsAny <CancellationToken>(),
It.IsAny <TaskScheduler>(),
It.IsAny <int>()))
.Returns(Task.Factory.StartNew(
() => { },
new CancellationToken(),
TaskCreationOptions.None,
new CurrentThreadTaskScheduler()))
.Verifiable();
}
var systemDiagnostics = new SystemDiagnostics((p, s) => { }, null);
var action = new DataDownloadProcessAction(
uploads,
layer.Object,
systemDiagnostics,
new CurrentThreadTaskScheduler());
var token = new UploadToken();
var msg = new DataDownloadRequestMessage(
EndpointIdExtensions.CreateEndpointIdForCurrentProcess(),
token);
action.Invoke(msg);
layer.Verify(l => l.SendMessageTo(It.IsAny <EndpointId>(), It.IsAny <ICommunicationMessage>(), It.IsAny <int>()), Times.Once());
layer.Verify(
l => l.UploadData(
It.IsAny <EndpointId>(),
It.IsAny <string>(),
It.IsAny <CancellationToken>(),
It.IsAny <TaskScheduler>(),
It.IsAny <int>()),
Times.Never());
}
public void Init()
{
receivedBlocks = new List <BufferedDataBlock>();
source = new byte[100000];
target = new List <byte>();
new Random(DateTime.Now.Millisecond).NextBytes(source);
token = new UploadToken {
TransferId = "MyToken", MaxBlockSize = 3000
};
}
public void CreateWithId()
{
var sender = new EndpointId("sendingEndpoint");
var id = new MessageId();
var token = new UploadToken();
var msg = new DataDownloadRequestMessage(sender, id, token);
Assert.AreSame(id, msg.Id);
Assert.AreSame(sender, msg.Sender);
Assert.AreSame(token, msg.Token);
}
public void Init()
{
receivedBlocks = new List <BufferedDataBlock>();
source = new byte[100000];
target = new List <byte>();
new Random(DateTime.Now.Millisecond).NextBytes(source);
//token indicates 10 blocks were already submitted
token = new UploadToken {
TransferId = "MyToken", MaxBlockSize = 3000, TransmittedBlockCount = 10
};
}
private string CompleteUpload(UploadToken token)
{
HttpWebRequest r = (HttpWebRequest)WebRequest.Create("https://api.vimeo.com" + token.complete_uri);
r.Method = "DELETE";
r.Headers[HttpRequestHeader.Authorization] = authorization;
HttpWebResponse response = (HttpWebResponse)r.GetResponse();
var s = response.Headers["Location"];
response.Close();
return(s.Replace("/videos/", ""));
}
private void OnUploadTokenAddComplete(UploadToken token, Exception error)
{
openTasks++;
if (error != null)
{
throw error;
}
UploadTokenService.Upload(token.Id, _FileStream)
.SetCompletion(new OnCompletedHandler <UploadToken>(OnUploadComplete))
.Execute(client);
openTasks--;
}
private UploadToken GetTokenInfo(string token)
{
string tokenPath = _tokenPath + token + ".token";
if (File.Exists(_server.MapPath(tokenPath)))
{
_fileHelper.FileName = token + ".token";
_fileHelper.FilePath = _server.MapPath(_tokenPath);
UploadToken uploadToken = JSONHelper.DeserializeJsonToObject <UploadToken>(_fileHelper.ReadFile());
return(uploadToken);
}
return(null);
}
public void upload()
{
string token = _request.QueryString["token"];
UploadToken uploadToken = GetTokenInfo(token);
if (uploadToken != null && uploadToken.size > uploadToken.upsize)
{
Stream stream = _request.InputStream;
if (stream != null && stream.Length > 0)
{
_fileHelper.FileName = uploadToken.name;
_fileHelper.FilePath = _server.MapPath(_filePath);
_fileHelper.WriteFile(stream);
uploadToken.upsize += stream.Length;
if (uploadToken.size > uploadToken.upsize)
{
SetTokenInfo(token, uploadToken);
}
else
{
//上传完成后删除令牌信息
DelTokenInfo(token);
//重命名（解决重复上传时内容叠加）
//FileInfo fileinfo = new FileInfo(_server.MapPath(_filePath) + uploadToken.name);
//string Rename = string.Format(@"{0}_{1}{2}", Path.GetFileNameWithoutExtension(fileinfo.FullName), DateTime.Now.ToString("mmssffff"), fileinfo.Extension);
//fileinfo.MoveTo(_server.MapPath(_filePath) + Rename);
//uploadToken.name = Rename;
//无需重命名
uploadToken.name = uploadToken.name;
}
}
}
UploadResult ur = new UploadResult();
ur.message = "";
ur.filePath = HttpUtility.UrlEncode(_filePath + uploadToken.name, Encoding.UTF8);
ur.start = uploadToken.upsize;
ur.success = true;
string result = JSONHelper.SerializeObject(ur);
_response.Write(result);
}
public void LocatePluginAssemblyDownloadFail()
{
var token = new UploadToken();
var commands = new Mock <IHostApplicationCommands>();
{
commands.Setup(c => c.PreparePluginContainerForTransfer(It.IsAny <AssemblyName>()))
.Returns(
Task <UploadToken> .Factory.StartNew(
() =>
{
return(token);
}));
}
var commandContainer = new Mock <ISendCommandsToRemoteEndpoints>();
{
commandContainer.Setup(c => c.CommandsFor <IHostApplicationCommands>(It.IsAny <EndpointId>()))
.Returns(commands.Object);
}
DownloadDataFromRemoteEndpoints layer = (id, u, file) => Task <FileInfo> .Factory.StartNew(
() =>
{
Assert.AreSame(token, u);
throw new FileNotFoundException();
});
var fileSystem = new Mock <IFileSystem>();
{
fileSystem.Setup(f => f.Path)
.Returns(new MockPath());
fileSystem.Setup(f => f.File)
.Returns(new MockFile(new Dictionary <string, string>()));
}
var endpoint = new EndpointId("a");
var resolver = new PluginLoadingAssemblyResolver(
commandContainer.Object,
layer,
fileSystem.Object,
endpoint);
Assert.IsNull(resolver.LocatePluginAssembly(new object(), new ResolveEventArgs("a")));
}
public void LocatePluginAssemblyDownloadFail()
{
var token = new UploadToken();
var commands = new Mock<IHostApplicationCommands>();
{
commands.Setup(c => c.PreparePluginContainerForTransfer(It.IsAny<AssemblyName>()))
.Returns(
Task<UploadToken>.Factory.StartNew(
() =>
{
return token;
}));
}
var commandContainer = new Mock<ISendCommandsToRemoteEndpoints>();
{
commandContainer.Setup(c => c.CommandsFor<IHostApplicationCommands>(It.IsAny<EndpointId>()))
.Returns(commands.Object);
}
DownloadDataFromRemoteEndpoints layer = (id, u, file) => Task<FileInfo>.Factory.StartNew(
() =>
{
Assert.AreSame(token, u);
throw new FileNotFoundException();
});
var fileSystem = new Mock<IFileSystem>();
{
fileSystem.Setup(f => f.Path)
.Returns(new MockPath());
fileSystem.Setup(f => f.File)
.Returns(new MockFile(new Dictionary<string, string>()));
}
var endpoint = new EndpointId("a");
var resolver = new PluginLoadingAssemblyResolver(
commandContainer.Object,
layer,
fileSystem.Object,
endpoint);
Assert.IsNull(resolver.LocatePluginAssembly(new object(), new ResolveEventArgs("a")));
}
private UploadStatus VerifyUpload(UploadToken token)
{
HttpWebRequest r = (HttpWebRequest)WebRequest.Create(token.upload_link_secure);
r.Method = "PUT";
r.ContentLength = 0;
r.Headers[HttpRequestHeader.ContentRange] = "bytes */*";
r.Headers[HttpRequestHeader.Authorization] = authorization;
try
{
HttpWebResponse response = (HttpWebResponse)r.GetResponse();
}
catch (WebException e)
{
var s = e.Response.Headers["Range"];
return(new UploadStatus(s));
}
throw new Exception();
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class.
/// </summary>
/// <param name="token">The upload token that identifies the transfer.</param>
/// <param name="lastTransmittedBlockNumber">The last transferred block number, if this stream
/// is created for an upload that is already in progress. Can be set along with the <paramref name="startOffset"/> in
/// order to resume an upload. Defaults to null in case of an initial upload.</param>
/// <param name="startOffset">The initial offset of the upload. Can be set along with the
/// <paramref name="lastTransmittedBlockNumber"/> in order to resume an upload. Defaults to
/// zero in case of an initial upload.</param>
/// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush
/// its internal buffer once it reaches the defined size.<br/>
/// If the threshold is 0, every write is immediately flushed.</param>
/// <param name="outputAction">An action that is being invoked with created
/// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param>
/// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction"
/// is a null reference.</exception>
/// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/>
/// is negative.</exception>
public BufferedBlockOutputStream(UploadToken token, int autoFlushThreshold, long startOffset, long?lastTransmittedBlockNumber, Action <BufferedDataBlock> outputAction)
{
if (token == null)
{
throw new ArgumentNullException("token");
}
if (outputAction == null)
{
throw new ArgumentNullException("outputAction");
}
if (autoFlushThreshold < 0)
{
throw new ArgumentOutOfRangeException("autoFlushThreshold",
"Threshold for automatic flushing cannot be negative");
}
Token = token;
AutoFlushThreshold = autoFlushThreshold;
OutputAction = outputAction;
InitialOffset = startOffset;
LastTransmittedBlockNumber = lastTransmittedBlockNumber;
}
private void WriteFileImpl(string virtualFilePath, Stream input, bool overwrite, long resourceLength, string contentType)
{
//get a token - this creates a new transfer and validates the request
UploadToken token = RequestUploadToken(virtualFilePath, overwrite, resourceLength, contentType);
if (resourceLength == 0)
{
//create an empty data block and submit it
var block = new BufferedDataBlock
{
TransferTokenId = token.TransferId,
BlockLength = 0,
BlockNumber = 0,
Data = new byte[0],
IsLastBlock = true
};
WriteBlock(block);
}
else
{
//we don't expose the underlying stream directly, but rather use blocks again,
//which disconnects the exposed stream from the underlying resource and allows for simple
//aborting
try
{
input.WriteTo(token, resourceLength, token.MaxBlockSize ?? 32768, WriteBlockStreamed);
}
catch (Exception)
{
CancelTransfer(token.TransferId, AbortReason.Undefined);
throw;
}
//we don't need to explicitly complete the transfer - the extension method
//implicitly closed the transfer by marking the last block:
//CompleteTransfer(token.TransferId);
}
}
public void upload()
{
string token = _request.QueryString["token"];
UploadToken uploadToken = GetTokenInfo(token);
if (uploadToken != null && uploadToken.size > uploadToken.upsize)
{
Stream stream = _request.InputStream;
if (stream != null && stream.Length > 0)
{
_fileHelper.FileName = uploadToken.name;
_fileHelper.FilePath = _server.MapPath(_filePath);
_fileHelper.WriteFile(stream);
uploadToken.upsize += stream.Length;
if (uploadToken.size > uploadToken.upsize)
{
SetTokenInfo(token, uploadToken);
}
else
{
//上传完成后删除令牌信息
DelTokenInfo(token);
}
}
}
UploadResult ur = new UploadResult();
ur.message = "";
ur.start = uploadToken.upsize;
ur.success = true;
string result = JsonHelper.SerializeObject(ur);
_response.Write(result);
}
private bool DispatchUploadRequest(HttpApplication application, HttpRequest request)
{
// if the request is Upload (POST) request, try to authenticate user
HttpContext context = HttpContext.Current;
string receivedUploadToken = request.Form["UploadToken"];
if (receivedUploadToken != null)
{
UploadToken uploadToken = UploadToken.GetUploadToken(new Guid(receivedUploadToken));
if (uploadToken == null)
{
throw new InvalidOperationException(string.Format("An UploadToken ({0}) has been sent, but a matching user cannot be found.", receivedUploadToken));
}
UserAccessProvider.ChangeToSystemAccount();
User uploadUser = (User)SenseNet.ContentRepository.Storage.Node.LoadNode(uploadToken.UserId);
UserAccessProvider.RestoreOriginalUser();
context.User = new PortalPrincipal(uploadUser);
return(true);
}
return(false);
}
/// <summary>
/// Prepares the report files for upload.
/// </summary>
/// <param name="testId">The ID of the test to which the files belong.</param>
/// <param name="testStepIndex">The index of the test step for which the report files are being uploaded.</param>
/// <param name="callingEndpoint">The ID of the endpoint that called the method.</param>
/// <param name="token">The upload token used to register the report files that need to be uploaded.</param>
/// <returns>An upload token that can be used to download the desired files.</returns>
public Task PrepareReportFilesForTransfer(int testId, int testStepIndex, EndpointId callingEndpoint, UploadToken token)
{
m_Diagnostics.Log(
LevelToLog.Trace,
MasterServiceConstants.LogPrefix,
string.Format(
CultureInfo.InvariantCulture,
Resources.Log_Messages_TransferingTestStepReportFiles_WithTestAndTestStep,
testId,
testStepIndex));
var downloadDirectory = TestHelpers.StoragePathForReportFiles(testId, m_Configuration, m_FileSystem);
if (!m_FileSystem.Directory.Exists(downloadDirectory))
{
m_FileSystem.Directory.CreateDirectory(downloadDirectory);
}
var fileName = string.Format(
CultureInfo.InvariantCulture,
"{0}_{1}.reportfiles",
testId,
testStepIndex);
var filePath = m_FileSystem.Path.Combine(
downloadDirectory,
string.Format(
CultureInfo.InvariantCulture,
"{0}.zip",
fileName));
var fileStream = m_DataDownload(
callingEndpoint,
token,
filePath);
var uploadTask = fileStream.ContinueWith(
file =>
{
try
{
var testPackage = new TestStepPackage(testStepIndex);
var unpackDirectory = Path.Combine(downloadDirectory, fileName);
testPackage.LoadAndUnpack(file.Result.FullName, unpackDirectory);
var notifications = m_ActiveTests.NotificationsFor(testId);
foreach (var notification in notifications)
{
foreach (var storedFile in testPackage.StoredFiles())
{
var relativePath = storedFile.FullName.Substring(downloadDirectory.Length).TrimStart(Path.DirectorySeparatorChar);
notification.StoreReportFile(storedFile.FullName, relativePath);
}
}
}
catch (Exception e)
{
m_Diagnostics.Log(
LevelToLog.Error,
MasterServiceConstants.LogPrefix,
string.Format(
CultureInfo.InvariantCulture,
Resources.Log_Messages_TransferingTestStepReportFilesFailed_WithTestAndTestStepAndError,
testId,
testStepIndex,
e));
}
});
return uploadTask;
}
/// <summary>
/// Creates a transfer object for a given file resource.
/// </summary>
/// <param name="submittedResourceFilePath">The resource identifier as submitted.</param>
/// <param name="fileItem">Represents the file resource to be uploaded.</param>
/// <param name="parentFolder">The file's parent folder.</param>
/// <param name="token">The token that is being issued for the transfer.</param>
/// <param name="claims">The access rights for the resource.</param>
/// <param name="lockGuard">File locks, if necessary. Can be a null reference
/// if no locking takes place.</param>
/// <param name="expirationJob">A scheduled job that invokes the
/// <see cref="TransferHandlerBase{TFile,TToken,TTransfer}.OnTransferExpiration"/>
/// method once the transfer expires. May be null if the token does not expire.</param>
/// <returns>A transfer object which encapsulates the information required to perform
/// the transfer.</returns>
protected override ZipUploadTransfer CreateTransfer(string submittedResourceFilePath, ZipFileItem fileItem, ZipFolderItem parentFolder, UploadToken token, FileClaims claims, ResourceLockGuard lockGuard, Job <UploadToken> expirationJob)
{
var transfer = new ZipUploadTransfer(token, fileItem)
{
ParentFolder = parentFolder,
Status = TransferStatus.Starting,
Owner = Config.Provider.Security.GetIdentity()
};
if (expirationJob != null)
{
transfer.ExpirationNotificationJob = expirationJob;
}
if (lockGuard != null)
{
transfer.ResourceLock = lockGuard;
}
return(transfer);
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class that initiates
/// a new upload (starting with block 0 at offset 0).
/// </summary>
/// <param name="token">The upload token that identifies the transfer.</param>
/// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush
/// its internal buffer once it reaches the defined size.<br/>
/// If the threshold is 0, every write is immediately flushed.</param>
/// <param name="outputAction">An action that is being invoked with created
/// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param>
/// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction"
/// is a null reference.</exception>
/// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/>
/// is negative.</exception>
public StreamedBlockOutputStream(UploadToken token, int autoFlushThreshold, Action<StreamedDataBlock> outputAction)
: base(token, autoFlushThreshold, b => { })
{
OutputAction = outputAction;
base.OutputAction = ParseBuffer;
}
/// <summary>
/// Starts a download.
/// </summary>
/// <param name="downloadOwningEndpoint">The endpoint ID of the endpoint that owns the data stream.</param>
/// <param name="token">The upload token that allows the receiver to indicate which data stream should be downloaded.</param>
public void StartDownload([InvokingEndpoint] EndpointId downloadOwningEndpoint, UploadToken token)
{
var path = Path.Combine(Assembly.GetExecutingAssembly().LocalDirectoryPath(), Path.GetRandomFileName());
var task = m_Download(downloadOwningEndpoint, token, path, TimeSpan.FromSeconds(15));
string text;
try
{
task.Wait();
text = new StreamReader(task.Result.FullName).ReadToEnd();
}
catch (AggregateException)
{
text = "Failed to download data.";
}
m_OnEcho(
downloadOwningEndpoint,
string.Format(
CultureInfo.InvariantCulture,
"Downloaded data from: {0}. Data: {1}",
downloadOwningEndpoint,
text));
}
/// <summary>
/// Starts the execution of the given test steps.
/// </summary>
/// <param name="testId">The ID of the test that is being executed.</param>
/// <param name="testSteps">The collection of test steps that should be executed.</param>
/// <param name="environmentParameters">The collection that contains the parameters related to the test environment.</param>
/// <param name="callingEndpoint">The ID of the endpoint that called the method.</param>
/// <param name="token">The upload token that can be used to upload the file.</param>
/// <returns>A task which completes when the execution has started successfully.</returns>
public Task Execute(
int testId,
List<TestStep> testSteps,
List<InputParameter> environmentParameters,
EndpointId callingEndpoint,
UploadToken token)
{
return Task.Factory.StartNew(() => StartNewTestExecution(testId, testSteps, environmentParameters, callingEndpoint, token));
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class.
/// </summary>
/// <param name="token">The upload token that identifies the transfer.</param>
/// <param name="lastTransmittedBlockNumber">The last transferred block number, if this stream
/// is created for an upload that is already in progress. Can be set along with the <paramref name="startOffset"/> in
/// order to resume an upload. Defaults to null in case of an initial upload.</param>
/// <param name="startOffset">The initial offset of the upload. Can be set along with the
/// <paramref name="lastTransmittedBlockNumber"/> in order to resume an upload. Defaults to
/// zero in case of an initial upload.</param>
/// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush
/// its internal buffer once it reaches the defined size.<br/>
/// If the threshold is 0, every write is immediately flushed.</param>
/// <param name="outputAction">An action that is being invoked with created
/// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param>
/// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction"
/// is a null reference.</exception>
/// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/>
/// is negative.</exception>
public BufferedBlockOutputStream(UploadToken token, int autoFlushThreshold, long startOffset, long? lastTransmittedBlockNumber, Action<BufferedDataBlock> outputAction)
{
if (token == null) throw new ArgumentNullException("token");
if (outputAction == null) throw new ArgumentNullException("outputAction");
if(autoFlushThreshold < 0)
{
throw new ArgumentOutOfRangeException("autoFlushThreshold",
"Threshold for automatic flushing cannot be negative");
}
Token = token;
AutoFlushThreshold = autoFlushThreshold;
OutputAction = outputAction;
InitialOffset = startOffset;
LastTransmittedBlockNumber = lastTransmittedBlockNumber;
}
/// <summary>
/// Loads the dataset into the dataset application.
/// </summary>
/// <param name="ownerId">The ID of the endpoint which requested the load of the dataset.</param>
/// <param name="token">The token that indicates which file should be uploaded.</param>
/// <returns>A task that will finish once the dataset is loaded.</returns>
public Task Load(EndpointId ownerId, UploadToken token)
{
// This one is more tricky than you think. We need to return a Task (not a Task<T>), however
// if we do Task<T>.ContinueWith we get a System.Threading.Tasks.ContinuationTaskFromResultTask<System.IO.Stream>
// object. When we give that back to the command system it tries to push the stream across the network ... not
// really what we want.
//
// So to fix this we create a task and then inside that task we load the file (via another task) and continue
// of the nested task. The continuation task is a child of the global task and all is well because the
// global task doesn't finish until the nested child task is done, and we get a Task object back
var globalTask = Task.Factory.StartNew(
() =>
{
var filePath = Path.GetTempFileName();
var task = m_DataDownload(ownerId, token, filePath);
task.ContinueWith(
t =>
{
using (m_Diagnostics.Profiler.Measure(BaseConstants.TimingGroup, "Loading dataset from file."))
{
m_LoadAction(new FileInfo(filePath));
}
},
TaskContinuationOptions.AttachedToParent | TaskContinuationOptions.ExecuteSynchronously);
});
return globalTask;
}
private bool DownloadTestData(
EndpointId callingEndpoint,
UploadToken token,
string storageDirectory,
ITestSectionBuilder builder,
out string testFile)
{
testFile = string.Empty;
try
{
testFile = DownloadTestData(callingEndpoint, token, storageDirectory);
m_Diagnostics.Log(
LevelToLog.Debug,
ExecutorServiceConstants.LogPrefix,
string.Format(
CultureInfo.InvariantCulture,
Resources.Log_Messages_DownloadedTestData_WithFile,
testFile));
builder.AddInformationMessage(Resources.ReportSection_Info_DownloadedTestData);
}
catch (Exception)
{
builder.AddErrorMessage(Resources.ReportSection_Error_FailedToDownloadTestData);
builder.FinalizeAndStore(false);
m_TestExecutionEvents.RaiseOnTestCompletion(TestExecutionResult.Failed);
return false;
}
return true;
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.Object"/> class.
/// </summary>
public LocalUploadTransfer(UploadToken token, FileItem fileItem)
: base(token, fileItem)
{
}
private string DownloadTestData(EndpointId callingEndpoint, UploadToken token, string storageDirectory)
{
var filePath = m_FileSystem.Path.Combine(
storageDirectory,
string.Format(
CultureInfo.InvariantCulture,
"{0}.zip",
Guid.NewGuid().ToString("D")));
var fileStream = m_DataDownload(callingEndpoint, token, filePath);
fileStream.Wait();
return fileStream.Result.FullName;
}
private void StartNewTestExecution(
int testId,
List<TestStep> testSteps,
List<InputParameter> environmentParameters,
EndpointId callingEndpoint,
UploadToken token)
{
var builder = m_SectionBuilders(Resources.ReportSection_Group_Name_Initialization);
builder.Initialize(Resources.ReportSection_Name_Initialization);
m_HostInformation.Id = callingEndpoint;
string testFile;
if (!DownloadTestData(callingEndpoint, token, m_StorageDirectory, builder, out testFile))
{
return;
}
m_TestInformation.TestId = testId;
m_TestInformation.TestSteps = testSteps;
m_TestInformation.TestPackage = testFile;
m_TestInformation.EnvironmentParameters = environmentParameters;
try
{
var pair = m_Layer.LocalConnectionFor(ChannelType.NamedPipe);
m_CurrentEndpoint = StartExecutorApplication(pair.Item1, pair.Item2);
m_TestInformation.CurrentState = TestExecutionState.Running;
m_Diagnostics.Log(
LevelToLog.Debug,
ExecutorServiceConstants.LogPrefix,
Resources.Log_Messages_StartedTestApplication);
}
catch (Exception e)
{
m_TestInformation.CurrentState = TestExecutionState.None;
m_Diagnostics.Log(
LevelToLog.Error,
ExecutorServiceConstants.LogPrefix,
string.Format(
CultureInfo.InvariantCulture,
Resources.Log_Messages_FailedToStartTestApplication_WithError,
e));
builder.AddErrorMessage(Resources.ReportSection_Error_FailedToStartTestingApplication);
builder.FinalizeAndStore(false);
m_TestExecutionEvents.RaiseOnTestCompletion(TestExecutionResult.Failed);
return;
}
builder.AddInformationMessage(
string.Format(
CultureInfo.InvariantCulture,
Resources.ReportSection_Info_StartedTestApplication_WithId,
m_CurrentEndpoint));
try
{
WaitForExecutorConnection(m_CurrentEndpoint);
}
catch (TestExecutionFailureException)
{
m_TestInformation.CurrentState = TestExecutionState.None;
m_Diagnostics.Log(
LevelToLog.Error,
ExecutorServiceConstants.LogPrefix,
Resources.Log_Messages_FailedToConnectToTestApplication_WithError);
builder.AddErrorMessage(Resources.ReportSection_Error_FailedToConnectToTestingApplication);
builder.FinalizeAndStore(false);
m_TestExecutionEvents.RaiseOnTestCompletion(TestExecutionResult.Failed);
return;
}
ConnectEvents(m_CurrentEndpoint);
builder.FinalizeAndStore(true);
}
/// <summary>
/// Prepares the report files for upload.
/// </summary>
/// <param name="testStepIndex">The index of the test step for which the report files are being uploaded.</param>
/// <param name="callingEndpoint">The ID of the endpoint that called the method.</param>
/// <param name="token">The upload token used to register the report files that need to be uploaded.</param>
/// <returns>An upload token that can be used to download the desired files.</returns>
public Task PrepareReportFilesForTransfer(int testStepIndex, EndpointId callingEndpoint, UploadToken token)
{
m_Diagnostics.Log(
LevelToLog.Trace,
ExecutorServiceConstants.LogPrefix,
string.Format(
CultureInfo.InvariantCulture,
Resources.Log_Messages_TransferingTestStepReportFiles_WithTestStep,
testStepIndex));
var filePath = m_FileSystem.Path.Combine(
m_StorageDirectory,
string.Format(
CultureInfo.InvariantCulture,
"{0}.zip",
Guid.NewGuid().ToString("D")));
var fileStream = m_DataDownload(callingEndpoint, token, filePath);
var uploadTask = fileStream.ContinueWith(
file =>
{
try
{
// Upload to the host here
var hostToken = m_Uploads.Register(file.Result.FullName);
if (!m_RemoteCommands.HasCommandFor(m_HostInformation.Id, typeof(IStoreTestReportDataCommands)))
{
throw new MissingCommandSetException();
}
var id = EndpointIdExtensions.CreateEndpointIdForCurrentProcess();
var command = m_RemoteCommands.CommandsFor<IStoreTestReportDataCommands>(m_HostInformation.Id);
var task = CommandSetGuard.GuardAgainstCommunicationFailure(
command.PrepareReportFilesForTransfer,
m_TestInformation.TestId,
testStepIndex,
id,
hostToken);
task.Wait();
}
catch (Exception e)
{
m_Diagnostics.Log(
LevelToLog.Error,
ExecutorServiceConstants.LogPrefix,
string.Format(
CultureInfo.InvariantCulture,
Resources.Log_Messages_TransferingTestStepReportFilesFailed_WithException,
e));
throw;
}
});
return uploadTask;
}
/// <summary>
/// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class that initiates
/// a new upload (starting with block 0 at offset 0).
/// </summary>
/// <param name="token">The upload token that identifies the transfer.</param>
/// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush
/// its internal buffer once it reaches the defined size.<br/>
/// If the threshold is 0, every write is immediately flushed.</param>
/// <param name="outputAction">An action that is being invoked with created
/// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param>
/// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction"
/// is a null reference.</exception>
/// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/>
/// is negative.</exception>
public BufferedBlockOutputStream(UploadToken token, int autoFlushThreshold, Action<BufferedDataBlock> outputAction)
: this(token, autoFlushThreshold, 0, null, outputAction)
{
}
public static UploadTokenAddRequestBuilder Add(UploadToken uploadToken = null)
{
return(new UploadTokenAddRequestBuilder(uploadToken));
}
private TTransfer InitTransferImpl(string submittedResourceFilePath, bool overwrite, long resourceLength, string contentType)
{
const FileSystemTask context = FileSystemTask.UploadTokenRequest;
//validate maximum file size
var maxFileSize = GetMaxFileUploadSize();
if (maxFileSize.HasValue && maxFileSize < resourceLength)
{
string msg = "Upload for file [{0}] denied: Resource length of [{1}] is above the maximum upload limit of [{2}] bytes.";
msg = String.Format(msg, submittedResourceFilePath, resourceLength, maxFileSize.Value);
throw new ResourceAccessException(msg);
}
//of course, the length cannot be negative
if (resourceLength < 0)
{
string msg = "Upload for file [{0}] denied: Resource length cannot be negative [{1}].";
msg = String.Format(msg, submittedResourceFilePath, resourceLength);
throw new ResourceAccessException(msg);
}
TFile fileItem = CreateFileItemImpl(submittedResourceFilePath, false, context);
if (fileItem.Exists && !overwrite)
{
AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.FileAlreadyExists, fileItem);
string msg = String.Format("Cannot upload file [{0}] without overwriting existing data - a file already exists at this location.", submittedResourceFilePath);
throw new ResourceOverwriteException(msg)
{
IsAudited = true
};
}
//get authorization
TFolder parentFolder = GetParentFolder(fileItem, context);
//validate file system specific restrictions
VerifyCanUploadFileToFileSystemLocation(submittedResourceFilePath, parentFolder, fileItem);
//get parent folder and check whether files can be added
FolderClaims folderClaims = GetParentFolderClaims(fileItem, parentFolder);
if (!folderClaims.AllowAddFiles)
{
//deny adding a file to that folder
AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.CreateFileDenied, fileItem);
string msg = "Cannot create file at [{0}] - adding files to the folder is not permitted.";
msg = String.Format(msg, submittedResourceFilePath);
throw new ResourceAccessException(msg)
{
IsAudited = true
};
}
//only overwrite a file if explicitly requested
FileClaims claims = GetFileClaims(fileItem);
if (fileItem.Exists)
{
if (!claims.AllowOverwrite)
{
AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.FileDataOverwriteDenied, fileItem);
string msg = "Overwriting file [{0}] was denied due to missing permission.";
msg = String.Format(msg, submittedResourceFilePath);
throw new ResourceOverwriteException(msg)
{
IsAudited = true
};
}
}
//try to get lock
ResourceLockGuard writeLock = LockResourceForUpload(fileItem);
if (writeLock != null && !writeLock.IsLockEnabled)
{
AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.FileReadLockDenied, fileItem);
string msg = "The file [{0}] is currently locked and cannot be accessed.";
msg = String.Format(msg, submittedResourceFilePath);
throw new ResourceLockedException(msg)
{
IsAudited = true
};
}
//create upload token
UploadToken token = CreateUploadToken(submittedResourceFilePath, fileItem, resourceLength, contentType);
//create expiration job if we have an expiration time
Job <UploadToken> job = null;
if (token.ExpirationTime.HasValue)
{
job = ScheduleExpiration(token);
}
//create and cache transfer instance
TTransfer transfer = CreateTransfer(submittedResourceFilePath, fileItem, parentFolder, token, claims, writeLock, job);
TransferStore.AddTransfer(transfer);
AuditHelper.AuditResourceOperation(Auditor, context, AuditEvent.UploadTokenIssued, fileItem);
return(transfer);
}
/// <summary>
/// Creates a transfer object for a given file resource.
/// </summary>
/// <param name="submittedResourceFilePath">The resource identifier as submitted.</param>
/// <param name="fileItem">Represents the file resource to be uploaded.</param>
/// <param name="parentFolder">The file's parent folder.</param>
/// <param name="token">The token that is being issued for the transfer.</param>
/// <param name="claims">The access rights for the resource.</param>
/// <param name="lockGuard">File locks, if necessary. Can be a null reference
/// if no locking takes place.</param>
/// <param name="expirationJob">A scheduled job that invokes the
/// <see cref="TransferHandlerBase{TFile,TToken,TTransfer}.OnTransferExpiration"/>
/// method once the transfer expires. May be null if the token does not expire.</param>
/// <returns>A transfer object which encapsulates the information required to perform
/// the transfer.</returns>
protected abstract TTransfer CreateTransfer(string submittedResourceFilePath, TFile fileItem, TFolder parentFolder, UploadToken token, FileClaims claims, ResourceLockGuard lockGuard, Job <UploadToken> expirationJob);
/////////////// Application (former Cluster) Messaging
protected internal abstract void PersistUploadToken(UploadToken value);
/// <summary>
/// Initializes a new instance of the <see cref="T:System.Object"/> class.
/// </summary>
public ZipUploadTransfer(UploadToken token, ZipFileItem fileItem) : base(token, fileItem)
{
}
