/// <summary> /// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class. /// </summary> /// <param name="token">The upload token that identifies the transfer.</param> /// <param name="lastTransmittedBlockNumber">The last transferred block number, if this stream /// is created for an upload that is already in progress. Can be set along with the <paramref name="startOffset"/> in /// order to resume an upload. Defaults to null in case of an initial upload.</param> /// <param name="startOffset">The initial offset of the upload. Can be set along with the /// <paramref name="lastTransmittedBlockNumber"/> in order to resume an upload. Defaults to /// zero in case of an initial upload.</param> /// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush /// its internal buffer once it reaches the defined size.<br/> /// If the threshold is 0, every write is immediately flushed.</param> /// <param name="outputAction">An action that is being invoked with created /// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param> /// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction" /// is a null reference.</exception> /// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/> /// is negative.</exception> public StreamedBlockOutputStream(UploadToken token, int autoFlushThreshold, long startOffset, long? lastTransmittedBlockNumber, Action<StreamedDataBlock> outputAction) : base(token, autoFlushThreshold, startOffset, lastTransmittedBlockNumber, b => { }) { OutputAction = outputAction; base.OutputAction = ParseBuffer; }
/// <summary> /// Loads the dataset into the dataset application. /// </summary> /// <param name="ownerId">The ID of the endpoint which requested the load of the dataset.</param> /// <param name="token">The token that indicates which file should be uploaded.</param> /// <returns>A task that will finish once the dataset is loaded.</returns> public Task Load(EndpointId ownerId, UploadToken token) { // This one is more tricky than you think. We need to return a Task (not a Task<T>), however // if we do Task<T>.ContinueWith we get a System.Threading.Tasks.ContinuationTaskFromResultTask<System.IO.Stream> // object. When we give that back to the command system it tries to push the stream across the network ... not // really what we want. // // So to fix this we create a task and then inside that task we load the file (via another task) and continue // of the nested task. The continuation task is a child of the global task and all is well because the // global task doesn't finish until the nested child task is done, and we get a Task object back var globalTask = Task.Factory.StartNew( () => { var filePath = Path.GetTempFileName(); var task = m_DataDownload(ownerId, token, filePath); task.ContinueWith( t => { using (m_Diagnostics.Profiler.Measure(BaseConstants.TimingGroup, "Loading dataset from file.")) { m_LoadAction(new FileInfo(filePath)); } }, TaskContinuationOptions.AttachedToParent | TaskContinuationOptions.ExecuteSynchronously); }); return(globalTask); }
/// <summary> /// Creates a token for the currently processed resource. /// </summary> /// <param name="submittedResourceFilePath">The identifier (file path) /// for the requested resource, as submitted by the requesting party.</param> /// <param name="fileItem">Represents the file resource to be uploaded.</param> /// <param name="resourceLength">The length of the resource to be uploaded in bytes.</param> /// <param name="contentType">The content type of the uploaded resource.</param> /// <returns>A token for the request.</returns> protected override UploadToken CreateUploadToken(string submittedResourceFilePath, ZipFileItem fileItem, long resourceLength, string contentType) { var fileInfo = fileItem.ResourceInfo; string transferId = Guid.NewGuid().ToString(); var token = new UploadToken { TransferId = transferId, ResourceIdentifier = submittedResourceFilePath, CreationTime = SystemTime.Now(), ContentType = fileInfo.ContentType, MaxResourceSize = resourceLength, MaxBlockSize = MaxBlockSize, ResourceName = fileInfo.Name, ResourceLength = resourceLength }; TimeSpan?expiration = Config.FileSystemConfiguration.UploadTokenExpirationTime; if (expiration.HasValue) { token.ExpirationTime = SystemTime.Now().Add(expiration.Value); } return(token); }
/// <summary> /// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class. /// </summary> /// <param name="token">The upload token that identifies the transfer.</param> /// <param name="lastTransmittedBlockNumber">The last transferred block number, if this stream /// is created for an upload that is already in progress. Can be set along with the <paramref name="startOffset"/> in /// order to resume an upload. Defaults to null in case of an initial upload.</param> /// <param name="startOffset">The initial offset of the upload. Can be set along with the /// <paramref name="lastTransmittedBlockNumber"/> in order to resume an upload. Defaults to /// zero in case of an initial upload.</param> /// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush /// its internal buffer once it reaches the defined size.<br/> /// If the threshold is 0, every write is immediately flushed.</param> /// <param name="outputAction">An action that is being invoked with created /// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param> /// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction" /// is a null reference.</exception> /// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/> /// is negative.</exception> public StreamedBlockOutputStream(UploadToken token, int autoFlushThreshold, long startOffset, long?lastTransmittedBlockNumber, Action <StreamedDataBlock> outputAction) : base(token, autoFlushThreshold, startOffset, lastTransmittedBlockNumber, b => { }) { OutputAction = outputAction; base.OutputAction = ParseBuffer; }
/// <summary> /// 获取令牌 /// </summary> public void tk() { UploadToken uploadToken = new UploadToken(); string name = _request.QueryString["name"]; string size = _request.QueryString["size"]; string ext = name.Substring(name.LastIndexOf('.')); string token = SimpleEncryptor.MD5(name + size); uploadToken.name = name; uploadToken.size = string.IsNullOrEmpty(size) ? 0 : int.Parse(size); uploadToken.token = token; if (!File.Exists(_server.MapPath(_tokenPath + token + ".token"))) { string modified = _request.QueryString["modified"]; uploadToken.filePath = ""; uploadToken.modified = modified; SetTokenInfo(token, uploadToken); } TokenResult tokenResult = new TokenResult(); tokenResult.message = ""; tokenResult.token = token; tokenResult.success = true; string result = JSONHelper.SerializeObject(tokenResult);; _response.Write(result); }
private string UploadChunck(UploadToken token, string fileName, long fileSize, int offset, int bytesToUpload, Action <long> cb) { HttpWebRequest r = (HttpWebRequest)WebRequest.Create(token.upload_link_secure); r.Method = "PUT"; r.ContentLength = fileSize; r.ContentType = "video/x-ms-wmv"; r.KeepAlive = true; r.Timeout = System.Threading.Timeout.Infinite; r.ReadWriteTimeout = System.Threading.Timeout.Infinite; r.Headers[HttpRequestHeader.Authorization] = authorization; if (offset > 0) { r.Headers[HttpRequestHeader.ContentRange] = String.Format("bytes {0}-{1}/{2}", offset, fileSize, fileSize); } var file = File.OpenRead(fileName); file.Seek(offset, SeekOrigin.Begin); const int BUFSIZE = 1048576; byte[] buffer = new byte[BUFSIZE]; string s = ""; try { using (Stream str = r.GetRequestStream()) { long bytesRead = 0; while (bytesRead < fileSize) { int n = file.Read(buffer, 0, BUFSIZE); if (n == 0) { break; } str.Write(buffer, 0, n); bytesRead += n; cb(bytesRead); } } file.Close(); file.Dispose(); file = null; } catch (WebException e) { s = e.Message + " - " + e.Status.ToString(); } catch (IOException e) { s = e.Message; } return(s); }
/// <summary> /// Initializes a new instance of the <see cref="DataDownloadRequestMessage"/> class. /// </summary> /// <param name="origin">The ID of the endpoint that send the message.</param> /// <param name="id">The ID of the current message.</param> /// <param name="token">The token that indicates which file should be uploaded.</param> /// <exception cref="ArgumentNullException"> /// Thrown if <paramref name="origin"/> is <see langword="null" />. /// </exception> /// <exception cref="ArgumentNullException"> /// Thrown if <paramref name="id"/> is <see langword="null" />. /// </exception> /// <exception cref="ArgumentNullException"> /// Thrown if <paramref name="token"/> is <see langword="null" />. /// </exception> public DataDownloadRequestMessage(EndpointId origin, MessageId id, UploadToken token) : base(origin, id) { { Lokad.Enforce.Argument(() => token); } Token = token; }
private void OnUploadComplete(UploadToken response, Exception error) { if (error != null) { throw error; } _FileStream.Close(); openTasks--; }
public void Create() { var sender = new EndpointId("sendingEndpoint"); var token = new UploadToken(); var msg = new DataDownloadRequestMessage(sender, token); Assert.IsNotNull(msg.Id); Assert.AreSame(sender, msg.Sender); Assert.AreSame(token, msg.Token); }
public void InvokeWithoutFileRegistration() { var uploads = new WaitingUploads(); var layer = new Mock <IProtocolLayer>(); { layer.Setup(l => l.Id) .Returns(new EndpointId("other")); layer.Setup(l => l.SendMessageTo(It.IsAny <EndpointId>(), It.IsAny <ICommunicationMessage>(), It.IsAny <int>())) .Callback <EndpointId, ICommunicationMessage, int>( (e, m, r) => { Assert.IsInstanceOf <FailureMessage>(m); }) .Verifiable(); layer.Setup(l => l.UploadData( It.IsAny <EndpointId>(), It.IsAny <string>(), It.IsAny <CancellationToken>(), It.IsAny <TaskScheduler>(), It.IsAny <int>())) .Returns(Task.Factory.StartNew( () => { }, new CancellationToken(), TaskCreationOptions.None, new CurrentThreadTaskScheduler())) .Verifiable(); } var systemDiagnostics = new SystemDiagnostics((p, s) => { }, null); var action = new DataDownloadProcessAction( uploads, layer.Object, systemDiagnostics, new CurrentThreadTaskScheduler()); var token = new UploadToken(); var msg = new DataDownloadRequestMessage( EndpointIdExtensions.CreateEndpointIdForCurrentProcess(), token); action.Invoke(msg); layer.Verify(l => l.SendMessageTo(It.IsAny <EndpointId>(), It.IsAny <ICommunicationMessage>(), It.IsAny <int>()), Times.Once()); layer.Verify( l => l.UploadData( It.IsAny <EndpointId>(), It.IsAny <string>(), It.IsAny <CancellationToken>(), It.IsAny <TaskScheduler>(), It.IsAny <int>()), Times.Never()); }
public void Init() { receivedBlocks = new List <BufferedDataBlock>(); source = new byte[100000]; target = new List <byte>(); new Random(DateTime.Now.Millisecond).NextBytes(source); token = new UploadToken { TransferId = "MyToken", MaxBlockSize = 3000 }; }
public void CreateWithId() { var sender = new EndpointId("sendingEndpoint"); var id = new MessageId(); var token = new UploadToken(); var msg = new DataDownloadRequestMessage(sender, id, token); Assert.AreSame(id, msg.Id); Assert.AreSame(sender, msg.Sender); Assert.AreSame(token, msg.Token); }
public void Init() { receivedBlocks = new List <BufferedDataBlock>(); source = new byte[100000]; target = new List <byte>(); new Random(DateTime.Now.Millisecond).NextBytes(source); //token indicates 10 blocks were already submitted token = new UploadToken { TransferId = "MyToken", MaxBlockSize = 3000, TransmittedBlockCount = 10 }; }
private string CompleteUpload(UploadToken token) { HttpWebRequest r = (HttpWebRequest)WebRequest.Create("https://api.vimeo.com" + token.complete_uri); r.Method = "DELETE"; r.Headers[HttpRequestHeader.Authorization] = authorization; HttpWebResponse response = (HttpWebResponse)r.GetResponse(); var s = response.Headers["Location"]; response.Close(); return(s.Replace("/videos/", "")); }
private void OnUploadTokenAddComplete(UploadToken token, Exception error) { openTasks++; if (error != null) { throw error; } UploadTokenService.Upload(token.Id, _FileStream) .SetCompletion(new OnCompletedHandler <UploadToken>(OnUploadComplete)) .Execute(client); openTasks--; }
private UploadToken GetTokenInfo(string token) { string tokenPath = _tokenPath + token + ".token"; if (File.Exists(_server.MapPath(tokenPath))) { _fileHelper.FileName = token + ".token"; _fileHelper.FilePath = _server.MapPath(_tokenPath); UploadToken uploadToken = JSONHelper.DeserializeJsonToObject <UploadToken>(_fileHelper.ReadFile()); return(uploadToken); } return(null); }
public void upload() { string token = _request.QueryString["token"]; UploadToken uploadToken = GetTokenInfo(token); if (uploadToken != null && uploadToken.size > uploadToken.upsize) { Stream stream = _request.InputStream; if (stream != null && stream.Length > 0) { _fileHelper.FileName = uploadToken.name; _fileHelper.FilePath = _server.MapPath(_filePath); _fileHelper.WriteFile(stream); uploadToken.upsize += stream.Length; if (uploadToken.size > uploadToken.upsize) { SetTokenInfo(token, uploadToken); } else { //上传完成后删除令牌信息 DelTokenInfo(token); //重命名(解决重复上传时内容叠加) //FileInfo fileinfo = new FileInfo(_server.MapPath(_filePath) + uploadToken.name); //string Rename = string.Format(@"{0}_{1}{2}", Path.GetFileNameWithoutExtension(fileinfo.FullName), DateTime.Now.ToString("mmssffff"), fileinfo.Extension); //fileinfo.MoveTo(_server.MapPath(_filePath) + Rename); //uploadToken.name = Rename; //无需重命名 uploadToken.name = uploadToken.name; } } } UploadResult ur = new UploadResult(); ur.message = ""; ur.filePath = HttpUtility.UrlEncode(_filePath + uploadToken.name, Encoding.UTF8); ur.start = uploadToken.upsize; ur.success = true; string result = JSONHelper.SerializeObject(ur); _response.Write(result); }
public void LocatePluginAssemblyDownloadFail() { var token = new UploadToken(); var commands = new Mock <IHostApplicationCommands>(); { commands.Setup(c => c.PreparePluginContainerForTransfer(It.IsAny <AssemblyName>())) .Returns( Task <UploadToken> .Factory.StartNew( () => { return(token); })); } var commandContainer = new Mock <ISendCommandsToRemoteEndpoints>(); { commandContainer.Setup(c => c.CommandsFor <IHostApplicationCommands>(It.IsAny <EndpointId>())) .Returns(commands.Object); } DownloadDataFromRemoteEndpoints layer = (id, u, file) => Task <FileInfo> .Factory.StartNew( () => { Assert.AreSame(token, u); throw new FileNotFoundException(); }); var fileSystem = new Mock <IFileSystem>(); { fileSystem.Setup(f => f.Path) .Returns(new MockPath()); fileSystem.Setup(f => f.File) .Returns(new MockFile(new Dictionary <string, string>())); } var endpoint = new EndpointId("a"); var resolver = new PluginLoadingAssemblyResolver( commandContainer.Object, layer, fileSystem.Object, endpoint); Assert.IsNull(resolver.LocatePluginAssembly(new object(), new ResolveEventArgs("a"))); }
public void LocatePluginAssemblyDownloadFail() { var token = new UploadToken(); var commands = new Mock<IHostApplicationCommands>(); { commands.Setup(c => c.PreparePluginContainerForTransfer(It.IsAny<AssemblyName>())) .Returns( Task<UploadToken>.Factory.StartNew( () => { return token; })); } var commandContainer = new Mock<ISendCommandsToRemoteEndpoints>(); { commandContainer.Setup(c => c.CommandsFor<IHostApplicationCommands>(It.IsAny<EndpointId>())) .Returns(commands.Object); } DownloadDataFromRemoteEndpoints layer = (id, u, file) => Task<FileInfo>.Factory.StartNew( () => { Assert.AreSame(token, u); throw new FileNotFoundException(); }); var fileSystem = new Mock<IFileSystem>(); { fileSystem.Setup(f => f.Path) .Returns(new MockPath()); fileSystem.Setup(f => f.File) .Returns(new MockFile(new Dictionary<string, string>())); } var endpoint = new EndpointId("a"); var resolver = new PluginLoadingAssemblyResolver( commandContainer.Object, layer, fileSystem.Object, endpoint); Assert.IsNull(resolver.LocatePluginAssembly(new object(), new ResolveEventArgs("a"))); }
private UploadStatus VerifyUpload(UploadToken token) { HttpWebRequest r = (HttpWebRequest)WebRequest.Create(token.upload_link_secure); r.Method = "PUT"; r.ContentLength = 0; r.Headers[HttpRequestHeader.ContentRange] = "bytes */*"; r.Headers[HttpRequestHeader.Authorization] = authorization; try { HttpWebResponse response = (HttpWebResponse)r.GetResponse(); } catch (WebException e) { var s = e.Response.Headers["Range"]; return(new UploadStatus(s)); } throw new Exception(); }
/// <summary> /// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class. /// </summary> /// <param name="token">The upload token that identifies the transfer.</param> /// <param name="lastTransmittedBlockNumber">The last transferred block number, if this stream /// is created for an upload that is already in progress. Can be set along with the <paramref name="startOffset"/> in /// order to resume an upload. Defaults to null in case of an initial upload.</param> /// <param name="startOffset">The initial offset of the upload. Can be set along with the /// <paramref name="lastTransmittedBlockNumber"/> in order to resume an upload. Defaults to /// zero in case of an initial upload.</param> /// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush /// its internal buffer once it reaches the defined size.<br/> /// If the threshold is 0, every write is immediately flushed.</param> /// <param name="outputAction">An action that is being invoked with created /// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param> /// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction" /// is a null reference.</exception> /// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/> /// is negative.</exception> public BufferedBlockOutputStream(UploadToken token, int autoFlushThreshold, long startOffset, long?lastTransmittedBlockNumber, Action <BufferedDataBlock> outputAction) { if (token == null) { throw new ArgumentNullException("token"); } if (outputAction == null) { throw new ArgumentNullException("outputAction"); } if (autoFlushThreshold < 0) { throw new ArgumentOutOfRangeException("autoFlushThreshold", "Threshold for automatic flushing cannot be negative"); } Token = token; AutoFlushThreshold = autoFlushThreshold; OutputAction = outputAction; InitialOffset = startOffset; LastTransmittedBlockNumber = lastTransmittedBlockNumber; }
private void WriteFileImpl(string virtualFilePath, Stream input, bool overwrite, long resourceLength, string contentType) { //get a token - this creates a new transfer and validates the request UploadToken token = RequestUploadToken(virtualFilePath, overwrite, resourceLength, contentType); if (resourceLength == 0) { //create an empty data block and submit it var block = new BufferedDataBlock { TransferTokenId = token.TransferId, BlockLength = 0, BlockNumber = 0, Data = new byte[0], IsLastBlock = true }; WriteBlock(block); } else { //we don't expose the underlying stream directly, but rather use blocks again, //which disconnects the exposed stream from the underlying resource and allows for simple //aborting try { input.WriteTo(token, resourceLength, token.MaxBlockSize ?? 32768, WriteBlockStreamed); } catch (Exception) { CancelTransfer(token.TransferId, AbortReason.Undefined); throw; } //we don't need to explicitly complete the transfer - the extension method //implicitly closed the transfer by marking the last block: //CompleteTransfer(token.TransferId); } }
public void upload() { string token = _request.QueryString["token"]; UploadToken uploadToken = GetTokenInfo(token); if (uploadToken != null && uploadToken.size > uploadToken.upsize) { Stream stream = _request.InputStream; if (stream != null && stream.Length > 0) { _fileHelper.FileName = uploadToken.name; _fileHelper.FilePath = _server.MapPath(_filePath); _fileHelper.WriteFile(stream); uploadToken.upsize += stream.Length; if (uploadToken.size > uploadToken.upsize) { SetTokenInfo(token, uploadToken); } else { //上传完成后删除令牌信息 DelTokenInfo(token); } } } UploadResult ur = new UploadResult(); ur.message = ""; ur.start = uploadToken.upsize; ur.success = true; string result = JsonHelper.SerializeObject(ur); _response.Write(result); }
private bool DispatchUploadRequest(HttpApplication application, HttpRequest request) { // if the request is Upload (POST) request, try to authenticate user HttpContext context = HttpContext.Current; string receivedUploadToken = request.Form["UploadToken"]; if (receivedUploadToken != null) { UploadToken uploadToken = UploadToken.GetUploadToken(new Guid(receivedUploadToken)); if (uploadToken == null) { throw new InvalidOperationException(string.Format("An UploadToken ({0}) has been sent, but a matching user cannot be found.", receivedUploadToken)); } UserAccessProvider.ChangeToSystemAccount(); User uploadUser = (User)SenseNet.ContentRepository.Storage.Node.LoadNode(uploadToken.UserId); UserAccessProvider.RestoreOriginalUser(); context.User = new PortalPrincipal(uploadUser); return(true); } return(false); }
/// <summary> /// Prepares the report files for upload. /// </summary> /// <param name="testId">The ID of the test to which the files belong.</param> /// <param name="testStepIndex">The index of the test step for which the report files are being uploaded.</param> /// <param name="callingEndpoint">The ID of the endpoint that called the method.</param> /// <param name="token">The upload token used to register the report files that need to be uploaded.</param> /// <returns>An upload token that can be used to download the desired files.</returns> public Task PrepareReportFilesForTransfer(int testId, int testStepIndex, EndpointId callingEndpoint, UploadToken token) { m_Diagnostics.Log( LevelToLog.Trace, MasterServiceConstants.LogPrefix, string.Format( CultureInfo.InvariantCulture, Resources.Log_Messages_TransferingTestStepReportFiles_WithTestAndTestStep, testId, testStepIndex)); var downloadDirectory = TestHelpers.StoragePathForReportFiles(testId, m_Configuration, m_FileSystem); if (!m_FileSystem.Directory.Exists(downloadDirectory)) { m_FileSystem.Directory.CreateDirectory(downloadDirectory); } var fileName = string.Format( CultureInfo.InvariantCulture, "{0}_{1}.reportfiles", testId, testStepIndex); var filePath = m_FileSystem.Path.Combine( downloadDirectory, string.Format( CultureInfo.InvariantCulture, "{0}.zip", fileName)); var fileStream = m_DataDownload( callingEndpoint, token, filePath); var uploadTask = fileStream.ContinueWith( file => { try { var testPackage = new TestStepPackage(testStepIndex); var unpackDirectory = Path.Combine(downloadDirectory, fileName); testPackage.LoadAndUnpack(file.Result.FullName, unpackDirectory); var notifications = m_ActiveTests.NotificationsFor(testId); foreach (var notification in notifications) { foreach (var storedFile in testPackage.StoredFiles()) { var relativePath = storedFile.FullName.Substring(downloadDirectory.Length).TrimStart(Path.DirectorySeparatorChar); notification.StoreReportFile(storedFile.FullName, relativePath); } } } catch (Exception e) { m_Diagnostics.Log( LevelToLog.Error, MasterServiceConstants.LogPrefix, string.Format( CultureInfo.InvariantCulture, Resources.Log_Messages_TransferingTestStepReportFilesFailed_WithTestAndTestStepAndError, testId, testStepIndex, e)); } }); return uploadTask; }
/// <summary> /// Creates a transfer object for a given file resource. /// </summary> /// <param name="submittedResourceFilePath">The resource identifier as submitted.</param> /// <param name="fileItem">Represents the file resource to be uploaded.</param> /// <param name="parentFolder">The file's parent folder.</param> /// <param name="token">The token that is being issued for the transfer.</param> /// <param name="claims">The access rights for the resource.</param> /// <param name="lockGuard">File locks, if necessary. Can be a null reference /// if no locking takes place.</param> /// <param name="expirationJob">A scheduled job that invokes the /// <see cref="TransferHandlerBase{TFile,TToken,TTransfer}.OnTransferExpiration"/> /// method once the transfer expires. May be null if the token does not expire.</param> /// <returns>A transfer object which encapsulates the information required to perform /// the transfer.</returns> protected override ZipUploadTransfer CreateTransfer(string submittedResourceFilePath, ZipFileItem fileItem, ZipFolderItem parentFolder, UploadToken token, FileClaims claims, ResourceLockGuard lockGuard, Job <UploadToken> expirationJob) { var transfer = new ZipUploadTransfer(token, fileItem) { ParentFolder = parentFolder, Status = TransferStatus.Starting, Owner = Config.Provider.Security.GetIdentity() }; if (expirationJob != null) { transfer.ExpirationNotificationJob = expirationJob; } if (lockGuard != null) { transfer.ResourceLock = lockGuard; } return(transfer); }
/// <summary> /// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class that initiates /// a new upload (starting with block 0 at offset 0). /// </summary> /// <param name="token">The upload token that identifies the transfer.</param> /// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush /// its internal buffer once it reaches the defined size.<br/> /// If the threshold is 0, every write is immediately flushed.</param> /// <param name="outputAction">An action that is being invoked with created /// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param> /// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction" /// is a null reference.</exception> /// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/> /// is negative.</exception> public StreamedBlockOutputStream(UploadToken token, int autoFlushThreshold, Action<StreamedDataBlock> outputAction) : base(token, autoFlushThreshold, b => { }) { OutputAction = outputAction; base.OutputAction = ParseBuffer; }
/// <summary> /// Starts a download. /// </summary> /// <param name="downloadOwningEndpoint">The endpoint ID of the endpoint that owns the data stream.</param> /// <param name="token">The upload token that allows the receiver to indicate which data stream should be downloaded.</param> public void StartDownload([InvokingEndpoint] EndpointId downloadOwningEndpoint, UploadToken token) { var path = Path.Combine(Assembly.GetExecutingAssembly().LocalDirectoryPath(), Path.GetRandomFileName()); var task = m_Download(downloadOwningEndpoint, token, path, TimeSpan.FromSeconds(15)); string text; try { task.Wait(); text = new StreamReader(task.Result.FullName).ReadToEnd(); } catch (AggregateException) { text = "Failed to download data."; } m_OnEcho( downloadOwningEndpoint, string.Format( CultureInfo.InvariantCulture, "Downloaded data from: {0}. Data: {1}", downloadOwningEndpoint, text)); }
/// <summary> /// Starts the execution of the given test steps. /// </summary> /// <param name="testId">The ID of the test that is being executed.</param> /// <param name="testSteps">The collection of test steps that should be executed.</param> /// <param name="environmentParameters">The collection that contains the parameters related to the test environment.</param> /// <param name="callingEndpoint">The ID of the endpoint that called the method.</param> /// <param name="token">The upload token that can be used to upload the file.</param> /// <returns>A task which completes when the execution has started successfully.</returns> public Task Execute( int testId, List<TestStep> testSteps, List<InputParameter> environmentParameters, EndpointId callingEndpoint, UploadToken token) { return Task.Factory.StartNew(() => StartNewTestExecution(testId, testSteps, environmentParameters, callingEndpoint, token)); }
/// <summary> /// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class. /// </summary> /// <param name="token">The upload token that identifies the transfer.</param> /// <param name="lastTransmittedBlockNumber">The last transferred block number, if this stream /// is created for an upload that is already in progress. Can be set along with the <paramref name="startOffset"/> in /// order to resume an upload. Defaults to null in case of an initial upload.</param> /// <param name="startOffset">The initial offset of the upload. Can be set along with the /// <paramref name="lastTransmittedBlockNumber"/> in order to resume an upload. Defaults to /// zero in case of an initial upload.</param> /// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush /// its internal buffer once it reaches the defined size.<br/> /// If the threshold is 0, every write is immediately flushed.</param> /// <param name="outputAction">An action that is being invoked with created /// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param> /// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction" /// is a null reference.</exception> /// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/> /// is negative.</exception> public BufferedBlockOutputStream(UploadToken token, int autoFlushThreshold, long startOffset, long? lastTransmittedBlockNumber, Action<BufferedDataBlock> outputAction) { if (token == null) throw new ArgumentNullException("token"); if (outputAction == null) throw new ArgumentNullException("outputAction"); if(autoFlushThreshold < 0) { throw new ArgumentOutOfRangeException("autoFlushThreshold", "Threshold for automatic flushing cannot be negative"); } Token = token; AutoFlushThreshold = autoFlushThreshold; OutputAction = outputAction; InitialOffset = startOffset; LastTransmittedBlockNumber = lastTransmittedBlockNumber; }
/// <summary> /// Loads the dataset into the dataset application. /// </summary> /// <param name="ownerId">The ID of the endpoint which requested the load of the dataset.</param> /// <param name="token">The token that indicates which file should be uploaded.</param> /// <returns>A task that will finish once the dataset is loaded.</returns> public Task Load(EndpointId ownerId, UploadToken token) { // This one is more tricky than you think. We need to return a Task (not a Task<T>), however // if we do Task<T>.ContinueWith we get a System.Threading.Tasks.ContinuationTaskFromResultTask<System.IO.Stream> // object. When we give that back to the command system it tries to push the stream across the network ... not // really what we want. // // So to fix this we create a task and then inside that task we load the file (via another task) and continue // of the nested task. The continuation task is a child of the global task and all is well because the // global task doesn't finish until the nested child task is done, and we get a Task object back var globalTask = Task.Factory.StartNew( () => { var filePath = Path.GetTempFileName(); var task = m_DataDownload(ownerId, token, filePath); task.ContinueWith( t => { using (m_Diagnostics.Profiler.Measure(BaseConstants.TimingGroup, "Loading dataset from file.")) { m_LoadAction(new FileInfo(filePath)); } }, TaskContinuationOptions.AttachedToParent | TaskContinuationOptions.ExecuteSynchronously); }); return globalTask; }
private bool DownloadTestData( EndpointId callingEndpoint, UploadToken token, string storageDirectory, ITestSectionBuilder builder, out string testFile) { testFile = string.Empty; try { testFile = DownloadTestData(callingEndpoint, token, storageDirectory); m_Diagnostics.Log( LevelToLog.Debug, ExecutorServiceConstants.LogPrefix, string.Format( CultureInfo.InvariantCulture, Resources.Log_Messages_DownloadedTestData_WithFile, testFile)); builder.AddInformationMessage(Resources.ReportSection_Info_DownloadedTestData); } catch (Exception) { builder.AddErrorMessage(Resources.ReportSection_Error_FailedToDownloadTestData); builder.FinalizeAndStore(false); m_TestExecutionEvents.RaiseOnTestCompletion(TestExecutionResult.Failed); return false; } return true; }
/// <summary> /// Initializes a new instance of the <see cref="T:System.Object"/> class. /// </summary> public LocalUploadTransfer(UploadToken token, FileItem fileItem) : base(token, fileItem) { }
private string DownloadTestData(EndpointId callingEndpoint, UploadToken token, string storageDirectory) { var filePath = m_FileSystem.Path.Combine( storageDirectory, string.Format( CultureInfo.InvariantCulture, "{0}.zip", Guid.NewGuid().ToString("D"))); var fileStream = m_DataDownload(callingEndpoint, token, filePath); fileStream.Wait(); return fileStream.Result.FullName; }
private void StartNewTestExecution( int testId, List<TestStep> testSteps, List<InputParameter> environmentParameters, EndpointId callingEndpoint, UploadToken token) { var builder = m_SectionBuilders(Resources.ReportSection_Group_Name_Initialization); builder.Initialize(Resources.ReportSection_Name_Initialization); m_HostInformation.Id = callingEndpoint; string testFile; if (!DownloadTestData(callingEndpoint, token, m_StorageDirectory, builder, out testFile)) { return; } m_TestInformation.TestId = testId; m_TestInformation.TestSteps = testSteps; m_TestInformation.TestPackage = testFile; m_TestInformation.EnvironmentParameters = environmentParameters; try { var pair = m_Layer.LocalConnectionFor(ChannelType.NamedPipe); m_CurrentEndpoint = StartExecutorApplication(pair.Item1, pair.Item2); m_TestInformation.CurrentState = TestExecutionState.Running; m_Diagnostics.Log( LevelToLog.Debug, ExecutorServiceConstants.LogPrefix, Resources.Log_Messages_StartedTestApplication); } catch (Exception e) { m_TestInformation.CurrentState = TestExecutionState.None; m_Diagnostics.Log( LevelToLog.Error, ExecutorServiceConstants.LogPrefix, string.Format( CultureInfo.InvariantCulture, Resources.Log_Messages_FailedToStartTestApplication_WithError, e)); builder.AddErrorMessage(Resources.ReportSection_Error_FailedToStartTestingApplication); builder.FinalizeAndStore(false); m_TestExecutionEvents.RaiseOnTestCompletion(TestExecutionResult.Failed); return; } builder.AddInformationMessage( string.Format( CultureInfo.InvariantCulture, Resources.ReportSection_Info_StartedTestApplication_WithId, m_CurrentEndpoint)); try { WaitForExecutorConnection(m_CurrentEndpoint); } catch (TestExecutionFailureException) { m_TestInformation.CurrentState = TestExecutionState.None; m_Diagnostics.Log( LevelToLog.Error, ExecutorServiceConstants.LogPrefix, Resources.Log_Messages_FailedToConnectToTestApplication_WithError); builder.AddErrorMessage(Resources.ReportSection_Error_FailedToConnectToTestingApplication); builder.FinalizeAndStore(false); m_TestExecutionEvents.RaiseOnTestCompletion(TestExecutionResult.Failed); return; } ConnectEvents(m_CurrentEndpoint); builder.FinalizeAndStore(true); }
/// <summary> /// Prepares the report files for upload. /// </summary> /// <param name="testStepIndex">The index of the test step for which the report files are being uploaded.</param> /// <param name="callingEndpoint">The ID of the endpoint that called the method.</param> /// <param name="token">The upload token used to register the report files that need to be uploaded.</param> /// <returns>An upload token that can be used to download the desired files.</returns> public Task PrepareReportFilesForTransfer(int testStepIndex, EndpointId callingEndpoint, UploadToken token) { m_Diagnostics.Log( LevelToLog.Trace, ExecutorServiceConstants.LogPrefix, string.Format( CultureInfo.InvariantCulture, Resources.Log_Messages_TransferingTestStepReportFiles_WithTestStep, testStepIndex)); var filePath = m_FileSystem.Path.Combine( m_StorageDirectory, string.Format( CultureInfo.InvariantCulture, "{0}.zip", Guid.NewGuid().ToString("D"))); var fileStream = m_DataDownload(callingEndpoint, token, filePath); var uploadTask = fileStream.ContinueWith( file => { try { // Upload to the host here var hostToken = m_Uploads.Register(file.Result.FullName); if (!m_RemoteCommands.HasCommandFor(m_HostInformation.Id, typeof(IStoreTestReportDataCommands))) { throw new MissingCommandSetException(); } var id = EndpointIdExtensions.CreateEndpointIdForCurrentProcess(); var command = m_RemoteCommands.CommandsFor<IStoreTestReportDataCommands>(m_HostInformation.Id); var task = CommandSetGuard.GuardAgainstCommunicationFailure( command.PrepareReportFilesForTransfer, m_TestInformation.TestId, testStepIndex, id, hostToken); task.Wait(); } catch (Exception e) { m_Diagnostics.Log( LevelToLog.Error, ExecutorServiceConstants.LogPrefix, string.Format( CultureInfo.InvariantCulture, Resources.Log_Messages_TransferingTestStepReportFilesFailed_WithException, e)); throw; } }); return uploadTask; }
/// <summary> /// Initializes a new instance of the <see cref="T:System.IO.Stream"/> class that initiates /// a new upload (starting with block 0 at offset 0). /// </summary> /// <param name="token">The upload token that identifies the transfer.</param> /// <param name="autoFlushThreshold">A threshold that causes the stream to aumatically flush /// its internal buffer once it reaches the defined size.<br/> /// If the threshold is 0, every write is immediately flushed.</param> /// <param name="outputAction">An action that is being invoked with created /// <see cref="BufferedDataBlock"/> instances when flushing the internal buffer.</param> /// <exception cref="ArgumentNullException">If <paramref name="token"/> or <paramref name="outputAction" /// is a null reference.</exception> /// <exception cref="ArgumentOutOfRangeException">Inf case the <paramref name="autoFlushThreshold"/> /// is negative.</exception> public BufferedBlockOutputStream(UploadToken token, int autoFlushThreshold, Action<BufferedDataBlock> outputAction) : this(token, autoFlushThreshold, 0, null, outputAction) { }
public static UploadTokenAddRequestBuilder Add(UploadToken uploadToken = null) { return(new UploadTokenAddRequestBuilder(uploadToken)); }
private TTransfer InitTransferImpl(string submittedResourceFilePath, bool overwrite, long resourceLength, string contentType) { const FileSystemTask context = FileSystemTask.UploadTokenRequest; //validate maximum file size var maxFileSize = GetMaxFileUploadSize(); if (maxFileSize.HasValue && maxFileSize < resourceLength) { string msg = "Upload for file [{0}] denied: Resource length of [{1}] is above the maximum upload limit of [{2}] bytes."; msg = String.Format(msg, submittedResourceFilePath, resourceLength, maxFileSize.Value); throw new ResourceAccessException(msg); } //of course, the length cannot be negative if (resourceLength < 0) { string msg = "Upload for file [{0}] denied: Resource length cannot be negative [{1}]."; msg = String.Format(msg, submittedResourceFilePath, resourceLength); throw new ResourceAccessException(msg); } TFile fileItem = CreateFileItemImpl(submittedResourceFilePath, false, context); if (fileItem.Exists && !overwrite) { AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.FileAlreadyExists, fileItem); string msg = String.Format("Cannot upload file [{0}] without overwriting existing data - a file already exists at this location.", submittedResourceFilePath); throw new ResourceOverwriteException(msg) { IsAudited = true }; } //get authorization TFolder parentFolder = GetParentFolder(fileItem, context); //validate file system specific restrictions VerifyCanUploadFileToFileSystemLocation(submittedResourceFilePath, parentFolder, fileItem); //get parent folder and check whether files can be added FolderClaims folderClaims = GetParentFolderClaims(fileItem, parentFolder); if (!folderClaims.AllowAddFiles) { //deny adding a file to that folder AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.CreateFileDenied, fileItem); string msg = "Cannot create file at [{0}] - adding files to the folder is not permitted."; msg = String.Format(msg, submittedResourceFilePath); throw new ResourceAccessException(msg) { IsAudited = true }; } //only overwrite a file if explicitly requested FileClaims claims = GetFileClaims(fileItem); if (fileItem.Exists) { if (!claims.AllowOverwrite) { AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.FileDataOverwriteDenied, fileItem); string msg = "Overwriting file [{0}] was denied due to missing permission."; msg = String.Format(msg, submittedResourceFilePath); throw new ResourceOverwriteException(msg) { IsAudited = true }; } } //try to get lock ResourceLockGuard writeLock = LockResourceForUpload(fileItem); if (writeLock != null && !writeLock.IsLockEnabled) { AuditHelper.AuditDeniedOperation(Auditor, context, AuditEvent.FileReadLockDenied, fileItem); string msg = "The file [{0}] is currently locked and cannot be accessed."; msg = String.Format(msg, submittedResourceFilePath); throw new ResourceLockedException(msg) { IsAudited = true }; } //create upload token UploadToken token = CreateUploadToken(submittedResourceFilePath, fileItem, resourceLength, contentType); //create expiration job if we have an expiration time Job <UploadToken> job = null; if (token.ExpirationTime.HasValue) { job = ScheduleExpiration(token); } //create and cache transfer instance TTransfer transfer = CreateTransfer(submittedResourceFilePath, fileItem, parentFolder, token, claims, writeLock, job); TransferStore.AddTransfer(transfer); AuditHelper.AuditResourceOperation(Auditor, context, AuditEvent.UploadTokenIssued, fileItem); return(transfer); }
/// <summary> /// Creates a transfer object for a given file resource. /// </summary> /// <param name="submittedResourceFilePath">The resource identifier as submitted.</param> /// <param name="fileItem">Represents the file resource to be uploaded.</param> /// <param name="parentFolder">The file's parent folder.</param> /// <param name="token">The token that is being issued for the transfer.</param> /// <param name="claims">The access rights for the resource.</param> /// <param name="lockGuard">File locks, if necessary. Can be a null reference /// if no locking takes place.</param> /// <param name="expirationJob">A scheduled job that invokes the /// <see cref="TransferHandlerBase{TFile,TToken,TTransfer}.OnTransferExpiration"/> /// method once the transfer expires. May be null if the token does not expire.</param> /// <returns>A transfer object which encapsulates the information required to perform /// the transfer.</returns> protected abstract TTransfer CreateTransfer(string submittedResourceFilePath, TFile fileItem, TFolder parentFolder, UploadToken token, FileClaims claims, ResourceLockGuard lockGuard, Job <UploadToken> expirationJob);
/////////////// Application (former Cluster) Messaging protected internal abstract void PersistUploadToken(UploadToken value);
/// <summary> /// Initializes a new instance of the <see cref="T:System.Object"/> class. /// </summary> public ZipUploadTransfer(UploadToken token, ZipFileItem fileItem) : base(token, fileItem) { }