/// <summary>
/// Uploads the actual contents of a file to the UFS.
/// The <see cref="UFSClient"/> should be logged on before this point, and the previous request to upload a file must have completed successfully.
/// Results are returned in a <see cref="UploadFileFinishedCallback"/>.
/// </summary>
/// <param name="details">The details to use for uploading the file.</param>
/// <returns>The Job ID of the request. This can be used to find the appropriate <see cref="UploadFileFinishedCallback"/>.</returns>
public void UploadFile(UploadDetails details)
{
const uint MaxBytesPerChunk = 10240;
byte[] compressedData = ZipUtil.Compress(details.FileData);
byte[] fileHash = CryptoHelper.SHAHash(details.FileData);
var buffer = new byte[MaxBytesPerChunk];
using (var ms = new MemoryStream(compressedData))
{
for (long readIndex = 0; readIndex < ms.Length; readIndex += buffer.Length)
{
var msg = new ClientMsgProtobuf <CMsgClientUFSFileChunk>(EMsg.ClientUFSUploadFileChunk);
msg.TargetJobID = details.RemoteJobID;
var bytesRead = ms.Read(buffer, 0, buffer.Length);
if (bytesRead < buffer.Length)
{
msg.Body.data = buffer.Take(bytesRead).ToArray();
}
else
{
msg.Body.data = buffer;
}
msg.Body.file_start = ( uint )readIndex;
msg.Body.sha_file = fileHash;
Send(msg);
}
}
}
/// <summary>
/// Begins a request to upload a file to the UFS.
/// The <see cref="UFSClient"/> should be logged on before this point.
/// Results are returned in a <see cref="UploadFileResponseCallback"/>.
/// </summary>
/// <param name="details">The details to use for uploading the file.</param>
/// <returns>The Job ID of the request. This can be used to find the appropriate <see cref="UploadFileResponseCallback"/>.</returns>
public JobID RequestFileUpload(UploadDetails details)
{
if (details == null)
{
throw new ArgumentNullException(nameof(details));
}
byte[] compressedData = ZipUtil.Compress(details.FileData);
var msg = new ClientMsgProtobuf <CMsgClientUFSUploadFileRequest>(EMsg.ClientUFSUploadFileRequest);
msg.SourceJobID = steamClient.GetNextJobID();
msg.Body.app_id = details.AppID;
msg.Body.can_encrypt = false;
msg.Body.file_name = details.FileName;
msg.Body.file_size = ( uint )compressedData.Length;
msg.Body.raw_file_size = ( uint )details.FileData.Length;
msg.Body.sha_file = CryptoHelper.SHAHash(details.FileData);
msg.Body.time_stamp = DateUtils.DateTimeToUnixTime(DateTime.UtcNow);
Send(msg);
return(msg.SourceJobID);
}
private async Task UploadItem(UploadDetails details)
{
if (!await ShouldUpload(details))
{
return;
}
try
{
var filePath = FileManager.GetFilePath(_config, details.DataId);
Console.WriteLine($"Uploading on {Environment.CurrentManagedThreadId}");
var url = await UploadFileCore(GetId(details.DataId), filePath);
Console.WriteLine($"Updating Realm on {Environment.CurrentManagedThreadId}");
var success = await _runner.Execute((realm) =>
{
using (var transaction = realm.BeginWrite())
{
var data = realm.Find <FileData>(details.DataId);
if (data == null)
{
transaction.Rollback();
return(false);
}
data.Url = url;
data.Status = DataStatus.Remote;
transaction.Commit();
return(true);
}
});
if (success)
{
OnFileUploaded?.Invoke(this, new FileUploadedEventArgs
{
FileDataId = details.DataId,
FilePath = filePath,
RealmConfig = _config,
});
}
else
{
Logger.Error($"Could not find data with Id: {details.DataId}");
await DeleteFileCore(details.DataId);
}
}
catch (Exception ex)
{
Logger.Error(ex.Message);
_ = Task.Delay(details.RetryAfter).ContinueWith(_ =>
{
EnqueueUpload(details.DataId, Math.Min(details.RetryAfter * 2, MaxRetryDelay));
});
}
}
public async Task <Unit> Handle(UpdateUploadDetailsFromMetadataCommand request, CancellationToken cancellationToken)
{
UploadDetails uploadDetails = await sender.Send(new GetUploadDetailsQuery( request.UploadId ));
foreach (FileDetails fileDetails in uploadDetails.Files)
{
await sender.Send(new UpdateFileDetailsFromMetadataCommand( fileDetails.FileId ));
}
return(Unit.Value);
}
//====== actions
public async Task <IActionResult> OnGet(int uploadId, [FromServices] IMediator mediator)
{
try
{
var cmd = new GetUploadDetailsQuery(new(uploadId) );
UploadDetails = await mediator.Send(cmd);
}
catch
{
// todo: should display some message
return(NotFound());
}
return(Page());
}
private async Task <bool> ShouldUpload(UploadDetails details)
{
var filePath = FileManager.GetFilePath(_config, details.DataId);
if (!File.Exists(filePath))
{
return(false);
}
return(await _runner.Execute(realm =>
{
var data = realm.Find <FileData>(details.DataId);
if (data == null)
{
realm.Refresh();
data = realm.Find <FileData>(details.DataId);
}
return data != null && data.Status == DataStatus.Local;
}));
}
public async Task <IActionResult> OnPostAsync([FromServices] IMediator mediator)
{
if (Files?.Count > 0)
{
SourceFile[] uploads = Files
.Select(x => new SourceFile(x.OpenReadStream(), new MimeType(x.ContentType), x.FileName))
.ToArray();
// TODO: should we dispose streams after upload?
var parameters = new UploadParameters(uploads, new UploadDescription(Description ?? string.Empty));
UploadId uploadId = await mediator.Send(new UploadFilesCommand( parameters ));
try
{
await mediator.Send(new UpdateUploadDetailsFromMetadataCommand( uploadId ));
}
catch
{
// TODO: error logging
}
UploadDetails result = await mediator.Send(new GetUploadDetailsQuery( uploadId ));
if (result.Files.Count == 1 && result.RejectedDuplicates.Count == 0)
{
TempData["OperationResultMessage"] = $"File uploaded successfully.";
return(RedirectToPage("View", new { fileId = result.Files[0].FileId.Value }));
}
TempData["OperationResultMessage"] = $"{result.Files.Count} files uploaded successfully.";
return(RedirectToPage("UploadResult", new { uploadId = result.Id.Value }));
}
ModelState.AddModelError(string.Empty, "Select at least one file.");
return(Page());
}
/// <summary>
/// Uploads the actual contents of a file to the UFS.
/// The <see cref="UFSClient"/> should be logged on before this point, and the previous request to upload a file must have completed successfully.
/// Results are returned in a <see cref="UploadFileFinishedCallback"/>.
/// </summary>
/// <param name="details">The details to use for uploading the file.</param>
/// <returns>The Job ID of the request. This can be used to find the appropriate <see cref="UploadFileFinishedCallback"/>.</returns>
public void UploadFile( UploadDetails details )
{
const uint MaxBytesPerChunk = 10240;
byte[] compressedData = ZipUtil.Compress( details.FileData );
byte[] fileHash = CryptoHelper.SHAHash( details.FileData );
var buffer = new byte[ MaxBytesPerChunk ];
using ( var ms = new MemoryStream( compressedData ) )
{
for ( long readIndex = 0; readIndex < ms.Length; readIndex += buffer.Length )
{
var msg = new ClientMsgProtobuf<CMsgClientUFSFileChunk>( EMsg.ClientUFSUploadFileChunk );
msg.TargetJobID = details.RemoteJobID;
var bytesRead = ms.Read( buffer, 0, buffer.Length );
if ( bytesRead < buffer.Length )
{
msg.Body.data = buffer.Take( bytesRead ).ToArray();
}
else
{
msg.Body.data = buffer;
}
msg.Body.file_start = ( uint )readIndex;
msg.Body.sha_file = fileHash;
Send( msg );
}
}
}
/// <summary>
/// Begins a request to upload a file to the UFS.
/// The <see cref="UFSClient"/> should be logged on before this point.
/// Results are returned in a <see cref="UploadFileResponseCallback"/>.
/// </summary>
/// <param name="details">The details to use for uploading the file.</param>
/// <returns>The Job ID of the request. This can be used to find the appropriate <see cref="UploadFileResponseCallback"/>.</returns>
public JobID RequestFileUpload( UploadDetails details )
{
byte[] compressedData = ZipUtil.Compress( details.FileData );
var msg = new ClientMsgProtobuf<CMsgClientUFSUploadFileRequest>( EMsg.ClientUFSUploadFileRequest );
msg.SourceJobID = steamClient.GetNextJobID();
msg.Body.app_id = details.AppID;
msg.Body.can_encrypt = false;
msg.Body.file_name = details.FileName;
msg.Body.file_size = ( uint )compressedData.Length;
msg.Body.raw_file_size = ( uint )details.FileData.Length;
msg.Body.sha_file = CryptoHelper.SHAHash( details.FileData );
msg.Body.time_stamp = DateUtils.DateTimeToUnixTime( DateTime.UtcNow );
Send( msg );
return msg.SourceJobID;
}