public async Task <ApiResponse <PagedResult <JobsListItem> > > GetJobs(JobsGetListRequest request)
{
using (var scope = GlobalServices.Container.BeginLifetimeScope())
{
var repository = scope.Resolve <ICloudSpeedRepository>();
var paramMap = request.ToParamMapDTO();
var total = await repository.CountFileJobs(paramMap);
var entities = await repository.GetFileJobs(paramMap, request.Skip, request.Limit);
var result = entities.Select(entity => new JobsListItem
{
Id = entity.Id,
Cid = entity.Cid,
JobId = entity.JobId,
Status = entity.Status.ToString(),
Error = entity.Error,
Created = entity.Created,
Updated = entity.Updated,
}).ToList().AsEnumerable();
foreach (var item in result)
{
var fileCid = await repository.GetFileCidsByCid(item.Cid);
var fileName = await repository.GetFileName(fileCid.Id);
var path = _uploadSetting.GetStoragePath(fileCid.Id);
var mimeType = fileName.GetMimeType();
item.FileName = fileName;
item.Format = mimeType;
item.FileSize = new FileInfo(path).ToFileSize();
}
return(ApiResponse.Ok(request.ToPagedResult(result, total)));
}
}
public async Task Maintain1(CancellationToken stoppingToken)
{
using (var scope = GlobalServices.Container.BeginLifetimeScope())
{
var repository = scope.Resolve <ICloudSpeedRepository>();
var skip = 0;
var limit = 10;
while (true)
{
var fileNames = await repository.GetFileNames(skip, limit);
if (!fileNames.Any())
{
break;
}
_logger.LogInformation(string.Format("{0} files will be maintain", fileNames.Count()));
foreach (var fn in fileNames)
{
if (fn.Size == 0)
{
var path = _uploadSetting.GetStoragePath(fn.Id);
var fileInfo = new FileInfo(path);
await repository.UpdateFileName(fn.Id, fileInfo.Exists?fileInfo.Length : 0, fn.Name.GetMimeType());
}
}
await repository.Commit();
skip += limit;
}
_logger.LogInformation("Maintain1 Done");
}
}
public async Task <ApiResponse <PagedResult <UploadsListItem> > > GetUploads(UploadsGetListRequest request)
{
using (var scope = GlobalServices.Container.BeginLifetimeScope())
{
var repository = scope.Resolve <ICloudSpeedRepository>();
var paramMap = await request.ToParamMapDTO();
var total = await repository.CountUploadLogs(paramMap);
var entities = await repository.GetUploadLogs(paramMap, request.Skip, request.Limit);
var uploadLogs = entities.ToList();
var result = new List <UploadsListItem>();
foreach (var uploadLog in uploadLogs)
{
var item = new UploadsListItem
{
Id = uploadLog.Id,
Created = uploadLog.Created,
};
var fileCid = await repository.GetFileCid(uploadLog.DataKey);
var fileName = await repository.GetFileName(fileCid.Id);
var path = _uploadSetting.GetStoragePath(fileCid.Id);
var member = await _memberManager.FindByIdAsync(uploadLog.UserId);
var mimeType = fileName.GetMimeType();
item.Cid = fileCid.Cid;
item.UserName = member?.UserName;
item.FileName = fileName;
item.Format = mimeType;
item.FileSize = new FileInfo(path).ToFileSize();
result.Add(item);
}
return(ApiResponse.Ok(request.ToPagedResult(result, total)));
}
}
public async Task <ApiResponse <CloudSpeedItem> > GetUploadLog(string id)
{
using (var scope = GlobalServices.Container.BeginLifetimeScope())
{
var repository = scope.Resolve <ICloudSpeedRepository>();
var uploadLog = await repository.GetUploadLog(id);
if (uploadLog == null)
{
return(ApiResponse.NotFound <CloudSpeedItem>("item not found"));
}
var fileName = await repository.GetFileName(uploadLog.DataKey);
var item = new CloudSpeedItem
{
Id = id,
Description = uploadLog.Description,
AlipayKey = uploadLog.AlipayFileKey,
WxpayKey = uploadLog.WxpayFileKey,
Created = uploadLog.Created,
FileName = fileName,
};
var fileFullPath = _uploadSetting.GetStoragePath(uploadLog.DataKey);
if (File.Exists(fileFullPath))
{
item.FileSize = new FileInfo(fileFullPath).Length;
item.MimeType = fileName.GetMimeType();
}
var fileCid = await repository.GetFileCid(uploadLog.DataKey);
if (fileCid != null && fileCid.Status == FileCidStatus.Success)
{
item.DataCid = fileCid.Cid;
}
if (!string.IsNullOrEmpty(uploadLog.HashedPassword))
{
item.Secret = true;
}
return(ApiResponse.Ok(item));
}
}
public async Task <DealResult> OnlineExecute(string fileDealId, string fileCidId, ClientStartDealRequest dealRequest)
{
var path = _uploadSetting.GetStoragePath(fileCidId);
if (!File.Exists(path))
{
_logger.LogError("data not found " + path);
return(new DealResult {
Success = false
});
}
var fileCid = await _cloudSpeedManager.GetFileCid(fileCidId);
string cid = fileCid.Cid;
if (string.IsNullOrEmpty(cid))
{
_logger.LogError("data cid is required");
return(new DealResult {
Success = false
});
}
var fileSize = new FileInfo(path).Length;
var lotusClient = GlobalServices.ServiceProvider.GetService <LotusClient>();
long dealSize = fileCid.DealSize;
if (dealSize == 0)
{
_logger.LogInformation("client deal size ...");
var dealSizeResult = await lotusClient.ClientDealSize(new Cid { Value = cid });
if (dealSizeResult.Success)
{
dealSize = dealSizeResult.Result.PieceSize;
_logger.LogInformation(string.Format("client deal size ...{0}", dealSize));
await _cloudSpeedManager.UpdateFileCidDealSize(fileCidId, dealSize, dealSizeResult.Result.PayloadSize);
}
else
{
_logger.LogError("client deal size failed: " + dealSizeResult.Error?.Message);
return(new DealResult {
Success = false
});
}
}
dealRequest.CalPriceByDealSize(dealSize);
var dealParams = await CreateTTGraphsyncClientStartDealParams(lotusClient, dealRequest);
if (dealParams == null)
{
_logger.LogError(string.Format("CreateTTGraphsyncClientStartDealParams empty."));
return(new DealResult {
Success = false
});
}
_logger.LogInformation("lotus client start dealing: {dataCid} (datacid) {miner} {price} {duration}", cid, dealRequest.Miner, dealRequest.Price, dealRequest.Duration);
var dealCid = await lotusClient.ClientStartDeal(dealParams);
if (dealCid.Success)
{
var did = dealCid.Result.Value;
_logger.LogInformation("lotus client start deal result: {datacid}(datacid) - {dealcid} (dealcid) {price} {duration}", cid, did, dealRequest.Price, dealRequest.Duration);
await _cloudSpeedManager.UpdateFileDeal(fileDealId, dealRequest.Miner, did, FileDealStatus.Processing);
return(new DealResult {
Success = true, DealCid = did, DealSize = fileSize, DealDate = DateTime.Now
});
}
else
{
await _cloudSpeedManager.UpdateFileDeal(fileDealId, FileDealStatus.Failed, dealCid.Error.Message);
_logger.LogError("lotus client start deal failed: {datacid}(datacid) - (errorMessage)", cid, dealCid.Error?.Message);
return(new DealResult {
Success = false
});
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("PowergateWorker running at: {time}", DateTimeOffset.Now);
var powergateClient = GlobalServices.ServiceProvider.GetService <PowergateClient>();
try
{
var ffsInfo = await powergateClient.Ffs.InfoAsync(new Ffs.Rpc.InfoRequest(), powergateClient.BotXFfsToken);
_logger.LogInformation("Ffs info: {info}", Newtonsoft.Json.JsonConvert.SerializeObject(ffsInfo.Info));
}
catch (System.Exception ex)
{
_logger.LogError(0, ex, "powergate ffs info fail:" + ex.ToString());
}
try
{
while (!stoppingToken.IsCancellationRequested)
{
{
int limit = 10;
int skip = 0;
while (true)
{
var fileCids = await _cloudSpeedManager.GetFileCids(FileCidStatus.None, skip, limit);
if (fileCids.Count() == 0)
{
break;
}
foreach (var fileCid in fileCids)
{
var path = _uploadSetting.GetStoragePath(fileCid.Id);
if (File.Exists(path))
{
var cid = string.Empty;
try
{
using (var stage = powergateClient.Ffs.Stage(powergateClient.BotXFfsToken, cancellationToken: stoppingToken))
{
using (var fsRead = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, UploadConstants.BigFileWriteSize, true))
{
var buffer = new byte[UploadConstants.BigFileWriteSize];
var readCount = 0;
while ((readCount = await fsRead.ReadAsync(buffer, 0, buffer.Length, cancellationToken: stoppingToken)) > 0)
{
await stage.RequestStream.WriteAsync(new Ffs.Rpc.StageRequest()
{
Chunk = ByteString.CopyFrom(buffer)
});
}
}
await stage.RequestStream.CompleteAsync();
var response = await stage.ResponseAsync;
if (response != null)
{
cid = response.Cid;
await _cloudSpeedManager.UpdateFileCid(fileCid.Id, cid, FileCidStatus.Success);
_logger.LogInformation("Ffs stage: {cid}", cid);
}
}
}
catch (System.Exception ex)
{
_logger.LogError(0, ex, "powergate ffs stage fail:" + ex.ToString());
}
if (!string.IsNullOrEmpty(cid))
{
await PushStorageConfig(powergateClient, cid, stoppingToken);
}
}
}
skip += limit;
}
}
{
int limit = 10;
int skip = 0;
while (true)
{
var fileJobs = await _cloudSpeedManager.GetFileJobs(FileJobStatus.Processing, skip, limit);
if (fileJobs.Count() == 0)
{
break;
}
foreach (var fileJob in fileJobs)
{
if (fileJob.Updated.AddMinutes(1) > DateTime.Now)
{
continue;//Too NEW
}
try
{
var storageJob = await powergateClient.Ffs.GetStorageJobAsync(new Ffs.Rpc.GetStorageJobRequest {
Jid = fileJob.JobId
},
powergateClient.BotXFfsToken, cancellationToken : stoppingToken);
if (storageJob == null || storageJob.Job == null)
{
continue;
}
if (storageJob.Job.Status == Ffs.Rpc.JobStatus.Executing || storageJob.Job.Status == Ffs.Rpc.JobStatus.Queued)
{
continue;
}
if (storageJob.Job.Status == Ffs.Rpc.JobStatus.Canceled)
{
_logger.LogWarning("Ffs job canceled: {jobId} {status}", fileJob.JobId, storageJob.Job.ErrCause);
await _cloudSpeedManager.UpdateFileJob(fileJob.Id, FileJobStatus.Canceled);
continue;
}
if (storageJob.Job.Status == Ffs.Rpc.JobStatus.Failed)
{
_logger.LogWarning("Ffs job failed: {jobId} {status}", fileJob.JobId, storageJob.Job.ErrCause);
await _cloudSpeedManager.UpdateFileJob(fileJob.Id, FileJobStatus.Failed, storageJob.Job.ErrCause);
if (!string.IsNullOrEmpty(storageJob.Job.ErrCause) && storageJob.Job.ErrCause.Contains("all proposals were rejected"))
{
await PushStorageConfig(powergateClient, fileJob.Cid, stoppingToken, true);
}
}
else if (storageJob.Job.Status == Ffs.Rpc.JobStatus.Success)
{
await _cloudSpeedManager.UpdateFileJob(fileJob.Id, FileJobStatus.Success);
}
else
{
_logger.LogWarning("Ffs job unknown status: {jobId} {status} {cause}", fileJob.JobId, storageJob.Job.Status, storageJob.Job.ErrCause);
}
}
catch (Grpc.Core.RpcException ex)
{
_logger.LogError(0, "powergate ffs job fail:" + ex.Status.Detail);
if (ex.Status.Detail.Contains("not found"))
{
await _cloudSpeedManager.UpdateFileJob(fileJob.Id, FileJobStatus.Failed, ex.Status.Detail);
}
}
catch (System.Exception ex)
{
_logger.LogError(0, ex, "powergate ffs job fail:" + ex.ToString());
}
}
skip += limit;
}
}
await Task.Delay(1000, stoppingToken);
}
}
catch (System.Exception ex)
{
_logger.LogError(0, ex, "PowergateWorker error:" + ex.ToString());
}
}
public async Task Upload(string source, CancellationToken stoppingToken)
{
var sourceInfo = new FileInfo(source);
if (!sourceInfo.Exists)
{
return;
}
var sourceSize = sourceInfo.Length;
if (_uploadSetting.MaxFileSize > 0 && sourceSize > _uploadSetting.MaxFileSize)
{
_logger.LogInformation("file too large {source}", source);
}
else if (_uploadSetting.MinFileSize > 0 && sourceSize < _uploadSetting.MinFileSize)
{
_logger.LogInformation("file too small {source}", source);
}
else
{
var md5 = _cloudSpeedManager.GetMD5HashFromFile(source);
if (string.IsNullOrEmpty(md5))
{
_logger.LogInformation("file exists {source}", source);
return;
}
var hasFileMd5 = await _cloudSpeedManager.CheckFileMd5ById(md5);
if (!hasFileMd5)
{
var dataKey = SequentialGuid.NewGuidString();
try
{
var editSetting = _configuration.GetSection("EditSetting").Get <IDictionary <string, string> >();
var target = _uploadSetting.GetStoragePath(dataKey);
var format = sourceInfo.Name.GetMimeType();
if (!editSetting.ContainsKey(format))
{
_logger.LogError(0, "file format not found at edit setting {format}", format);
return;
}
var userName = editSetting[format].ToLower();
var member = await _memberManager.FindByNameAsync(userName);
if (member == null)
{
_logger.LogError(0, "member not found {userName}", userName);
return;
}
File.Copy(source, target);
await _cloudSpeedManager.CreateFileMd5(md5, dataKey);
await _cloudSpeedManager.CreateFileName(dataKey, sourceInfo.Name, sourceSize);
await _cloudSpeedManager.CreateUploadLog(member.Id, new PanePostRequest { DataKey = dataKey });
_logger.LogInformation("file upload successfully {dataKey} - {source}", dataKey, source);
}
catch (System.Exception ex)
{
_logger.LogError(0, ex, "file upload failed {dataKey} - {source}", dataKey, source);
}
}
else
{
_logger.LogInformation("file exists {source}", source);
}
}
}
private string GetBigStoragePath(string key)
{
return(_uploadSetting.GetStoragePath(key));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("LotusWorker running at: {time}", DateTimeOffset.Now);
var _1 = CreateSentryBox1(stoppingToken);
var _2 = CreateSentryBox2(stoppingToken);
try
{
var lotusClient = GlobalServices.ServiceProvider.GetService <LotusClient>();
foreach (var miner in lotusClient.MinerClients)
{
var minerInfo = await lotusClient.StateMinerInfo(new StateMinerInfoRequest { Miner = miner.Key });
if (minerInfo.Success)
{
_logger.LogInformation("ClientQueryAsk for {miner}", miner.Key);
var ask = await lotusClient.ClientQueryAsk(new ClientQueryAskRequest
{
PeerId = minerInfo.Result.PeerId,
Miner = miner.Key
});
if (ask.Success)
{
_logger.LogInformation("ClientQueryAsk for {miner}: {ask}", miner.Key, JsonConvert.SerializeObject(ask));
}
}
}
while (!stoppingToken.IsCancellationRequested)
{
{
int limit = 10;
int skip = 0;
while (true)
{
var fileCids = await _cloudSpeedManager.GetFileCids(FileCidStatus.None, skip, limit);
if (fileCids.Count() == 0)
{
_logger.LogDebug(0, "file cid empty with status none");
break;
}
foreach (var fileCid in fileCids)
{
var path = _uploadSetting.GetStoragePath(fileCid.Id);
if (File.Exists(path))
{
var fileSize = new FileInfo(path).Length;
if (fileSize >= _uploadSetting.MaxFileSize && _uploadSetting.MaxFileSize > 0)
{
var error = $"file size should be less that {fileSize} < {_uploadSetting.MaxFileSize} ";
await _cloudSpeedManager.UpdateFileCid(fileCid.Id, FileCidStatus.Failed, error);
_logger.LogWarning("Lotus ClientImport fail : {0} {1}", error, path);
continue;
}
if (fileSize <= _uploadSetting.MinFileSize)
{
var error = $"file size should be more that {fileSize} < {_uploadSetting.MinFileSize} ";
await _cloudSpeedManager.UpdateFileCid(fileCid.Id, FileCidStatus.Failed, error);
_logger.LogWarning("Lotus ClientImport fail : {0} {1}", error, path);
continue;
}
var cid = string.Empty;
try
{
_logger.LogInformation("Lotus ClientImport: {cid}", cid);
var result = await lotusClient.ClientImport(new FileRef
{
Path = path,
IsCAR = false
});
if (result.Success)
{
cid = result.Result.Root.Value;
_logger.LogInformation("Lotus ClientImport: {cid}", cid);
await _cloudSpeedManager.UpdateFileCid(fileCid.Id, cid, FileCidStatus.Success);
_logger.LogWarning(0, string.Format("client deal size ..."));
var dealSize = await lotusClient.ClientDealSize(new Cid { Value = cid });
if (dealSize.Success)
{
_logger.LogWarning(0, string.Format("client deal size ...{0}", dealSize.Result.PieceSize));
await _cloudSpeedManager.UpdateFileCidDealSize(fileCid.Id, dealSize.Result.PieceSize, dealSize.Result.PayloadSize);
}
await _cloudSpeedManager.CreateFileDeal(cid);
}
}
catch (Exception ex)
{
_logger.LogError(0, ex, "Lotus ClientImport fail:" + ex.ToString());
}
}
}
skip += limit;
}
}
await Task.Delay(5000, stoppingToken);
}
}
catch (Exception ex)
{
_logger.LogError(0, ex, "LotusWorker error:" + ex.ToString());
}
}