protected virtual void RequestFailure(StorageRequest request, int code, string message)
{
request.Result = false;
request.StatusCode = code;
request.Message = message;
request.HandledBy = this.Name;
}
public async Task <List <string> > UploadFile(List <IFormFile> files)
{
List <string> responses = new List <string>();
foreach (var formFile in files)
{
DateTime timeStamp = DateTime.Now;
if (formFile.Length > 0)
{
var FileName = Path.GetFileNameWithoutExtension(formFile.FileName) + '_' + timeStamp.ToString("yyyyMMddHHmmssffff") + Path.GetExtension(formFile.FileName);
StorageRequest storageRequest = new StorageRequest()
{
FolderName = "recursos/",
FileName = FileName,
File = formFile
};
StorageResponse storageResponse = await StorageService.UploadObject(storageRequest);
responses.Add(storageResponse.FileName);
}
}
return(responses);
}
protected virtual void RequestSeuccess(StorageRequest request)
{
request.Result = true;
request.StatusCode = 200;
request.Message = "OK";
request.HandledBy = this.Name;
}
/// <summary>
/// Authenticates the user for the storage access role(s)
/// </summary>
/// <code>
///
///IEnumerator AuthenticateAsync()
///{
/// Terminal.LogImportant("AuthenticateAsync");
/// /// get task
/// var result1 = Repository.Authenticate(AuthKey, Roles);
/// /// wait for it
/// yield return StartCoroutine(result1.WaitRoutine());
/// /// client error
/// result1.ThrowIfFaulted();
/// /// server error
/// if (result1.Result.hasError)
/// throw new Exception(result1.Result.error.message);
/// Terminal.LogSuccess("Authenticated");
///}
/// </code>
/// <returns></returns>
public UnityTask <StorageResponse <bool> > Authenticate(string authToken, string[] roles, int timeout = 1800)
{
AuthenticationToken = authToken;
if (roles == null || roles.Length == 0)
{
return(new UnityTask <StorageResponse <bool> >(new Exception("roles is required")));
}
if (string.IsNullOrEmpty(AuthenticationToken))
{
return(new UnityTask <StorageResponse <bool> >(new Exception("authToken is required")));
}
var body = new StorageRequest();
body.applicationKey = ApplicationKey;
body.privateKey = PrivateKey;
body.authenticationToken = AuthenticationToken;
body.timeout = timeout;
body.roles = roles;
//body.policies = policies;
var json = JsonMapper.ToJson(body);
return(Post <bool>("authenticate", json));
}
public async Task <IActionResult> AddValue([FromBody] StorageRequest request)
{
var redisValue = new RedisValue($"{request.Content}_{Guid.NewGuid()}");
var isSuccess = await _redis.GetDatabase()?.SetAddAsync(RedisSetKey, redisValue);
if (isSuccess)
{
return(Ok());
}
return(Problem());
}
/// <summary>
/// Retrieves a paginated list of the names of all the roles created by the user�s application.
/// </summary>
/// <returns></returns>
public UnityTask <StorageResponse <string[]> > ListRoles()
{
var body = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey
};
var json = JsonMapper.ToJson(body);
return(Post <string[]>("listRoles", json));
}
public void GivenAnEmailRequest_WhenNotValidBase64Data_ReturnsIsValidFalse()
{
var sut = new StorageRequest
{
Base64Data = "-- InValid Base 64 Data --",
ContentType = "Application/Json",
ContainerName = "Container Name"
};
Assert.IsFalse(sut.IsValid());
}
public override void Handle(StorageRequest request)
{
if (request.Type == RequestType.Check)
{
this.RequestSeuccess(request);
}
else
{
base.Handle(request);
}
}
public async Task <StorageResponseDTO> Store(StorageRequest request)
{
CloudStorageAccount storageAccount = GetStorageAccount();
CloudBlobContainer container = await GetContainer(storageAccount, request.ContainerName);
CloudBlockBlob blob = CreateBlob(container, request.ContentType);
byte[] dataToUpload = CreateDataToUpload(request);
return(await UploadToStorage(blob, dataToUpload));
}
/// <summary>
/// Retrieves the policies that compose the role.
/// </summary>
/// <returns></returns>
public UnityTask <StorageResponse <Role> > GetRole(string role)
{
var body = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey,
role = role
};
var json = JsonMapper.ToJson(body);
return(Post <Role>("getRole", json));
}
/// <summary>
/// Deletes a table
/// </summary>
/// <param name="table"></param>
/// <returns></returns>
public UnityTask <StorageResponse <TableMetadata> > DeleteTable(string table)
{
var request = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey,
authenticationToken = AuthenticationToken,
table = table,
};
var dto = JsonMapper.ToJson(request);
return(Post <TableMetadata>("deleteTable", dto));
}
public string Handle(object request, RequestType type)
{
var currentRequest = new StorageRequest(request, type);
foreach (var handler in this.Chain)
{
handler.Handle(currentRequest);
if (currentRequest.Result)
{
break;
}
}
return(currentRequest.ToString());
}
/// <summary>
/// Returns a listing of tables
/// </summary>
/// <param name="limit"></param>
/// <param name="startTable"></param>
/// <returns></returns>
public UnityTask <StorageResponse <TableList> > ListTables(int limit = 0, string startTable = null)
{
var request = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey,
authenticationToken = AuthenticationToken,
startTable = startTable,
limit = limit,
};
var dto = JsonMapper.ToJson(request);
return(Post <TableList>("listTables", dto));
}
public async Task <StorageResponseDTO> Execute(StorageRequestDTO requestDTO)
{
var request = new StorageRequest
{
Base64Data = requestDTO.Base64Data,
ContentType = requestDTO.ContentType,
ContainerName = requestDTO.ContainerName
};
if (!request.IsValid())
{
throw new BadRequestException();
}
return(await _mediator.Send(request));
}
public void GivenAnEmailRequest_WhenValid_ReturnsIsValidTrue()
{
string base64Decoded = "base64 encoded string";
string base64Encoded;
byte[] data = System.Text.ASCIIEncoding.ASCII.GetBytes(base64Decoded);
base64Encoded = System.Convert.ToBase64String(data);
var sut = new StorageRequest
{
Base64Data = base64Encoded,
ContentType = "Application/Json",
ContainerName = "Container Name"
};
Assert.IsTrue(sut.IsValid());
}
/// <summary>
/// 获取数据列表(分页)
/// </summary>
public PageModel GetDataListForPage(StorageRequest request)
{
#region 模糊搜索条件
var where = new Where <TbStorage>();
if (!string.IsNullOrWhiteSpace(request.StorageName))
{
where.And(d => d.StorageName.Like(request.StorageName));
}
if (!string.IsNullOrWhiteSpace(request.ProcessFactoryCode))
{
where.And(d => d.ProcessFactoryCode == request.ProcessFactoryCode);
}
#endregion
try
{
var data = Db.Context.From <TbStorage>()
.Select(
TbStorage._.ID
, TbStorage._.StorageName
, TbStorage._.ProcessFactoryCode
, TbStorage._.StorageAttribute
, TbStorage._.AreaCode
, TbStorage._.StorageAdd
, TbStorage._.Tel
, TbStorage._.InsertUserCode
, TbStorage._.InsertTime
, TbUser._.UserName.As("InsertUserName")
, TbCompany._.CompanyFullName.As("ProcessFactoryName")
, TbSysDictionaryData._.DictionaryText.As("StorageAttributeText"))
.AddSelect(Db.Context.From <TbUser>()
.Select(p => p.UserName)
.Where(TbUser._.UserCode == TbStorage._.UserCode), "UserName")
.LeftJoin <TbUser>((a, c) => a.InsertUserCode == c.UserCode)
.LeftJoin <TbSysDictionaryData>((a, c) => a.StorageAttribute == c.DictionaryCode)
.LeftJoin <TbCompany>((a, c) => a.ProcessFactoryCode == c.CompanyCode)
.Where(where)
.OrderByDescending(p => p.ID)
.ToPageList(request.rows, request.page);
return(data);
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Returns the item by Id
/// </summary>
/// <typeparam name="T">The type to fetch</typeparam>
/// <param name="id">the identities</param>
/// <returns></returns>
public UnityTask <StorageResponse <T> > Get <T>(DataKey id) where T : class
{
var metadata = GetMetadata <T>();
var request = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey,
authenticationToken = AuthenticationToken,
table = metadata.Table,
key = id,
};
var dto = JsonMapper.ToJson(request);
return(Post <T>("getItem", dto));
}
/// <summary>
/// Is the user authenticated ?
/// </summary>
/// <returns></returns>
public UnityTask <StorageResponse <bool> > IsAuthenticated()
{
if (string.IsNullOrEmpty(AuthenticationToken))
{
return(new UnityTask <StorageResponse <bool> >(new StorageResponse <bool>(false)));
}
var body = new StorageRequest
{
applicationKey = ApplicationKey,
authenticationToken = AuthenticationToken
};
var json = JsonMapper.ToJson(body);
return(Post <bool>("isAuthenticated", json));
}
public virtual void Handle(StorageRequest request)
{
try
{
switch (request.Type)
{
case RequestType.Add:
this.Add(request.Body);
this.RequestSeuccess(request);
break;
case RequestType.Check:
if (this.Check(request.Body))
{
this.RequestSeuccess(request);
}
else
{
this.RequestFailure(request, 404, "Not Found");
}
break;
case RequestType.Buy:
if (this.Remove(request.Body))
{
this.RequestSeuccess(request);
}
else
{
this.RequestFailure(request, 404, "Not Found");
}
break;
default:
break;
}
}
catch (ApplicationException ex)
{
this.RequestFailure(request, 403, ex.Message);
}
}
/// <summary>
/// Removes the item
/// </summary>
/// <typeparam name="T">The type to fetch</typeparam>
/// <param name="item">the item to delete</param>
/// <returns></returns>
public UnityTask <StorageResponse <T> > Delete <T>(T item) where T : class
{
var metadata = GetMetadata <T>();
var request = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey,
authenticationToken = AuthenticationToken,
table = metadata.Table,
key = metadata.GetKey(item),
item = item
};
var dto = JsonMapper.ToJson(request);
return(Post <T>("deleteItem", dto));
}
private async Task CloseAndUpload(AppRepository app, S3StorageRepository S3, Del message, StreamWriter jsonwrite, string file_path, string file_name)
{
if (jsonwrite != null)
{
jsonwrite.Close();
StorageRequest storage = new StorageRequest
{
Name = file_name,
Type = "import",
EncryptionKeyId = null,
Path = file_path,
};
message("Uploading " + file_name);
await S3.UploadAsync(app, storage);
}
}
/// <summary>
/// Puts a table's metadata
/// </summary>
/// <param name="table"></param>
/// <returns></returns>
public UnityTask <StorageResponse <TableMetadata> > UpdateTable(TableMetadata table)
{
var request = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey,
authenticationToken = AuthenticationToken,
table = table.name,
key = table.key,
provisionType = table.provisionType,
provisionLoad = table.provisionLoad,
throughput = table.throughput,
};
var dto = JsonMapper.ToJson(request);
return(Post <TableMetadata>("updateTable", dto));
}
public async Task <StorageResponse> UploadObject(StorageRequest storageRequest)
{
var client = new AmazonS3Client(S3Configuration.AccessKeyId, S3Configuration.AccessSecretKey, S3Configuration.Region);
byte[] fileBytes = new Byte[storageRequest.File.Length];
storageRequest.File.OpenReadStream().Read(fileBytes, 0, Int32.Parse(storageRequest.File.Length.ToString()));
var fileNameTemp = storageRequest.FileName ?? Guid.NewGuid() + storageRequest.File.FileName.Trim();
PutObjectResponse response = null;
using (var stream = new MemoryStream(fileBytes))
{
var request = new PutObjectRequest
{
BucketName = S3Configuration.Bucket,
Key = (storageRequest.FolderName ?? "") + fileNameTemp,
InputStream = stream,
ContentType = storageRequest.File.ContentType,
CannedACL = S3CannedACL.PublicRead
};
response = await client.PutObjectAsync(request);
}
if (response.HttpStatusCode == System.Net.HttpStatusCode.OK)
{
return(new StorageResponse
{
Success = true,
FileName = string.Format(S3Configuration.EndpointAmazon, S3Configuration.Bucket) + (storageRequest.FolderName ?? "") + fileNameTemp
});
}
else
{
return(new StorageResponse
{
Success = false,
FileName = storageRequest.FileName
});
}
}
/// <summary>
/// Decrements a numeric property
/// </summary>
/// <typeparam name="T">the item type</typeparam>
/// <param name="item">the item</param>
/// <param name="propertyName">the property</param>
/// <param name="change">the incremental change</param>
/// <returns></returns>
public UnityTask <StorageResponse <T> > Decr <T>(T item, string propertyName, int change = 1) where T : class
{
var metadata = GetMetadata <T>();
var request = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey,
authenticationToken = AuthenticationToken,
table = metadata.Table,
key = metadata.GetKey(item),
item = item,
property = propertyName,
value = change
};
var dto = JsonMapper.ToJson(request);
return(Post <T>("decr", dto));
}
/// <summary>
/// Returns a unordered item listing
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="query"></param>
/// <returns></returns>
public UnityTask <StorageResponse <ItemList <T> > > List <T>(ItemListRequest <T> query) where T : class
{
var metadata = GetMetadata <T>();
var request = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey,
authenticationToken = AuthenticationToken,
table = metadata.Table,
key = query._datakey,
properties = query._properties,
startKey = query._startKey,
limit = query._limit,
filter = query._filters
};
var dto = JsonMapper.ToJson(request);
return(Post <ItemList <T> >("listItems", dto));
}
public async Task Handle_CallsStoreWithRequest_ReturnsResult()
{
var mockProcessor = new Mock <IStorageProcessor>();
var sut = new StoreDataHandler(mockProcessor.Object);
var request = new StorageRequest
{
Base64Data = "basedata",
ContainerName = "containername",
ContentType = "application/json"
};
var response = await sut.Handle(request, It.IsAny <CancellationToken>());
Func <StorageRequest, bool> matches = r => r.Base64Data == request.Base64Data &&
r.ContentType == request.ContentType &&
r.ContainerName == request.ContainerName;
mockProcessor.Verify(p => p.Store(It.Is <StorageRequest>(r => matches(r))));
}
/// <summary>
/// Saves an existing object server side
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="entity"></param>
/// <param name="callback"></param>
/// <returns></returns>
public void Update <T>(T entity, Action <Response> callback) where T : class
{
if (!IsAuthenticated)
{
callback(new Response(new Exception("Not authenticated")));
return;
}
var meta = StorageMetadata.GetMetadata <T>();
var model = new StorageRequest
{
ObjectId = meta.GetId(entity),
ObjectScore = float.Parse(meta.GetScore(entity)),
ObjectType = meta.TableName,
ObjectData = JsonSerializer.Serialize(entity),
ModifiedOn = meta.GetModified(entity),
};
HttpPostAsync("Update", model, callback);
}
/// <summary>
/// Update the item
/// </summary>
/// <typeparam name="T">The type to fetch</typeparam>
/// <param name="item">the item to save</param>
/// <returns></returns>
public UnityTask <StorageResponse <T> > Update <T>(T item) where T : class
{
var metadata = GetMetadata <T>();
var itemDto = JsonMapper.ToObject(JsonMapper.ToJson(item));
RemoveFields(itemDto, metadata.HasSecondary ? new[] { metadata.Primary, metadata.Secondary } : new[] { metadata.Primary });
var request = new StorageRequest
{
applicationKey = ApplicationKey,
privateKey = PrivateKey,
authenticationToken = AuthenticationToken,
table = metadata.Table,
key = metadata.GetKey(item),
item = itemDto.ToJson()
};
var dto = JsonMapper.ToJson(request);
return(Post <T>("updateItem", dto));
}
private static byte[] CreateDataToUpload(StorageRequest request)
{
return(Convert.FromBase64String(request.Base64Data));
}
private async Task <bool> LoadDocuments(AppRepository app, Del message, Del threadMessage, Del uploadStatusMessage, Del uploadedMessage, Del skippedMessage, Del handleUploadCompleted)
{
app.LogMessage("FS: Getting documents remaining to be exported");
uploadStatusMessage("Getting documents remaining to be exported");
var fileList = new List <FileInfo>();
var folderList = new List <DirectoryInfo>();
foreach (string folder in app.Settings.FolderLists)
{
DirectoryInfo d = new DirectoryInfo(folder);
var r = d.GetFiles("*.*", SearchOption.AllDirectories);
fileList.AddRange(r);
uploadStatusMessage("Loading subfolders " + d.Name);
for (int i = 0; i < r.Count(); i++)
{
folderList.Add(d);
}
}
FileInfo[] Files = fileList.ToArray();
DirectoryInfo[] Dirs = folderList.ToArray();
var fileIndex = 0;
var completedThreads = 0;
var fileCounts = Files.Count();
var skipped = 0;
var uploaded = 0;
var errors = 0;
var documents = Database.GetCollection <Document>("documents");
documents.EnsureIndex(x => x.FilePath);
ThreadsNum = ThreadsNum < fileCounts ? ThreadsNum : fileCounts;
Thread[] threadsArray = new Thread[ThreadsNum];
for (int i = 0; i < ThreadsNum; i++)
{
threadsArray[i] = new Thread(async(object param) =>
{
int threadId = (int)param;
while (fileIndex < fileCounts)
{
if (IsUploadingStop)
{
Thread.Sleep(1000);
}
else
{
try
{
var activeIndex = fileIndex;
fileIndex++;
if (activeIndex < ThreadsNum)
{
activeIndex = threadId;
}
var file = Files[activeIndex];
Document document = new Document();
document.FilePath = file.FullName;
document.DateModified = file.LastWriteTimeUtc;
uploadStatusMessage(GetWorkingFolder(Dirs[activeIndex].ToString()));
var existing = documents.FindOne(x => x.FilePath == document.FilePath);
StorageRequest storage = new StorageRequest
{
Path = file.FullName,
Name = file.Name,
Type = makeCloudFolderPath(Dirs[activeIndex].ToString(), file.Name, file.FullName, app.Settings.IncludeFolderName),
EncryptionKeyId = null,
};
S3StorageRepository s3 = new S3StorageRepository(Database);
if (existing == null || existing.DateModified < document.DateModified)
{
var display = "";
var uploadResponse = false;
try
{
string statusofThread = MakeStatusOfThread(threadId, file.Length, file.Name);
threadMessage(statusofThread);
uploadResponse = await s3.UploadAsync(app, storage);
}
catch (Exception e)
{
app.LogMessage(file.FullName + " - " + e.Message);
errors++;
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " ERROR";
}
try
{
if (uploadResponse)
{
if (existing == null)
{
documents.Insert(document);
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " UPLOADED";
}
else
{
existing.DateModified = file.LastWriteTimeUtc;
documents.Update(existing);
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " UPDATED";
}
uploaded++;
string _uploadedMsg = "Completed " + uploaded.ToString() + " of " + fileCounts.ToString();
uploadedMessage(_uploadedMsg);
string _skippedMsg = "Skipped " + skipped.ToString() + " of " + fileCounts.ToString();
skippedMessage(_skippedMsg);
}
}
catch (Exception e)
{
app.LogMessage(file.FullName + " - Error during LiteDB - " + e.Message);
errors++;
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " ERROR";
}
message(display);
}
else
{
var display = activeIndex.ToString() + " of " + fileCounts.ToString() + " SKIPPED";
message(display);
skipped++;
string _uploadedMsg = "Completed " + uploaded.ToString() + " of " + fileCounts.ToString();
uploadedMessage(_uploadedMsg);
string _skippedMsg = "Skipped " + skipped.ToString() + " of " + fileCounts.ToString();
skippedMessage(_skippedMsg);
}
}
catch (Exception e)
{
app.LogMessage("Error during LiteDB - " + e.Message);
errors++;
}
}
}
if (threadId < threadsArray.Length)
{
completedThreads++;
message("FS: Thread " + threadId + " DONE");
string statusofThread = MakeStatusOfThread(threadId, 0, "Finished");
threadMessage(statusofThread);
if (completedThreads + 1 > ThreadsNum)
{
IsRunning = false;
message("FS: Skipped " + skipped.ToString());
message("FS: Uploaded " + uploaded.ToString());
message("FS: Done loading documents");
app.LogMessage("FS: Skipped " + skipped.ToString());
app.LogMessage("FS: Uploaded " + uploaded.ToString());
app.LogMessage("FS: Done loading documents");
handleUploadCompleted("completed");
if (uploaded > 0)
{
string endMessage = "{\"text\":\"End uploading\",\"blocks\":[{\"type\": \"section\",\"text\": {\"type\": \"mrkdwn\",\"text\": \"End uploading:\"}},{\"type\": \"section\",\"block_id\": \"section789\",\"fields\": [{\"type\": \"mrkdwn\",\"text\": \">*Bucket* : " + app.Settings.BucketName + "\n>*Uploaded* : " + uploaded.ToString() + "\n>*Skipped:* : " + skipped.ToString() + "\n*Errors:* : " + errors.ToString() + "\n>\"}]}]}";
await app.Slack.TryPostJsonAsync(endMessage);
}
}
}
});
threadsArray[i].Start(i);
}
return(true);
}
