public async Task <BinaryMetadata> GetBinaryMetadata(string key)
{
var binaryMetadata = _memoryCache.Get <BinaryMetadata>(key);
if (binaryMetadata != null)
{
return(binaryMetadata);
}
try
{
var metadataResponse = await _cephS3Client.GetObjectMetadataAsync(_filesBucketName, key);
var metadataWrapper = MetadataCollectionWrapper.For(metadataResponse.Metadata);
var filename = metadataWrapper.Read <string>(MetadataElement.Filename);
return(new BinaryMetadata(filename, metadataResponse.ContentLength));
}
catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
throw new ObjectNotFoundException($"Binary with the key '{key}' not found.");
}
catch (Exception ex)
{
throw new S3Exception(ex);
}
}
public async Task VerifySessionExpirationForBinary(string key)
{
var sessionId = key.AsSessionId();
var(_, _, expiresAt) = _memoryCache.Get <(SessionDescriptor, AuthorInfo, DateTime)>(sessionId);
if (expiresAt == default)
{
GetObjectMetadataResponse response;
try
{
response = await _cephS3Client.GetObjectMetadataAsync(_filesBucketName, sessionId.AsS3ObjectKey(Tokens.SessionPostfix));
}
catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
throw new ObjectNotFoundException($"Session '{sessionId}' does not exist");
}
catch (Exception ex)
{
throw new S3Exception(ex);
}
var metadataWrapper = MetadataCollectionWrapper.For(response.Metadata);
expiresAt = metadataWrapper.Read <DateTime>(MetadataElement.ExpiresAt);
}
if (SessionDescriptor.IsSessionExpired(expiresAt))
{
throw new SessionExpiredException(sessionId, expiresAt);
}
}
public void ShouldWorkForAlmostJson()
{
var wrapper = MetadataCollectionWrapper.For(new MetadataCollection());
const string Value = " { Вася Пупкин } ";
wrapper.Write(MetadataElement.AuthorName, Value);
var res = wrapper.Read <string>(MetadataElement.AuthorName);
Assert.Equal(Value, res);
}
public void ShouldWorkForNonUsAsciiSymbols()
{
var wrapper = MetadataCollectionWrapper.For(new MetadataCollection());
var value = "Вася Пупкин";
wrapper.Write(MetadataElement.AuthorName, value);
var res = wrapper.Read <string>(MetadataElement.AuthorName);
Assert.Equal(value, res);
}
public void ShouldWrapFilename()
{
var wrapper = MetadataCollectionWrapper.For(new MetadataCollection());
var value = "my_file.png";
wrapper.Write(MetadataElement.Filename, value);
var res = wrapper.Read <string>(MetadataElement.Filename);
Assert.Equal(value, res);
}
public void ShouldWorkForAlreadyEncodedValue()
{
var wrapper = MetadataCollectionWrapper.For(new MetadataCollection());
var value = "Вася Пупкин";
var encodedValue = Uri.EscapeDataString(value);
wrapper.Write(MetadataElement.AuthorName, "utf-8''" + encodedValue);
var res = wrapper.Read <string>(MetadataElement.AuthorName);
Assert.Equal(value, res);
}
public async Task <(SessionDescriptor SessionDescriptor, AuthorInfo AuthorInfo, DateTime ExpiresAt)> GetSessionDescriptor(Guid sessionId)
{
var result =
await _memoryCache.GetOrCreateAsync(
sessionId,
async entry =>
{
try
{
using (var objectResponse = await _cephS3Client.GetObjectAsync(_filesBucketName, sessionId.AsS3ObjectKey(Tokens.SessionPostfix)))
{
var metadataWrapper = MetadataCollectionWrapper.For(objectResponse.Metadata);
var expiresAt = metadataWrapper.Read <DateTime>(MetadataElement.ExpiresAt);
var author = metadataWrapper.Read <string>(MetadataElement.Author);
var authorLogin = metadataWrapper.Read <string>(MetadataElement.AuthorLogin);
var authorName = metadataWrapper.Read <string>(MetadataElement.AuthorName);
string json;
using (var reader = new StreamReader(objectResponse.ResponseStream, Encoding.UTF8))
{
json = reader.ReadToEnd();
}
var sessionDescriptor = JsonConvert.DeserializeObject <SessionDescriptor>(json, SerializerSettings.Default);
var tuple = (SessionDescriptor : sessionDescriptor, AuthorInfo : new AuthorInfo(author, authorLogin, authorName), ExpiresAt : expiresAt);
entry.SetValue(tuple)
.SetAbsoluteExpiration(expiresAt);
return(tuple);
}
}
catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
throw new ObjectNotFoundException($"Session '{sessionId}' does not exist");
}
catch (SessionExpiredException)
{
throw;
}
catch (Exception ex)
{
throw new S3Exception(ex);
}
});
if (SessionDescriptor.IsSessionExpired(result.ExpiresAt))
{
throw new SessionExpiredException(sessionId, result.ExpiresAt);
}
return(result);
}
public void ShouldWrapUsingInvariantCulture()
{
var wrapper = MetadataCollectionWrapper.For(new MetadataCollection());
CultureInfo.CurrentCulture = new CultureInfo("en-US");
var value = DateTime.UtcNow;
wrapper.Write(MetadataElement.ExpiresAt, value);
CultureInfo.CurrentCulture = new CultureInfo("ru-RU");
var res = wrapper.Read <DateTime>(MetadataElement.ExpiresAt);
Assert.Equal(value, res);
}
public async Task <IReadOnlyCollection <ObjectMetadataRecord> > GetObjectMetadatas(IReadOnlyCollection <long> ids)
{
var uniqueIds = new HashSet <long>(ids);
var partitioner = Partitioner.Create(uniqueIds);
var result = new ObjectMetadataRecord[uniqueIds.Count];
var tasks = partitioner
.GetOrderablePartitions(_degreeOfParallelism)
.Select(async x =>
{
while (x.MoveNext())
{
var id = x.Current.Value;
ObjectMetadataRecord record;
try
{
var objectVersions = await GetObjectLatestVersions(id);
var versionId = objectVersions.Where(v => v.Id.EndsWith(Tokens.ObjectPostfix))
.Select(v => v.VersionId)
.SingleOrDefault();
if (versionId == null)
{
record = null;
}
else
{
var response = await _s3Client.GetObjectMetadataAsync(_bucketName, id.AsS3ObjectKey(Tokens.ObjectPostfix), versionId);
var metadataWrapper = MetadataCollectionWrapper.For(response.Metadata);
var author = metadataWrapper.Read <string>(MetadataElement.Author);
var authorLogin = metadataWrapper.Read <string>(MetadataElement.AuthorLogin);
var authorName = metadataWrapper.Read <string>(MetadataElement.AuthorName);
record = new ObjectMetadataRecord(
id,
versionId,
response.LastModified,
new AuthorInfo(author, authorLogin, authorName));
}
}
catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
record = null;
}
result[x.Current.Key] = record;
}
});
await Task.WhenAll(tasks);
return(result.Where(x => x != null).ToList());
}
public async Task <TemplateDescriptor> GetTemplateDescriptor(long id, string versionId)
{
var objectVersionId = string.IsNullOrEmpty(versionId) ? await GetTemplateLatestVersion(id) : versionId;
var templateDescriptor = await _memoryCache.GetOrCreateAsync(
id.AsCacheEntryKey(objectVersionId),
async entry =>
{
try
{
using (var response = await _s3Client.GetObjectAsync(_bucketName, id.ToString(), objectVersionId))
{
var metadataWrapper = MetadataCollectionWrapper.For(response.Metadata);
var author = metadataWrapper.Read <string>(MetadataElement.Author);
var authorLogin = metadataWrapper.Read <string>(MetadataElement.AuthorLogin);
var authorName = metadataWrapper.Read <string>(MetadataElement.AuthorName);
string json;
using (var reader = new StreamReader(response.ResponseStream, Encoding.UTF8))
{
json = reader.ReadToEnd();
}
var descriptor = new TemplateDescriptor
{
Id = id,
VersionId = objectVersionId,
LastModified = response.LastModified,
Author = author,
AuthorLogin = authorLogin,
AuthorName = authorName
};
JsonConvert.PopulateObject(json, descriptor, SerializerSettings.Default);
entry.SetValue(descriptor)
.SetPriority(CacheItemPriority.NeverRemove);
return(descriptor);
}
}
catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
throw new ObjectNotFoundException($"Template '{id}' version '{objectVersionId}' not found");
}
});
return(templateDescriptor);
}
public async Task <IReadOnlyCollection <ObjectMetadataRecord> > GetTemplateMetadatas(IReadOnlyCollection <long> ids)
{
var uniqueIds = new HashSet <long>(ids);
var partitioner = Partitioner.Create(uniqueIds);
var result = new ObjectMetadataRecord[uniqueIds.Count];
var tasks = partitioner
.GetOrderablePartitions(_degreeOfParallelism)
.Select(async x =>
{
while (x.MoveNext())
{
var templateId = x.Current.Value;
ObjectMetadataRecord record;
try
{
var versionId = await GetTemplateLatestVersion(templateId);
var response = await _s3Client.GetObjectMetadataAsync(_bucketName, templateId.ToString(), versionId);
var metadataWrapper = MetadataCollectionWrapper.For(response.Metadata);
var author = metadataWrapper.Read <string>(MetadataElement.Author);
var authorLogin = metadataWrapper.Read <string>(MetadataElement.AuthorLogin);
var authorName = metadataWrapper.Read <string>(MetadataElement.AuthorName);
record = new ObjectMetadataRecord(
templateId,
versionId,
response.LastModified,
new AuthorInfo(author, authorLogin, authorName));
}
catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
record = null;
}
catch (ObjectNotFoundException)
{
record = null;
}
result[x.Current.Key] = record;
}
});
await Task.WhenAll(tasks);
return(result.Where(x => x != null).ToList());
}
public async Task <MultipartUploadSession> InitiateMultipartUpload(
Guid sessionId,
int templateCode,
IUploadedFileMetadata uploadedFileMetadata)
{
if (string.IsNullOrEmpty(uploadedFileMetadata.FileName))
{
throw new MissingFilenameException($"Filename has not been provided for the item '{templateCode}'");
}
var(sessionDescriptor, _, expiresAt) = await _sessionStorageReader.GetSessionDescriptor(sessionId);
if (sessionDescriptor.BinaryElementTemplateCodes.All(x => x != templateCode))
{
throw new InvalidTemplateException(
$"Binary content is not expected for the item '{templateCode}' within template '{sessionDescriptor.TemplateId}' " +
$"with version Id '{sessionDescriptor.TemplateVersionId}'.");
}
var elementDescriptor = await GetElementDescriptor(sessionDescriptor.TemplateId, sessionDescriptor.TemplateVersionId, templateCode);
EnsureFileMetadataIsValid(elementDescriptor, sessionDescriptor.Language, uploadedFileMetadata);
var fileKey = Guid.NewGuid().ToString();
var key = sessionId.AsS3ObjectKey(fileKey);
var request = new InitiateMultipartUploadRequest
{
BucketName = _filesBucketName,
Key = key,
ContentType = uploadedFileMetadata.ContentType
};
var metadataWrapper = MetadataCollectionWrapper.For(request.Metadata);
metadataWrapper.Write(MetadataElement.Filename, uploadedFileMetadata.FileName);
var uploadResponse = await _cephS3Client.InitiateMultipartUploadAsync(request);
return(new MultipartUploadSession(
sessionId,
sessionDescriptor,
expiresAt,
elementDescriptor,
uploadedFileMetadata,
fileKey,
uploadResponse.UploadId));
}
private async Task PutTemplate(long id, AuthorInfo authorInfo, ITemplateDescriptor templateDescriptor)
{
await VerifyElementDescriptorsConsistency(templateDescriptor.Elements);
var putRequest = new PutObjectRequest
{
Key = id.ToString(),
BucketName = _bucketName,
ContentType = ContentType.Json,
ContentBody = JsonConvert.SerializeObject(templateDescriptor, SerializerSettings.Default),
CannedACL = S3CannedACL.PublicRead,
};
var metadataWrapper = MetadataCollectionWrapper.For(putRequest.Metadata);
metadataWrapper.Write(MetadataElement.Author, authorInfo.Author);
metadataWrapper.Write(MetadataElement.AuthorLogin, authorInfo.AuthorLogin);
metadataWrapper.Write(MetadataElement.AuthorName, authorInfo.AuthorName);
await _s3Client.PutObjectAsync(putRequest);
}
public async Task Setup(Guid sessionId, long templateId, string templateVersionId, Language language, AuthorInfo authorInfo)
{
if (language == Language.Unspecified)
{
throw new SessionCannotBeCreatedException("Language must be explicitly specified.");
}
var templateDescriptor = await _templatesStorageReader.GetTemplateDescriptor(templateId, templateVersionId);
var sessionDescriptor = new SessionDescriptor
{
TemplateId = templateDescriptor.Id,
TemplateVersionId = templateDescriptor.VersionId,
Language = language,
BinaryElementTemplateCodes = templateDescriptor.GetBinaryElementTemplateCodes()
};
var request = new PutObjectRequest
{
BucketName = _filesBucketName,
Key = sessionId.AsS3ObjectKey(Tokens.SessionPostfix),
CannedACL = S3CannedACL.PublicRead,
ContentType = ContentType.Json,
ContentBody = JsonConvert.SerializeObject(sessionDescriptor, SerializerSettings.Default)
};
var expiresAt = SessionDescriptor.CurrentTime().Add(_sessionExpiration);
var metadataWrapper = MetadataCollectionWrapper.For(request.Metadata);
metadataWrapper.Write(MetadataElement.ExpiresAt, expiresAt);
metadataWrapper.Write(MetadataElement.Author, authorInfo.Author);
metadataWrapper.Write(MetadataElement.AuthorLogin, authorInfo.AuthorLogin);
metadataWrapper.Write(MetadataElement.AuthorName, authorInfo.AuthorName);
await _eventSender.SendAsync(_sessionsTopicName, new SessionCreatingEvent(sessionId, expiresAt));
await _cephS3Client.PutObjectAsync(request);
_createdSessionsMetric.Inc();
}
private async Task <IReadOnlyDictionary <int, IReadOnlyCollection <BinaryValidationError> > > VerifyObjectBinaryElementsConsistency(
Language language,
IReadOnlyCollection <IObjectElementDescriptor> elements)
{
var errors = new Dictionary <int, IReadOnlyCollection <BinaryValidationError> >();
foreach (var binaryElement in elements.GetBinaryElements())
{
var elementErrors = new List <BinaryValidationError>();
var constraints = binaryElement.Constraints.For(language);
var fileKeys = binaryElement.Value.ExtractFileKeys();
foreach (var fileKey in fileKeys)
{
using (var getResponse = await _s3Client.GetObjectAsync(_filesBucketName, fileKey))
{
var memoryStream = new MemoryStream();
using (getResponse.ResponseStream)
{
await getResponse.ResponseStream.CopyToAsync(memoryStream);
memoryStream.Position = 0;
}
using (memoryStream)
{
var metadataWrapper = MetadataCollectionWrapper.For(getResponse.Metadata);
var fileName = metadataWrapper.Read <string>(MetadataElement.Filename);
IUploadedFileMetadata binaryMetadata;
if (binaryElement.Value is ICompositeBitmapImageElementValue compositeBitmapImageElementValue &&
compositeBitmapImageElementValue.Raw != fileKey)
{
var image = compositeBitmapImageElementValue.SizeSpecificImages.First(x => x.Raw == fileKey);
binaryMetadata = new UploadedImageMetadata(
fileName,
getResponse.Headers.ContentType,
getResponse.ContentLength,
image.Size);
}
public async Task <(Stream, BinaryMetadata)> GetBinaryData(string fileKey)
{
try
{
using (var getResponse = await _cephS3Client.GetObjectAsync(_filesBucketName, fileKey))
{
var memoryStream = new MemoryStream();
using (getResponse.ResponseStream)
{
await getResponse.ResponseStream.CopyToAsync(memoryStream);
memoryStream.Position = 0;
}
var metadataWrapper = MetadataCollectionWrapper.For(getResponse.Metadata);
var fileName = metadataWrapper.Read <string>(MetadataElement.Filename);
return(memoryStream, new BinaryMetadata(fileName, getResponse.ContentLength, getResponse.Headers.ContentType));
}
}
catch (AmazonS3Exception ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
throw new ObjectNotFoundException($"Binary with the key '{fileKey}' not found.");
}
}
public async Task <string> CompleteMultipartUpload(MultipartUploadSession uploadSession)
{
var uploadKey = uploadSession.SessionId.AsS3ObjectKey(uploadSession.FileKey);
var partETags = uploadSession.Parts.Select(x => new PartETag(x.PartNumber, x.Etag)).ToList();
var uploadResponse = await _cephS3Client.CompleteMultipartUploadAsync(
new CompleteMultipartUploadRequest
{
BucketName = _filesBucketName,
Key = uploadKey,
UploadId = uploadSession.UploadId,
PartETags = partETags
});
uploadSession.Complete();
if (SessionDescriptor.IsSessionExpired(uploadSession.SessionExpiresAt))
{
throw new SessionExpiredException(uploadSession.SessionId, uploadSession.SessionExpiresAt);
}
try
{
using (var getResponse = await _cephS3Client.GetObjectAsync(_filesBucketName, uploadKey))
{
var memoryStream = new MemoryStream();
using (getResponse.ResponseStream)
{
getResponse.ResponseStream.CopyTo(memoryStream);
memoryStream.Position = 0;
}
using (memoryStream)
{
var sessionDescriptor = uploadSession.SessionDescriptor;
var elementDescriptor = uploadSession.ElementDescriptor;
EnsureFileContentIsValid(
elementDescriptor.TemplateCode,
elementDescriptor.Type,
elementDescriptor.Constraints.For(sessionDescriptor.Language),
memoryStream,
uploadSession.UploadedFileMetadata);
}
var metadataWrapper = MetadataCollectionWrapper.For(getResponse.Metadata);
var fileName = metadataWrapper.Read <string>(MetadataElement.Filename);
var fileExtension = Path.GetExtension(fileName).ToLowerInvariant();
var fileKey = Path.ChangeExtension(uploadSession.SessionId.AsS3ObjectKey(uploadResponse.ETag), fileExtension);
var copyRequest = new CopyObjectRequest
{
ContentType = uploadSession.UploadedFileMetadata.ContentType,
SourceBucket = _filesBucketName,
SourceKey = uploadKey,
DestinationBucket = _filesBucketName,
DestinationKey = fileKey,
MetadataDirective = S3MetadataDirective.REPLACE,
CannedACL = S3CannedACL.PublicRead
};
foreach (var metadataKey in getResponse.Metadata.Keys)
{
copyRequest.Metadata.Add(metadataKey, getResponse.Metadata[metadataKey]);
}
await _cephS3Client.CopyObjectAsync(copyRequest);
_uploadedBinariesMetric.Inc();
_memoryCache.Set(fileKey, new BinaryMetadata(fileName, getResponse.ContentLength), uploadSession.SessionExpiresAt);
return(fileKey);
}
}
finally
{
await _cephS3Client.DeleteObjectAsync(_filesBucketName, uploadKey);
}
}
public async Task CorrectJsonShouldBePassedToS3PutObjectForScalableBitmapImageElementWithoutAnchor()
{
const int TemplateCode = 100;
const Language Language = Language.Ru;
var constraints = new ScalableBitmapImageElementConstraints
{
SupportedFileFormats = new List <FileFormat> {
FileFormat.Png
},
ImageSizeRange = new ImageSizeRange {
Min = ImageSize.Empty, Max = new ImageSize {
Width = 500, Height = 500
}
}
};
var templateDescriptor = new TemplateDescriptor
{
Id = TemplateId,
VersionId = TemplateVersionId,
Properties = new JObject(),
Elements = new[]
{
new ElementDescriptor(
ElementDescriptorType.ScalableBitmapImage,
TemplateCode,
new JObject(),
new ConstraintSet(new[]
{
new ConstraintSetItem(Language, constraints)
}))
}
};
_objectsStorageReaderMock.Setup(m => m.IsObjectExists(ObjectId))
.ReturnsAsync(false);
_templatesStorageReaderMock.Setup(m => m.GetTemplateDescriptor(TemplateId, TemplateVersionId))
.ReturnsAsync(templateDescriptor);
_objectsStorageReaderMock.Setup(m => m.GetObjectLatestVersion(It.IsAny <long>()))
.ReturnsAsync(
new VersionedObjectDescriptor <string>(
ObjectId.AsS3ObjectKey(Tokens.ObjectPostfix),
ObjectVersionId,
DateTime.UtcNow)
);
var response = new GetObjectMetadataResponse();
var metadataWrapper = MetadataCollectionWrapper.For(response.Metadata);
metadataWrapper.Write(MetadataElement.ExpiresAt, DateTime.UtcNow.AddDays(1));
_cephS3ClientMock.Setup(m => m.GetObjectMetadataAsync(It.IsAny <string>(), It.IsAny <string>()))
.ReturnsAsync(response);
var memoryStream = new MemoryStream();
using (var stream = File.OpenRead(Path.Combine("images", "64x48.png")))
{
await stream.CopyToAsync(memoryStream);
memoryStream.Position = 0;
}
var fileKey = Guid.NewGuid().AsS3ObjectKey("key.raw");
var getObjectResponse = new GetObjectResponse {
ResponseStream = memoryStream
};
metadataWrapper = MetadataCollectionWrapper.For(getObjectResponse.Metadata);
metadataWrapper.Write(MetadataElement.Filename, "file name.png");
_s3ClientMock.Setup(m => m.GetObjectAsync(It.IsAny <string>(), fileKey, It.IsAny <CancellationToken>()))
.ReturnsAsync(getObjectResponse);
var requests = new List <PutObjectRequest>();
_s3ClientMock.Setup(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>()))
.Callback <PutObjectRequest>(request => requests.Add(request))
.ReturnsAsync(new PutObjectResponse());
var objectDescriptor = new ObjectDescriptor
{
Language = Language,
TemplateId = TemplateId,
TemplateVersionId = TemplateVersionId,
Properties = new JObject(),
Elements = new[]
{
new ObjectElementDescriptor
{
Type = ElementDescriptorType.ScalableBitmapImage,
TemplateCode = TemplateCode,
Constraints = new ConstraintSet(new[]
{
new ConstraintSetItem(Language, constraints)
}),
Value = new ScalableBitmapImageElementValue
{
Raw = fileKey
}
}
}
};
await _objectsManagementService.Create(ObjectId, AuthorInfo, objectDescriptor);
var elementContent = requests[0].ContentBody;
var elementJson = JObject.Parse(elementContent);
var valueJson = elementJson["value"];
Assert.Equal(fileKey, valueJson["raw"]);
Assert.NotNull(valueJson["filename"]);
Assert.NotNull(valueJson["filesize"]);
Assert.Equal(nameof(Anchor.Middle).ToLower(), valueJson["anchor"]);
}
public async Task CorrectJsonShouldBePassedToS3PutObjectForScalableBitmapImageElementWithoutAnchor()
{
const int TemplateCode = 100;
const Language Language = Language.Ru;
var constraints = new ScalableBitmapImageElementConstraints();
var templateDescriptor = new TemplateDescriptor
{
Id = TemplateId,
VersionId = TemplateVersionId,
Properties = new JObject(),
Elements = new[]
{
new ElementDescriptor(
ElementDescriptorType.ScalableBitmapImage,
TemplateCode,
new JObject(),
new ConstraintSet(new[]
{
new ConstraintSetItem(Language, constraints)
}))
}
};
_objectsStorageReaderMock.Setup(m => m.IsObjectExists(It.IsAny <long>()))
.ReturnsAsync(() => false);
_templatesStorageReaderMock.Setup(m => m.GetTemplateDescriptor(It.IsAny <long>(), It.IsAny <string>()))
.ReturnsAsync(() => templateDescriptor);
_templatesStorageReaderMock.Setup(m => m.GetTemplateLatestVersion(It.IsAny <long>()))
.ReturnsAsync(() => TemplateVersionId);
_objectsStorageReaderMock.Setup(m => m.GetObjectLatestVersions(It.IsAny <long>()))
.ReturnsAsync(() => new[]
{
new VersionedObjectDescriptor <string>(
ObjectId.AsS3ObjectKey(Tokens.ObjectPostfix),
ObjectVersionId,
DateTime.UtcNow)
});
var response = new GetObjectMetadataResponse();
var metadataWrapper = MetadataCollectionWrapper.For(response.Metadata);
metadataWrapper.Write(MetadataElement.ExpiresAt, DateTime.UtcNow.AddDays(1));
_cephS3ClientMock.Setup(m => m.GetObjectMetadataAsync(It.IsAny <string>(), It.IsAny <string>()))
.ReturnsAsync(() => response);
var requests = new List <PutObjectRequest>();
_s3ClientMock.Setup(m => m.PutObjectAsync(It.IsAny <PutObjectRequest>()))
.Callback <PutObjectRequest>(request => requests.Add(request))
.ReturnsAsync(new PutObjectResponse());
var fileKey = Guid.NewGuid().AsS3ObjectKey("key.raw");
var objectDescriptor = new ObjectDescriptor
{
Language = Language,
TemplateId = TemplateId,
TemplateVersionId = TemplateVersionId,
Properties = new JObject(),
Elements = new[]
{
new ObjectElementDescriptor
{
Type = ElementDescriptorType.ScalableBitmapImage,
TemplateCode = TemplateCode,
Constraints = new ConstraintSet(new[]
{
new ConstraintSetItem(Language, constraints)
}),
Value = new ScalableBitmapImageElementValue
{
Raw = fileKey
}
}
}
};
await _objectsManagementService.Create(ObjectId, AuthorInfo, objectDescriptor);
var elementContent = requests[0].ContentBody;
var elementJson = JObject.Parse(elementContent);
var valueJson = elementJson["value"];
Assert.Equal(fileKey, valueJson["raw"]);
Assert.NotNull(valueJson["filename"]);
Assert.NotNull(valueJson["filesize"]);
Assert.Equal(nameof(Anchor.Middle).ToLower(), valueJson["anchor"]);
}
private async Task <string> PutObject(long id,
string versionId,
AuthorInfo authorInfo,
IEnumerable <IObjectElementDescriptor> existingObjectElements,
IObjectDescriptor objectDescriptor)
{
PreprocessObjectElements(objectDescriptor.Elements);
await VerifyObjectElementsConsistency(id, objectDescriptor.Language, objectDescriptor.Elements);
var metadataForBinaries = await RetrieveMetadataForBinaries(id, existingObjectElements, objectDescriptor.Elements);
await _eventSender.SendAsync(_objectEventsTopic, new ObjectVersionCreatingEvent(id, versionId));
var totalBinariesCount = 0;
PutObjectRequest putRequest;
MetadataCollectionWrapper metadataWrapper;
foreach (var elementDescriptor in objectDescriptor.Elements)
{
var(elementPersistenceValue, binariesCount) = ConvertToPersistenceValue(elementDescriptor.Value, metadataForBinaries);
var elementPersistenceDescriptor = new ObjectElementPersistenceDescriptor(elementDescriptor, elementPersistenceValue);
totalBinariesCount += binariesCount;
putRequest = new PutObjectRequest
{
Key = id.AsS3ObjectKey(elementDescriptor.Id),
BucketName = _bucketName,
ContentType = ContentType.Json,
ContentBody = JsonConvert.SerializeObject(elementPersistenceDescriptor, SerializerSettings.Default),
CannedACL = S3CannedACL.PublicRead
};
metadataWrapper = MetadataCollectionWrapper.For(putRequest.Metadata);
metadataWrapper.Write(MetadataElement.Author, authorInfo.Author);
metadataWrapper.Write(MetadataElement.AuthorLogin, authorInfo.AuthorLogin);
metadataWrapper.Write(MetadataElement.AuthorName, authorInfo.AuthorName);
await _s3Client.PutObjectAsync(putRequest);
}
var objectKey = id.AsS3ObjectKey(Tokens.ObjectPostfix);
var objectVersions = await _objectsStorageReader.GetObjectLatestVersions(id);
var elementVersions = objectVersions.Where(x => !x.Id.EndsWith(Tokens.ObjectPostfix)).ToList();
var objectPersistenceDescriptor = new ObjectPersistenceDescriptor
{
TemplateId = objectDescriptor.TemplateId,
TemplateVersionId = objectDescriptor.TemplateVersionId,
Language = objectDescriptor.Language,
Properties = objectDescriptor.Properties,
Elements = elementVersions
};
putRequest = new PutObjectRequest
{
Key = objectKey,
BucketName = _bucketName,
ContentType = ContentType.Json,
ContentBody = JsonConvert.SerializeObject(objectPersistenceDescriptor, SerializerSettings.Default),
CannedACL = S3CannedACL.PublicRead
};
metadataWrapper = MetadataCollectionWrapper.For(putRequest.Metadata);
metadataWrapper.Write(MetadataElement.Author, authorInfo.Author);
metadataWrapper.Write(MetadataElement.AuthorLogin, authorInfo.AuthorLogin);
metadataWrapper.Write(MetadataElement.AuthorName, authorInfo.AuthorName);
metadataWrapper.Write(
MetadataElement.ModifiedElements,
string.Join(Tokens.ModifiedElementsDelimiter.ToString(), objectDescriptor.Elements.Select(x => x.TemplateCode)));
await _s3Client.PutObjectAsync(putRequest);
_referencedBinariesMetric.Inc(totalBinariesCount);
objectVersions = await _objectsStorageReader.GetObjectLatestVersions(id);
return(objectVersions.Where(x => x.Id.EndsWith(Tokens.ObjectPostfix))
.Select(x => x.VersionId)
.Single());
}
public async Task CorrectJsonShouldBePassedToS3PutObjectForCompositeBitmapImageElement()
{
const int TemplateCode = 100;
const Language Language = Language.Ru;
var constraints = new CompositeBitmapImageElementConstraints
{
SupportedFileFormats = new List <FileFormat> {
FileFormat.Png
},
ImageSizeRange = new ImageSizeRange {
Min = ImageSize.Empty, Max = new ImageSize {
Width = 500, Height = 500
}
}
};
var templateDescriptor = new TemplateDescriptor
{
Id = TemplateId,
VersionId = TemplateVersionId,
Properties = new JObject(),
Elements = new[]
{
new ElementDescriptor(
ElementDescriptorType.CompositeBitmapImage,
TemplateCode,
new JObject(),
new ConstraintSet(new[]
{
new ConstraintSetItem(Language, constraints)
}))
}
};
_objectsStorageReaderMock.Setup(m => m.IsObjectExists(It.IsAny <long>()))
.ReturnsAsync(() => false);
_templatesStorageReaderMock.Setup(m => m.GetTemplateDescriptor(It.IsAny <long>(), It.IsAny <string>()))
.ReturnsAsync(() => templateDescriptor);
_objectsStorageReaderMock.Setup(m => m.GetObjectLatestVersion(It.IsAny <long>()))
.ReturnsAsync(new VersionedObjectDescriptor <long>(ObjectId, ObjectVersionId, DateTime.UtcNow));
var response = new GetObjectMetadataResponse();
var metadataWrapper = MetadataCollectionWrapper.For(response.Metadata);
metadataWrapper.Write(MetadataElement.ExpiresAt, DateTime.UtcNow.AddDays(1));
_cephS3ClientMock.Setup(m => m.GetObjectMetadataAsync(It.IsAny <string>(), It.IsAny <string>()))
.ReturnsAsync(() => response);
var memoryStream = new MemoryStream();
using (var stream = File.OpenRead(Path.Combine("images", "64x48.png")))
{
await stream.CopyToAsync(memoryStream);
memoryStream.Position = 0;
}
var fileKey = Guid.NewGuid().AsS3ObjectKey("key.raw");
var getObjectResponse = new GetObjectResponse {
ResponseStream = memoryStream
};
metadataWrapper = MetadataCollectionWrapper.For(getObjectResponse.Metadata);
metadataWrapper.Write(MetadataElement.Filename, "file name.png");
_cephS3ClientMock.Setup(m => m.GetObjectAsync(It.IsAny <string>(), fileKey, It.IsAny <CancellationToken>()))
.ReturnsAsync(getObjectResponse);
var objectDescriptor = new ObjectDescriptor
{
Language = Language,
TemplateId = TemplateId,
TemplateVersionId = TemplateVersionId,
Properties = new JObject(),
Elements = new[]
{
new ObjectElementDescriptor
{
Type = ElementDescriptorType.CompositeBitmapImage,
TemplateCode = TemplateCode,
Constraints = new ConstraintSet(new[]
{
new ConstraintSetItem(Language, constraints)
}),
Value = new CompositeBitmapImageElementValue
{
Raw = fileKey,
CropArea = new CropArea(),
SizeSpecificImages = Array.Empty <SizeSpecificImage>()
}
}
}
};
await _objectsManagementService.Create(ObjectId, AuthorInfo, objectDescriptor);
Assert.True(await _inMemoryContext.Objects.AnyAsync());
var element = await _inMemoryContext.ObjectElements.FirstOrDefaultAsync();
Assert.NotNull(element);
var elementJson = JObject.Parse(element.Data);
var valueJson = elementJson["value"];
Assert.NotNull(valueJson);
Assert.Equal(fileKey, valueJson["raw"]);
Assert.NotNull(valueJson["filename"]);
Assert.NotNull(valueJson["filesize"]);
Assert.NotNull(valueJson["cropArea"]);
Assert.NotNull(valueJson["sizeSpecificImages"]);
}
