public IUploadInfo GetUploadInfo(IContentIdentifier contentIdentifier)
{
_authorizationChecker.CheckUserIsInRole(Security.Roles.Replicator);
var result = GetMultiPartUploadInfo(contentIdentifier, -1);
if (result == null)
{
return(null);
}
if (result?.PartUploadInfos?.Count() != 1)
{
throw new UserException("Multipart upload not supported");
}
var partUploadInfo = result?.PartUploadInfos?.FirstOrDefault();
var uploadInfo =
partUploadInfo == null ?
null :
_uploadInfoFactory(partUploadInfo.URL, result.HttpMethod, partUploadInfo.RequestHttpHeaders, result.UploadIdentifier);
return(uploadInfo);
}
public void PutContent(IContentIdentifier contentIdentifier, ISource source)
{
var contentStorageSource = source as IContentStorageSource;
if (contentStorageSource != null)
{
var downloadInfo = contentStorageSource.GetDownloadInfo(contentIdentifier);
var urlContentUploadInfo = GetUrlContentUploadInfo(contentIdentifier, downloadInfo);
if (urlContentUploadInfo != null)
{
UploadUrlContent(urlContentUploadInfo);
return;
}
}
var contentStream = source.GetContentStream(contentIdentifier);
_contentStorageUploader.UploadContent(
contentStream.Stream,
ContentStorageServiceURL,
ContainerName,
contentIdentifier,
contentStream.Size,
AuthenticationToken
);
}
public IDownloadInfo GetDownloadInfo(IContentIdentifier contentIdentifier)
{
var result = _contentStorageSource.GetDownloadInfo(contentIdentifier);
Logger.LogTrace($"Content size: {contentIdentifier.Hash} ==> {result.Size}");
return(result);
}
public static void SetStringContent(this IContainer container, IContentIdentifier contentIdentifier, string stringContent)
{
var content = container.ContentExists(contentIdentifier) ? container.GetContent(contentIdentifier) : container.CreateContent(contentIdentifier);
using (var stream = (string.IsNullOrEmpty(stringContent) ? new System.IO.MemoryStream() : new System.IO.MemoryStream(Encoding.UTF8.GetBytes(stringContent))))
{
content.LoadFromStream(stream);
}
}
private string GetSystemContentName(IContentIdentifier contentIdentifier)
{
if (contentIdentifier.Guid.Equals(SystemString))
{
return(System.IO.Path.ChangeExtension(contentIdentifier.Hash + ContentNameSeparator + SystemString, contentIdentifier.Extension));
}
return(null);
}
private IMultiPartIDUploadInfo GetMultiPartIDUploadInfo(IContentIdentifier contentIdentifier, long size, HttpClient httpClient, string contentStorageServiceURL, string containerName, string authenticationToken)
{
var contentID = _contentIdentifierSerializer.Serialize(contentIdentifier);
var requestURL = _contentStorageServiceURLProvider.GetURLForGetMultiPartUploadInfo(contentStorageServiceURL, containerName, contentID, size, authenticationToken);
var serializedIDUploadInfo = httpClient.CheckedGetStringAsync(requestURL).Result;
return(_serializer.Deserialize <IMultiPartIDUploadInfo>(serializedIDUploadInfo));
}
public IContent CreateContent(IContentIdentifier contentIdentifier)
{
if (ContentExists(contentIdentifier))
{
throw new UserException("Content already exists");
}
var contentName = _contentNameProvider.GetContentName(contentIdentifier);
return(_fileSystemContentFactory(_root, _dirName, contentName));
}
public IContentStream GetContentStream(IContentIdentifier contentIdentifier)
{
var downloadInfo = GetDownloadInfo(contentIdentifier);
using (var httpClient = GetHttpClient())
{
var stream = httpClient.CheckedGetStreamAsync(downloadInfo.Url).Result;
return(_contentStreamFactory(downloadInfo.Size, stream));
}
}
public IContent CreateContent(IContentIdentifier contentIdentifier)
{
if (ContentExists(contentIdentifier))
{
throw new UserException("Content already exists");
}
var contentName = _contentNameProvider.GetContentName(contentIdentifier);
return(_amazonContentFactory(_bucketName, MakeKey(contentName), null));
}
public IContent GetContent(IContentIdentifier contentIdentifier)
{
if (!ContentExists(contentIdentifier))
{
throw new UserException("Content does not exists");
}
var contentName = _contentNameProvider.GetContentName(contentIdentifier);
return(GetContent(contentName));
}
private IDownloadInfo GetContainerContentDownloadInfo(IContentIdentifier contentIdentifier, string containerName)
{
var container = GetContainer(containerName);
if (container.ContentExists(contentIdentifier))
{
var content = container.GetContent(contentIdentifier) as IURLContent;
return(content.GetDownloadInfo(URLExpirySeconds));
}
return(null);
}
public IContent CreateContent(IContentIdentifier contentIdentifier)
{
if (ContentExists(contentIdentifier))
{
throw new UserException("Content already exists");
}
var contentName = _contentNameProvider.GetContentName(contentIdentifier);
var blockBlobReference = GetBlockBlobReference(contentName);
return(_azureContentFactory(blockBlobReference));
}
public IDownloadInfo GetDownloadInfo(IContentIdentifier contentIdentifier)
{
var contentID = ContentIdentifierSerializer.Serialize(contentIdentifier);
var requestURL = ContentStorageServiceURLProvider.GetURLForGetContentDownloadInfo(ContentStorageServiceURL, ContainerName, contentID, AuthenticationToken);
using (var httpClient = GetHttpClient())
{
var downloadinfoJson = httpClient.CheckedGetStringAsync(requestURL).Result;
return(Serializer.Deserialize <IDownloadInfo>(downloadinfoJson));
}
}
public void PutContent(IContentIdentifier contentIdentifier, ISource source)
{
Logger.LogTrace($"Start putting content: {contentIdentifier.Hash}");
try
{
_destination.PutContent(contentIdentifier, source);
Logger.LogTrace($"End putting content: {contentIdentifier.Hash}");
}
catch (Exception e)
{
Logger.LogError($"Error putting content: {contentIdentifier.Hash}" + Environment.NewLine + e.GetAggregateMessages(), e);
throw;
}
}
public async Task <IContentIdentifier> CommitContentAsync(IContentIdentifier uncommittedContentIdentifier)
{
System.Diagnostics.Debug.Assert(uncommittedContentIdentifier.Uncommitted);
var uncommittedContent = GetContent(uncommittedContentIdentifier);
_contentHashValidator.ValidateHash(uncommittedContent, uncommittedContentIdentifier.Hash);
var uncommittedContentName = _contentNameProvider.GetContentName(uncommittedContentIdentifier);
var contentIdentifier = _contentIdentifierGenerator.GetCommittedContentIdentifier(uncommittedContentIdentifier);
var contentName = _contentNameProvider.GetContentName(contentIdentifier);
await RenameContentAsync(uncommittedContentName, contentName);
return(contentIdentifier);
}
public void PutContent(IContentIdentifier contentIdentifier, ISource source)
{
if (Container.ContentExists(contentIdentifier))
{
return;
}
var uncommittedContentIdentifier = _contentIdentifierGenerator.GetUncommittedContentIdentifier(contentIdentifier);
var content = (Container.ContentExists(uncommittedContentIdentifier) ? Container.GetContent(uncommittedContentIdentifier) : Container.CreateContent(uncommittedContentIdentifier));
var contentStream = source.GetContentStream(contentIdentifier);
content.LoadFromStream(contentStream.Stream);
Container.CommitContentAsync(uncommittedContentIdentifier).Wait();
}
public IContentStream GetContentStream(IContentIdentifier contentIdentifier)
{
Logger.LogTrace($"Start getting content: {contentIdentifier.Hash}");
try
{
var result = _source.GetContentStream(contentIdentifier);
Logger.LogTrace($"End getting content: {contentIdentifier.Hash}");
return(result);
}
catch (Exception e)
{
Logger.LogError($"Error getting content: {contentIdentifier.Hash}", e);
throw;
}
}
public static string GetStringContent(this IContainer container, IContentIdentifier contentIdentifier)
{
if (!container.ContentExists(contentIdentifier))
{
return(null);
}
var content = container.GetContent(contentIdentifier);
using (var stream = content.GetReadStream())
{
using (var reader = new System.IO.StreamReader(stream))
{
return(reader.ReadToEnd());
}
}
}
public string GetContentName(IContentIdentifier contentIdentifier)
{
if (contentIdentifier == null)
{
return(null);
}
IEnumerable <string> contentNameParts = new[]
{
contentIdentifier.Hash,
contentIdentifier.ModifiedMoment.ToUniversalTime().ToString(DateTimeFormat),
contentIdentifier.Guid
};
if (contentIdentifier.Uncommitted)
{
contentNameParts = contentNameParts.Concat(new[] { UncommittedString });
}
return(GetSystemContentName(contentIdentifier) ?? GetContentName(contentNameParts, contentIdentifier.Extension));
}
private IEnumerable <IContentIdentifier> GetContentIdentifiers(DateTimeOffset afterMoment)
{
afterMoment = new[] { afterMoment, new DateTimeOffset(1900, 1, 1, 0, 0, 0, TimeSpan.FromTicks(0)) }.Max();
using (var httpClient = GetHttpClient())
{
IContentIdentifier lastContentIdentifier = null;
while (true)
{
var afterContentID = (lastContentIdentifier == null) ? null : ContentIdentifierSerializer.Serialize(lastContentIdentifier);
var moment = (lastContentIdentifier == null) ? (DateTimeOffset?)afterMoment : null;
var requestURL = ContentStorageServiceURLProvider.GetURLForGetContentIdentifiers(ContentStorageServiceURL, ContainerName, moment, null, afterContentID, AuthenticationToken);
using (var stream = httpClient.CheckedGetStreamAsync(requestURL).Result)
{
var contentIdentifiers =
stream
.ReadAllLines(Encoding.UTF8)
.Select(x => ContentIdentifierSerializer.Deserialize(x))
.ToList();
var hasContentIdentifiers = false;
foreach (var contentIdentifier in contentIdentifiers)
{
yield return(contentIdentifier);
lastContentIdentifier = contentIdentifier;
hasContentIdentifiers = true;
}
if (!hasContentIdentifiers)
{
break;
}
}
}
}
}
private IHttpRequestInfo GetUrlContentUploadInfo(IContentIdentifier contentIdentifier, IDownloadInfo downloadInfo)
{
var contentID = ContentIdentifierSerializer.Serialize(contentIdentifier);
var urlForGetUrlContentUploadInfo =
ContentStorageServiceURLProvider.GetURLForGetUrlContentUploadInfo(
ContentStorageServiceURL,
ContainerName,
contentID,
downloadInfo.Size,
AuthenticationToken);
using (var httpClient = GetHttpClient())
{
var headers =
new[] {
new KeyValuePair <string, string>("ContentUrl", downloadInfo.Url),
new KeyValuePair <string, string>("ContentStorageServiceUrl", ContentStorageServiceURL)
};
using (var response = httpClient.GetAsync(urlForGetUrlContentUploadInfo, headers: headers).Result)
{
if (response.IsSuccessStatusCode)
{
var stringResult = response.Content.ReadAsStringAsync().Result;
return(Serializer.Deserialize <IHttpRequestInfo>(stringResult));
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
else
{
response.CheckSuccessAsync().Wait();
return(null);
}
}
}
}
public IDownloadInfo GetContentDownloadInfo(IContentIdentifier contentIdentifier)
{
_authorizationChecker.CheckUserIsInRole(Security.Roles.Replicator, Security.Roles.Backup);
var result = GetContainerContentDownloadInfo(contentIdentifier, _containerName);
if (result == null)
{
var readOnlySubcontainerNames = InternalGetReadOnlySubcontainerNames();
result =
readOnlySubcontainerNames
.Select(x => GetContainerContentDownloadInfo(contentIdentifier, x))
.Where(x => x != null)
.FirstOrDefault();
}
if (result == null)
{
throw new ApplicationException("Content does not exist");
}
return(result);
}
public IMultiPartUploadInfo GetMultiPartUploadInfo(IContentIdentifier contentIdentifier, long size)
{
_authorizationChecker.CheckUserIsInRole(Security.Roles.Replicator);
var container = GetContainer();
if (container.ContentExists(contentIdentifier))
{
var existingContentName = _contentNameProvider.GetContentName(contentIdentifier);
_committedContentNamesRepository.EnsureContent(_containerName, existingContentName, contentIdentifier.ModifiedMoment, CancellationToken.None);
return(null);
}
CheckContainerState(false);
var uncommittedContentIdentifier = _contentIdentifierGenerator.GetUncommittedContentIdentifier(contentIdentifier);
var content = (container.ContentExists(uncommittedContentIdentifier) ? container.GetContent(uncommittedContentIdentifier) : container.CreateContent(uncommittedContentIdentifier)) as IURLContent;
return(content.GetMultiPartUploadInfo(URLExpirySeconds, size));
}
public bool ContentExists(IContentIdentifier contentIdentifier)
{
var contentName = _contentNameProvider.GetContentName(contentIdentifier);
return(ContentExists(contentName));
}
public IUploadIdentifier GetUploadIdentifier(IContentIdentifier contentIdentifier, string hostIdentifier)
{
return(_uploadIdentifierFactory(contentIdentifier.Hash, contentIdentifier.Extension, contentIdentifier.Guid, hostIdentifier));
}
public IDocumentIdentifier GetDocumentIdentifier(IContentIdentifier contentIdentifier)
{
return(_documentIdentifierFactory(contentIdentifier.Hash, contentIdentifier.Extension));
}
private Task <DateTimeOffset> ReplicateContentAsync(IRoute route, IContentIdentifier contentIdentifier)
{
return(Task.Factory.StartNew(() => ReplicateContent(route, contentIdentifier)));
}
private DateTimeOffset ReplicateContent(IRoute route, IContentIdentifier contentIdentifier)
{
route.Destination.PutContent(contentIdentifier, route.Source);
return(contentIdentifier.ModifiedMoment);
}
public string GetContentDownloadURL(IContentIdentifier contentIdentifier)
{
return(GetContentDownloadInfo(contentIdentifier)?.Url);
}
public void UploadContent(Stream stream, string contentStorageServiceURL, string containerName, IContentIdentifier contentIdentifier, long size, string authenticationToken)
{
using (var httpClient = GetHttpClient())
{
var multiPartIDUploadInfo = GetMultiPartIDUploadInfo(contentIdentifier, size, httpClient, contentStorageServiceURL, containerName, authenticationToken);
if ((multiPartIDUploadInfo == null) || string.IsNullOrEmpty(multiPartIDUploadInfo.UploadID))
{
return;
}
var results = MultiPartUpload(multiPartIDUploadInfo, stream, size);
CommitMultiPartUpload(multiPartIDUploadInfo.UploadID, results, httpClient, contentStorageServiceURL, containerName, authenticationToken);
}
}
public DefaultCompilationManager(IContentIdentifier identifier) : this()
{
Requires.NotNull(identifier, "identifier");
ContentIdentifier = identifier;
}