public async Task <string> GetIdentityServiceAccessTokenAsync(IIdentityServiceClientInfo identityServiceClientInfo, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(identityServiceClientInfo.Url))
{
return(null);
}
var tokenRequest = new PasswordTokenRequest
{
Address = identityServiceClientInfo.Url.TrimEnd('/') + "/connect/token",
ClientId = identityServiceClientInfo.ClientId,
ClientSecret = identityServiceClientInfo.ClientSecret,
Scope = identityServiceClientInfo.Scope,
UserName = identityServiceClientInfo.UserName,
Password = identityServiceClientInfo.UserPassword
};
using (var httpClient = new HttpClient())
{
var response = await httpClient.RequestPasswordTokenAsync(tokenRequest, cancellationToken);
return(response.AccessToken);
}
}
public async Task ConfigAsync(HttpRequestMessage request, IIdentityServiceClientInfo identityServiceClientInfo, CancellationToken cancellationToken)
{
var accessToken = await GetIdentityServiceAccessTokenAsync(identityServiceClientInfo, cancellationToken);
if (!string.IsNullOrEmpty(accessToken))
{
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken);
}
}
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken)
{
var content =
await HttpUtils.GetStringAsync(collectUrl, HttpMethod.Get,
null, collectHeaders, null, timeout,
request => _identityServiceHttpRequestConfigurator.ConfigAsync(request, collectIdentityServiceClientInfo, cancellationToken),
cancellationToken : cancellationToken);
System.Console.WriteLine(content);
}
public IEnumerable <(IFileInfo CollectFileInfo, string CollectUrl)> GetCollectItems(string dataCollectionName, string collectFileIdentifiersUrl, IEnumerable <KeyValuePair <string, string> > collectFileIdentifiersHeaders, string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo identityServiceClientInfo, CancellationToken cancellationToken)
{
try
{
_logger.LogInformation("Started getting collect items for data collection '{dataCollectionName}'", dataCollectionName);
var result =
_collectItemsProvider.GetCollectItems(dataCollectionName, collectFileIdentifiersUrl, collectFileIdentifiersHeaders, collectUrl, collectHeaders, identityServiceClientInfo, cancellationToken)
.OrderBy(x => x.CollectFileInfo?.Name)
.ToList();
_logger.LogInformation("Finished getting collect items for data collection '{dataCollectionName}'", dataCollectionName);
_logger.LogInformation($"Retrieved {result.Count()} collect items for data collection '{dataCollectionName}'", dataCollectionName);
foreach (var collectItem in result)
{
_logger.LogInformation(collectItem.CollectFileInfo?.Name);
}
return(result);
}
catch (Exception e)
{
_logger.LogCritical("Error getting collect items for data collection '{dataCollectionName}': {errorMessage}", dataCollectionName, e.GetAggregateMessages());
throw;
}
}
public IEnumerable <(IFileInfo CollectFileInfo, string CollectUrl)> GetCollectItems(string dataCollectionName, string collectFileIdentifiersUrl, IEnumerable <KeyValuePair <string, string> > collectFileIdentifiersHeaders, string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo identityServiceClientInfo, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(collectFileIdentifiersUrl))
{
if (collectUrl.StartsWith("@"))
{
var jsonResult =
Policy
.Handle <Exception>()
.WaitAndRetry(new[] { TimeSpan.FromSeconds(5) })
.Execute(
ct => HttpUtils.GetStringAsync(collectUrl.Substring(1), collectHeaders, null, null, ct).Result,
cancellationToken
);
var urlsString = HttpUtils.FixJsonResult(jsonResult);
if (!string.IsNullOrEmpty(urlsString))
{
var urls = GetStrings(urlsString);
if (urls == null)
{
urls = urlsString.Split(Environment.NewLine);
}
foreach (var url in urls)
{
yield return(null, url);
}
}
}
else
{
yield return(null, collectUrl);
}
}
else
{
var query = collectFileIdentifiersUrl.Split("@");
var queryFilter = (query.Length > 1 ? query.First() : null);
var selector = query.Last().Split("|");
var queryUrl = selector.First();
var queryPropertyNames = (selector.Count() == 2) ? selector.Last().Split(',', ';') : Enumerable.Empty <string>();
var queryNamePropertyName = queryPropertyNames.FirstOrDefault();
var querySizePropertyName = queryPropertyNames.Skip(1).FirstOrDefault();
var queryMD5PropertyName = queryPropertyNames.Skip(2).FirstOrDefault();
var queryGroupIdPropertyName = queryPropertyNames.Skip(3).FirstOrDefault();
var collectFileIdentifiersJson = GetCollectItemsJson(queryUrl, collectFileIdentifiersHeaders, identityServiceClientInfo, cancellationToken).Result;
if (!string.IsNullOrEmpty(collectFileIdentifiersJson))
{
var namePropertyNames = string.IsNullOrEmpty(queryNamePropertyName) ? DefaultNamePropertyNames : new[] { queryNamePropertyName };
var sizePropertyNames = string.IsNullOrEmpty(querySizePropertyName) ? DefaultSizePropertyNames : new[] { querySizePropertyName };
var md5PropertyNames = string.IsNullOrEmpty(queryMD5PropertyName) ? DefaultMD5PropertyNames : new[] { queryMD5PropertyName };
var groupIdPropertyNames = string.IsNullOrEmpty(queryGroupIdPropertyName) ? DefaultGroupIdPropertyNames : new[] { queryGroupIdPropertyName };
var collectFileInfos = GetCollectFileInfos(collectFileIdentifiersJson, namePropertyNames, sizePropertyNames, md5PropertyNames, groupIdPropertyNames);
foreach (var collectFileInfo in collectFileInfos)
{
if (HttpUtils.IsUrl(collectFileInfo.Name))
{
yield return(null, collectFileInfo.Name);
}
else
{
Contract.Assert(!string.IsNullOrEmpty(collectUrl));
if ((string.IsNullOrEmpty(queryFilter)) ||
(FileMaskUtils.FileNameMatchesFilter(collectFileInfo.Name, queryFilter)))
{
yield return(collectFileInfo, collectUrl.Replace("[filename]", collectFileInfo.Name, StringComparison.InvariantCultureIgnoreCase));
}
}
}
}
}
}
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken)
{
using (var response = await HttpUtils.SendAsync(collectUrl, HttpMethod.Get, collectUrl, null, collectHeaders, null, timeout, null, HttpCompletionOption.ResponseHeadersRead, cancellationToken))
{
var sourceMD5 = response.ContentMD5();
using (var sourceStream = await response.Content.ReadAsStreamAsync())
{
var key = dataCollectionName + '/' + fileName;
await MultiPartUploadAsync(sourceStream, sourceMD5, DestinationConfig.RootBase(), DestinationConfig.RootDir('/', true) + key, cancellationToken);
}
}
}
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken)
{
var endpointUrl = GetEndpointUrl(DestinationConfig.CollectPostEndpoint, dataCollectionName, fileName);
if (string.IsNullOrEmpty(endpointUrl))
{
return;
}
await HttpUtils.SendAsync(endpointUrl, HttpMethod.Post,
collectUrl, null, collectHeaders, null, timeout,
request => _identityServiceHttpRequestConfigurator.ConfigAsync(request, DestinationConfig.IdentityServiceClientInfo, cancellationToken),
cancellationToken : cancellationToken);
}
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken)
{
var container = await GetContainerAsync(dataCollectionName, cancellationToken);
await SmartCopyToBlobAsync(collectUrl, collectHeaders, container, GetBlobName(dataCollectionName, fileName), timeout, finishWait, cancellationToken);
}
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken)
{
int bufferSize = 1 * 1024 * 1024;
using (var response = await HttpUtils.SendAsync(collectUrl, HttpMethod.Get, collectUrl, null, collectHeaders, null, timeout, null, HttpCompletionOption.ResponseHeadersRead, cancellationToken))
{
var sourceMD5 = response.ContentMD5();
var fullFileName = GetFullFileName(dataCollectionName, fileName);
Directory.CreateDirectory(Path.GetDirectoryName(fullFileName));
using (var sourceStream = await response.Content.ReadAsStreamAsync())
{
using (var fileStream = new FileStream(fullFileName, FileMode.Create))
{
await CopyUtils.CopyAsync(
(buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct),
async (buffer, count, cancellationToken2) => await fileStream.WriteAsync(buffer, 0, count, cancellationToken2),
bufferSize,
cancellationToken
);
}
}
using (var fileStream = new FileStream(fullFileName, FileMode.Open, FileAccess.Read))
{
var newMD5 = await CopyUtils.GetMD5HashStringAsync(fileStream, bufferSize, cancellationToken);
if ((!string.IsNullOrEmpty(sourceMD5)) &&
(!string.Equals(newMD5, sourceMD5, StringComparison.InvariantCultureIgnoreCase)))
{
throw new Exception("Invalid destination MD5");
}
File.WriteAllText(GetMD5FileName(fullFileName), newMD5);
}
}
}