public async Task <string> GetIdentityServiceAccessTokenAsync(IIdentityServiceClientInfo identityServiceClientInfo, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(identityServiceClientInfo.Url)) { return(null); } var tokenRequest = new PasswordTokenRequest { Address = identityServiceClientInfo.Url.TrimEnd('/') + "/connect/token", ClientId = identityServiceClientInfo.ClientId, ClientSecret = identityServiceClientInfo.ClientSecret, Scope = identityServiceClientInfo.Scope, UserName = identityServiceClientInfo.UserName, Password = identityServiceClientInfo.UserPassword }; using (var httpClient = new HttpClient()) { var response = await httpClient.RequestPasswordTokenAsync(tokenRequest, cancellationToken); return(response.AccessToken); } }
public async Task ConfigAsync(HttpRequestMessage request, IIdentityServiceClientInfo identityServiceClientInfo, CancellationToken cancellationToken) { var accessToken = await GetIdentityServiceAccessTokenAsync(identityServiceClientInfo, cancellationToken); if (!string.IsNullOrEmpty(accessToken)) { request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); } }
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken) { var content = await HttpUtils.GetStringAsync(collectUrl, HttpMethod.Get, null, collectHeaders, null, timeout, request => _identityServiceHttpRequestConfigurator.ConfigAsync(request, collectIdentityServiceClientInfo, cancellationToken), cancellationToken : cancellationToken); System.Console.WriteLine(content); }
public IEnumerable <(IFileInfo CollectFileInfo, string CollectUrl)> GetCollectItems(string dataCollectionName, string collectFileIdentifiersUrl, IEnumerable <KeyValuePair <string, string> > collectFileIdentifiersHeaders, string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo identityServiceClientInfo, CancellationToken cancellationToken) { try { _logger.LogInformation("Started getting collect items for data collection '{dataCollectionName}'", dataCollectionName); var result = _collectItemsProvider.GetCollectItems(dataCollectionName, collectFileIdentifiersUrl, collectFileIdentifiersHeaders, collectUrl, collectHeaders, identityServiceClientInfo, cancellationToken) .OrderBy(x => x.CollectFileInfo?.Name) .ToList(); _logger.LogInformation("Finished getting collect items for data collection '{dataCollectionName}'", dataCollectionName); _logger.LogInformation($"Retrieved {result.Count()} collect items for data collection '{dataCollectionName}'", dataCollectionName); foreach (var collectItem in result) { _logger.LogInformation(collectItem.CollectFileInfo?.Name); } return(result); } catch (Exception e) { _logger.LogCritical("Error getting collect items for data collection '{dataCollectionName}': {errorMessage}", dataCollectionName, e.GetAggregateMessages()); throw; } }
public IEnumerable <(IFileInfo CollectFileInfo, string CollectUrl)> GetCollectItems(string dataCollectionName, string collectFileIdentifiersUrl, IEnumerable <KeyValuePair <string, string> > collectFileIdentifiersHeaders, string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo identityServiceClientInfo, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(collectFileIdentifiersUrl)) { if (collectUrl.StartsWith("@")) { var jsonResult = Policy .Handle <Exception>() .WaitAndRetry(new[] { TimeSpan.FromSeconds(5) }) .Execute( ct => HttpUtils.GetStringAsync(collectUrl.Substring(1), collectHeaders, null, null, ct).Result, cancellationToken ); var urlsString = HttpUtils.FixJsonResult(jsonResult); if (!string.IsNullOrEmpty(urlsString)) { var urls = GetStrings(urlsString); if (urls == null) { urls = urlsString.Split(Environment.NewLine); } foreach (var url in urls) { yield return(null, url); } } } else { yield return(null, collectUrl); } } else { var query = collectFileIdentifiersUrl.Split("@"); var queryFilter = (query.Length > 1 ? query.First() : null); var selector = query.Last().Split("|"); var queryUrl = selector.First(); var queryPropertyNames = (selector.Count() == 2) ? selector.Last().Split(',', ';') : Enumerable.Empty <string>(); var queryNamePropertyName = queryPropertyNames.FirstOrDefault(); var querySizePropertyName = queryPropertyNames.Skip(1).FirstOrDefault(); var queryMD5PropertyName = queryPropertyNames.Skip(2).FirstOrDefault(); var queryGroupIdPropertyName = queryPropertyNames.Skip(3).FirstOrDefault(); var collectFileIdentifiersJson = GetCollectItemsJson(queryUrl, collectFileIdentifiersHeaders, identityServiceClientInfo, cancellationToken).Result; if (!string.IsNullOrEmpty(collectFileIdentifiersJson)) { var namePropertyNames = string.IsNullOrEmpty(queryNamePropertyName) ? DefaultNamePropertyNames : new[] { queryNamePropertyName }; var sizePropertyNames = string.IsNullOrEmpty(querySizePropertyName) ? DefaultSizePropertyNames : new[] { querySizePropertyName }; var md5PropertyNames = string.IsNullOrEmpty(queryMD5PropertyName) ? DefaultMD5PropertyNames : new[] { queryMD5PropertyName }; var groupIdPropertyNames = string.IsNullOrEmpty(queryGroupIdPropertyName) ? DefaultGroupIdPropertyNames : new[] { queryGroupIdPropertyName }; var collectFileInfos = GetCollectFileInfos(collectFileIdentifiersJson, namePropertyNames, sizePropertyNames, md5PropertyNames, groupIdPropertyNames); foreach (var collectFileInfo in collectFileInfos) { if (HttpUtils.IsUrl(collectFileInfo.Name)) { yield return(null, collectFileInfo.Name); } else { Contract.Assert(!string.IsNullOrEmpty(collectUrl)); if ((string.IsNullOrEmpty(queryFilter)) || (FileMaskUtils.FileNameMatchesFilter(collectFileInfo.Name, queryFilter))) { yield return(collectFileInfo, collectUrl.Replace("[filename]", collectFileInfo.Name, StringComparison.InvariantCultureIgnoreCase)); } } } } } }
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken) { using (var response = await HttpUtils.SendAsync(collectUrl, HttpMethod.Get, collectUrl, null, collectHeaders, null, timeout, null, HttpCompletionOption.ResponseHeadersRead, cancellationToken)) { var sourceMD5 = response.ContentMD5(); using (var sourceStream = await response.Content.ReadAsStreamAsync()) { var key = dataCollectionName + '/' + fileName; await MultiPartUploadAsync(sourceStream, sourceMD5, DestinationConfig.RootBase(), DestinationConfig.RootDir('/', true) + key, cancellationToken); } } }
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken) { var endpointUrl = GetEndpointUrl(DestinationConfig.CollectPostEndpoint, dataCollectionName, fileName); if (string.IsNullOrEmpty(endpointUrl)) { return; } await HttpUtils.SendAsync(endpointUrl, HttpMethod.Post, collectUrl, null, collectHeaders, null, timeout, request => _identityServiceHttpRequestConfigurator.ConfigAsync(request, DestinationConfig.IdentityServiceClientInfo, cancellationToken), cancellationToken : cancellationToken); }
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken) { var container = await GetContainerAsync(dataCollectionName, cancellationToken); await SmartCopyToBlobAsync(collectUrl, collectHeaders, container, GetBlobName(dataCollectionName, fileName), timeout, finishWait, cancellationToken); }
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken) { int bufferSize = 1 * 1024 * 1024; using (var response = await HttpUtils.SendAsync(collectUrl, HttpMethod.Get, collectUrl, null, collectHeaders, null, timeout, null, HttpCompletionOption.ResponseHeadersRead, cancellationToken)) { var sourceMD5 = response.ContentMD5(); var fullFileName = GetFullFileName(dataCollectionName, fileName); Directory.CreateDirectory(Path.GetDirectoryName(fullFileName)); using (var sourceStream = await response.Content.ReadAsStreamAsync()) { using (var fileStream = new FileStream(fullFileName, FileMode.Create)) { await CopyUtils.CopyAsync( (buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct), async (buffer, count, cancellationToken2) => await fileStream.WriteAsync(buffer, 0, count, cancellationToken2), bufferSize, cancellationToken ); } } using (var fileStream = new FileStream(fullFileName, FileMode.Open, FileAccess.Read)) { var newMD5 = await CopyUtils.GetMD5HashStringAsync(fileStream, bufferSize, cancellationToken); if ((!string.IsNullOrEmpty(sourceMD5)) && (!string.Equals(newMD5, sourceMD5, StringComparison.InvariantCultureIgnoreCase))) { throw new Exception("Invalid destination MD5"); } File.WriteAllText(GetMD5FileName(fullFileName), newMD5); } } }