public Fetcher(DataSourceSyncRunner runner, int dataSourceId) { Requires.NonNull(runner, nameof(runner)); Runner = runner; DS = Gdb.DataSources.Find(dataSourceId); Requires.NonNull(DS, nameof(DS)); PopulateEvidence(dataSourceId); BlobRootPath = $"{BlobStorageServices.GetDataSourceFetchItemRoot(runner.TenantName, dataSourceId)}{Runner.StartedAtUtc.ToYYYYMMDD()}/{Runner.StartedAtUtc.ToHHMMSS()}/"; }
public CommunicationsController( BlobStorageServices blobs, TraffkTenantModelDbContext db, CurrentContextServices current, ILogger logger ) : base(AspHelpers.MainNavigationPageKeys.Messaging, db, current, logger) { Blobs = blobs; }
public CommunicationBlastRunner(BlobStorageServices blobs, TraffkRdbContext rdb, ITrackingEmailer emailer) { Requires.NonNull(blobs, nameof(blobs)); Requires.NonNull(rdb, nameof(rdb)); Requires.NonNull(emailer, nameof(emailer)); Blobs = blobs; Rdb = rdb; Emailer = emailer; }
public TenantsController( BlobStorageServices blobs, UserManager <ApplicationUser> userManager, TraffkTenantModelDbContext db, CurrentContextServices current, ILogger logger, IBackgroundJobClient backgrounder ) : base(AspHelpers.MainNavigationPageKeys.Setup, db, current, logger) { UserManager = userManager; Blobs = blobs; Backgrounder = backgrounder; }
public ReportingController( TraffkTenantModelDbContext db, CurrentContextServices current, ILogger logger, ICacher cacher, IReportVisualService reportVisualService, IBackgroundJobClient backgrounder, ITraffkRecurringJobManager recurringJobManager, BlobStorageServices blobStorageService ) : base(AspHelpers.MainNavigationPageKeys.Reporting, db, current, logger, cacher) { ReportVisualService = reportVisualService; Backgrounder = backgrounder; RecurringJobManager = recurringJobManager; BlobStorageService = blobStorageService; IsOnline = ReportVisualService.IsOnline; }
public TenantedJobRunner( TraffkTenantModelDbContext db, TraffkGlobalDbContext globalContext, IJobInfoFinder jobInfoFinder, CurrentTenantServices current, ITableauAdminService tableauAdminService, ILogger logger, ITableauVisualServices tableauVisualService, BlobStorageServices blobStorageService, IBackgroundJobClient innerBackgroundJobClient) : base(globalContext, jobInfoFinder, logger) { DB = db; Current = current; TableauAdminService = tableauAdminService; TableauVisualService = tableauVisualService; BlobStorageService = blobStorageService; InnerBackgroundJobClient = innerBackgroundJobClient; }
async T.Task IEtlJobs.ExecuteAsync(EtlPackages etlPackage, int dataSourceFetchId) { var fetch = await GlobalContext.DataSourceFetches.Include(z => z.DataSource).Include(z => z.DataSourceFetchDataSourceFetchItems).SingleAsync(z => z.DataSource.TenantId == this.JobInfo.TenantId && z.DataSourceFetchId == dataSourceFetchId); Directory.CreateDirectory(FetchFolder); var fetchItems = fetch.DataSourceFetchDataSourceFetchItems.Where(z => z.DataSourceFetchItemTypeStringValue == DataSourceFetchItem.DataSourceFetchItemTypes.Original.ToString()); Stuff.TaskWaitAllForEach( fetchItems, async fi => { var fn = $"{FetchFolder}{fi.DataSourceFetchItemId}\\{fi.Name}"; Directory.CreateDirectory(Path.GetDirectoryName(fn)); var uri = new Uri(fi.Url); if (uri.Scheme == BlobStorageServices.AzureBlobServiceProtocol) { uri = BlobStorageServices.GetReadonlySharedAccessSignatureUrl(BlobConfigOptions.Value, uri); } using (var client = HttpClientFactory.Create()) { using (var st = await client.GetStreamAsync(uri)) { using (var fst = File.Create(fn)) { await st.CopyToAsync(fst); } } if (MimeType.Application.Zip.DoesExtensionMatch(fn)) { var unzipFolder = Path.Combine(Path.GetDirectoryName(fn), Path.GetFileNameWithoutExtension(fn)); ZipFile.ExtractToDirectory(fn, unzipFolder); } } }); switch (etlPackage) { case EtlPackages.CmsGov: await LoadCmsGovAsync(fetch); return; case EtlPackages.InternationalClassificationDiseases: await LoadInternationalClassificationDiseasesAsync(fetch); break; case EtlPackages.NationalDrugCodes: await LoadNationalDrugCodeAsync(fetch); break; case EtlPackages.ZipCodes: await LoadZipCodesAsync(fetch); break; default: throw new UnexpectedSwitchValueException(etlPackage); } }
private async Task FetchTheItemAsync(DataSourceFetche fetch, FileDetails details, DataSourceFetchItem.DataSourceFetchItemTypes dataSourceFetchItemType, DataSourceFetchItem parentFetchItem, Func <FileDetails, Task <string> > fetchAsync) { string tfn = null; var item = new DataSourceFetchItem { DataSourceFetch = fetch, DataSourceFetchItemType = dataSourceFetchItemType, ParentDataSourceFetchItem = parentFetchItem, Size = details.Size, Name = details.Name, }; item.DataSourceFetchItemProperties.LastModifiedAtUtc = details.LastModifiedAtUtc; item.DataSourceFetchItemProperties.ContentMD5 = details.ContentMD5; item.DataSourceFetchItemProperties.ETag = details.ETag; try { Trace.WriteLine($"Checking {details.FullName} size={details.Size} LastWriteTimeUtc={details.LastModifiedAtUtc}"); var sameDataSourceReplicatedDataSourceFetchItem = FindEvidenceItems(details.CreateEvidence()).FirstOrDefault(); if (sameDataSourceReplicatedDataSourceFetchItem != null) { item.DataSourceFetchItemType = DataSourceFetchItem.DataSourceFetchItemTypes.Duplicate; item.SameDataSourceReplicatedDataSourceFetchItem = sameDataSourceReplicatedDataSourceFetchItem; return; } // Logger.LogInformation("Downloading", file.FullName, file.Length, tfn); tfn = await fetchAsync(details); using (var st = File.OpenRead(tfn)) { item.Size = st.Length; using (var muxer = new StreamMuxer(st, true)) { var p = new BlobStorageServices.FileProperties { LastModifiedAtUtc = details.LastModifiedAtUtc }; p.Metadata[BlobStorageServices.MetaKeyNames.SourcePath] = details.Folder; p.Metadata[BlobStorageServices.MetaKeyNames.SourceFullName] = details.FullName; var urns = new List <string>(); Parallel.ForEach( new[] { Hash.CommonHashAlgorithmNames.Md5, Hash.CommonHashAlgorithmNames.Sha1, Hash.CommonHashAlgorithmNames.Sha512, }, hashAlgName => { var urn = Hash.Compute(muxer.OpenRead(), hashAlgName).Urn; if (urn == null) { return; //yes... in some cases this somehow happens... } urns.Add(urn); }); if (urns.Count > 0) { p.Metadata[BlobStorageServices.MetaKeyNames.Urns] = CSV.FormatLine(urns, false); sameDataSourceReplicatedDataSourceFetchItem = FindEvidenceItems(urns).FirstOrDefault(); if (sameDataSourceReplicatedDataSourceFetchItem != null) { item.DataSourceFetchItemType = DataSourceFetchItem.DataSourceFetchItemTypes.Duplicate; item.SameDataSourceReplicatedDataSourceFetchItem = sameDataSourceReplicatedDataSourceFetchItem; return; } } var res = await BlobStorageServices.StoreStreamAsync( Runner.BlobConfig, BlobStorageServices.ContainerNames.Secure, $"{BlobRootPath}{details.Folder.Substring(1)}{details.Name}", muxer.OpenRead(), p, amt => Trace.WriteLine($"Uploading {amt}/{muxer.Length}") ); item.DataSourceFetchItemProperties = new DataSourceFetchItemProperties(); item.DataSourceFetchItemProperties.Set(p); item.Url = res.Uri.ToString(); PopulateEvidence(item); } } } catch (Exception ex) { item.DataSourceFetchItemType = DataSourceFetchItem.DataSourceFetchItemTypes.Errored; item.DataSourceFetchItemProperties.Error = new ExceptionError(ex); Trace.WriteLine(ex); } finally { if (item != null) { await GdbLocker.GoAsync(async() => { Gdb.DataSourceFetchItems.Add(item); await Gdb.SaveChangesAsync(); }); } } var ext = Path.GetExtension(details.Name).ToLower(); if (ext == ".pgp" || details.Name.ToLower().Contains(".pgp.")) { var name = details.Name; if (name.ToLower().EndsWith(".pgp")) { name = name.Left(name.Length - 4); } else if (name.ToLower().EndsWith(".pgp.asc")) { name = name.Left(name.Length - 8); } else if (name.ToLower().Contains(".pgp.")) { name = new Regex(@"\.pgp\.", RegexOptions.IgnoreCase).Replace(name, "."); } await FetchTheItemAsync( fetch, new FileDetails(details, name), DataSourceFetchItem.DataSourceFetchItemTypes.Decrypted, item, async _ => { var utfp = Path.GetTempFileName(); using (var st = File.OpenRead(tfn)) { await Runner.DecryptAsync(st, utfp); } return(utfp); } ); } else if ( MimeType.Application.Zip.DoesExtensionMatch(details.Name) && DS.DataSourceSettings.DecompressItems && dataSourceFetchItemType != DataSourceFetchItem.DataSourceFetchItemTypes.UnpackedRecompressedSingleton) { var relUnzipFolder = Path.GetFileNameWithoutExtension(details.Name); var unzipFolder = Path.Combine(Path.GetDirectoryName(tfn), relUnzipFolder); using (var st = File.OpenRead(tfn)) { using (var za = new ZipArchive(st, ZipArchiveMode.Read)) { if (za.Entries.Count < 2) { return; } } } ZipFile.ExtractToDirectory(tfn, unzipFolder); await TaskWhenAllOneAtATime( Directory.GetFiles(unzipFolder, "*.*", SearchOption.AllDirectories).ConvertAll( unzipped => { string rezipped = unzipped; bool isRezipped = false; if (!MimeType.Application.Zip.DoesExtensionMatch(unzipped)) { rezipped = unzipped + MimeType.Application.Zip.PrimaryFileExtension; using (var st = File.Create(rezipped)) { using (var za = new ZipArchive(st, ZipArchiveMode.Create)) { za.CreateEntryFromFile(unzipped, Path.GetFileName(unzipped)); } isRezipped = true; } } return(FetchTheItemAsync( fetch, new FileDetails(new FileInfo(rezipped), Path.Combine(details.Folder, relUnzipFolder)), isRezipped ? DataSourceFetchItem.DataSourceFetchItemTypes.UnpackedRecompressedSingleton : DataSourceFetchItem.DataSourceFetchItemTypes.Unpacked, item, _ => Task.FromResult(rezipped) )); })); Stuff.Noop(); } }
private async Task GetEm(string folder) { var urls = new[] { "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170501_20170531_20170607.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170501_20170531_20170608.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170501_20170531_20170607.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170601_20170630_20170710.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170501_20170531_20170607.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170601_20170630_20170709.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20170601_20170630_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20170701_20170731_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/sfgtestfile.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20170501_20170531_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20170601_20170630_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20170501_20170531_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170601_20170630_20170709.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20140101_20170430_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170501_20170531_20170607.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170501_20170531_20170608.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170601_20170630_20170710.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.ctl.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20170701_20170731_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20170701_20170731_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170601_20170630_20170709.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170601_20170630_20170709.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20170501_20170531_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/sfgtestfile.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.txt.pgp", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.ctl", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20170601_20170630_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20140101_20170430_.txt", "abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20140101_20170430_.txt", }; var blobConfig = ServiceProvider.GetRequiredService <BSSC>().BlobConfigOptions.Value; var httpClientFactory = ServiceProvider.GetRequiredService <Utility.IHttpClientFactory>(); int x = 0; foreach (var u in urls) { var uri = new Uri(u); if (uri.Scheme == BlobStorageServices.AzureBlobServiceProtocol) { uri = BlobStorageServices.GetReadonlySharedAccessSignatureUrl(blobConfig, uri); } using (var client = httpClientFactory.Create()) { var parts = uri.LocalPath.Replace("/", "\\").Split(new[] { '\\' }, StringSplitOptions.RemoveEmptyEntries); string fn = Path.Combine(folder, parts.Skip(5).Format("\\")); Trace.WriteLine($"download {++x}/{urls.Length} from {u} to {fn}"); using (var st = await client.GetStreamAsync(uri)) { Directory.CreateDirectory(Path.GetDirectoryName(fn)); using (var fst = File.Create(fn)) { await st.CopyToAsync(fst); } } } } }