public Fetcher(DataSourceSyncRunner runner, int dataSourceId)
{
Requires.NonNull(runner, nameof(runner));
Runner = runner;
DS = Gdb.DataSources.Find(dataSourceId);
Requires.NonNull(DS, nameof(DS));
PopulateEvidence(dataSourceId);
BlobRootPath = $"{BlobStorageServices.GetDataSourceFetchItemRoot(runner.TenantName, dataSourceId)}{Runner.StartedAtUtc.ToYYYYMMDD()}/{Runner.StartedAtUtc.ToHHMMSS()}/";
}
public CommunicationsController(
BlobStorageServices blobs,
TraffkTenantModelDbContext db,
CurrentContextServices current,
ILogger logger
)
: base(AspHelpers.MainNavigationPageKeys.Messaging, db, current, logger)
{
Blobs = blobs;
}
public CommunicationBlastRunner(BlobStorageServices blobs, TraffkRdbContext rdb, ITrackingEmailer emailer)
{
Requires.NonNull(blobs, nameof(blobs));
Requires.NonNull(rdb, nameof(rdb));
Requires.NonNull(emailer, nameof(emailer));
Blobs = blobs;
Rdb = rdb;
Emailer = emailer;
}
public TenantsController(
BlobStorageServices blobs,
UserManager <ApplicationUser> userManager,
TraffkTenantModelDbContext db,
CurrentContextServices current,
ILogger logger,
IBackgroundJobClient backgrounder
)
: base(AspHelpers.MainNavigationPageKeys.Setup, db, current, logger)
{
UserManager = userManager;
Blobs = blobs;
Backgrounder = backgrounder;
}
public ReportingController(
TraffkTenantModelDbContext db,
CurrentContextServices current,
ILogger logger,
ICacher cacher,
IReportVisualService reportVisualService,
IBackgroundJobClient backgrounder,
ITraffkRecurringJobManager recurringJobManager,
BlobStorageServices blobStorageService
)
: base(AspHelpers.MainNavigationPageKeys.Reporting, db, current, logger, cacher)
{
ReportVisualService = reportVisualService;
Backgrounder = backgrounder;
RecurringJobManager = recurringJobManager;
BlobStorageService = blobStorageService;
IsOnline = ReportVisualService.IsOnline;
}
public TenantedJobRunner(
TraffkTenantModelDbContext db,
TraffkGlobalDbContext globalContext,
IJobInfoFinder jobInfoFinder,
CurrentTenantServices current,
ITableauAdminService tableauAdminService,
ILogger logger,
ITableauVisualServices tableauVisualService,
BlobStorageServices blobStorageService,
IBackgroundJobClient innerBackgroundJobClient)
: base(globalContext, jobInfoFinder, logger)
{
DB = db;
Current = current;
TableauAdminService = tableauAdminService;
TableauVisualService = tableauVisualService;
BlobStorageService = blobStorageService;
InnerBackgroundJobClient = innerBackgroundJobClient;
}
async T.Task IEtlJobs.ExecuteAsync(EtlPackages etlPackage, int dataSourceFetchId)
{
var fetch = await GlobalContext.DataSourceFetches.Include(z => z.DataSource).Include(z => z.DataSourceFetchDataSourceFetchItems).SingleAsync(z => z.DataSource.TenantId == this.JobInfo.TenantId && z.DataSourceFetchId == dataSourceFetchId);
Directory.CreateDirectory(FetchFolder);
var fetchItems = fetch.DataSourceFetchDataSourceFetchItems.Where(z => z.DataSourceFetchItemTypeStringValue == DataSourceFetchItem.DataSourceFetchItemTypes.Original.ToString());
Stuff.TaskWaitAllForEach(
fetchItems,
async fi => {
var fn = $"{FetchFolder}{fi.DataSourceFetchItemId}\\{fi.Name}";
Directory.CreateDirectory(Path.GetDirectoryName(fn));
var uri = new Uri(fi.Url);
if (uri.Scheme == BlobStorageServices.AzureBlobServiceProtocol)
{
uri = BlobStorageServices.GetReadonlySharedAccessSignatureUrl(BlobConfigOptions.Value, uri);
}
using (var client = HttpClientFactory.Create())
{
using (var st = await client.GetStreamAsync(uri))
{
using (var fst = File.Create(fn))
{
await st.CopyToAsync(fst);
}
}
if (MimeType.Application.Zip.DoesExtensionMatch(fn))
{
var unzipFolder = Path.Combine(Path.GetDirectoryName(fn), Path.GetFileNameWithoutExtension(fn));
ZipFile.ExtractToDirectory(fn, unzipFolder);
}
}
});
switch (etlPackage)
{
case EtlPackages.CmsGov:
await LoadCmsGovAsync(fetch);
return;
case EtlPackages.InternationalClassificationDiseases:
await LoadInternationalClassificationDiseasesAsync(fetch);
break;
case EtlPackages.NationalDrugCodes:
await LoadNationalDrugCodeAsync(fetch);
break;
case EtlPackages.ZipCodes:
await LoadZipCodesAsync(fetch);
break;
default:
throw new UnexpectedSwitchValueException(etlPackage);
}
}
private async Task FetchTheItemAsync(DataSourceFetche fetch, FileDetails details, DataSourceFetchItem.DataSourceFetchItemTypes dataSourceFetchItemType, DataSourceFetchItem parentFetchItem, Func <FileDetails, Task <string> > fetchAsync)
{
string tfn = null;
var item = new DataSourceFetchItem
{
DataSourceFetch = fetch,
DataSourceFetchItemType = dataSourceFetchItemType,
ParentDataSourceFetchItem = parentFetchItem,
Size = details.Size,
Name = details.Name,
};
item.DataSourceFetchItemProperties.LastModifiedAtUtc = details.LastModifiedAtUtc;
item.DataSourceFetchItemProperties.ContentMD5 = details.ContentMD5;
item.DataSourceFetchItemProperties.ETag = details.ETag;
try
{
Trace.WriteLine($"Checking {details.FullName} size={details.Size} LastWriteTimeUtc={details.LastModifiedAtUtc}");
var sameDataSourceReplicatedDataSourceFetchItem = FindEvidenceItems(details.CreateEvidence()).FirstOrDefault();
if (sameDataSourceReplicatedDataSourceFetchItem != null)
{
item.DataSourceFetchItemType = DataSourceFetchItem.DataSourceFetchItemTypes.Duplicate;
item.SameDataSourceReplicatedDataSourceFetchItem = sameDataSourceReplicatedDataSourceFetchItem;
return;
}
// Logger.LogInformation("Downloading", file.FullName, file.Length, tfn);
tfn = await fetchAsync(details);
using (var st = File.OpenRead(tfn))
{
item.Size = st.Length;
using (var muxer = new StreamMuxer(st, true))
{
var p = new BlobStorageServices.FileProperties
{
LastModifiedAtUtc = details.LastModifiedAtUtc
};
p.Metadata[BlobStorageServices.MetaKeyNames.SourcePath] = details.Folder;
p.Metadata[BlobStorageServices.MetaKeyNames.SourceFullName] = details.FullName;
var urns = new List <string>();
Parallel.ForEach(
new[]
{
Hash.CommonHashAlgorithmNames.Md5,
Hash.CommonHashAlgorithmNames.Sha1,
Hash.CommonHashAlgorithmNames.Sha512,
},
hashAlgName =>
{
var urn = Hash.Compute(muxer.OpenRead(), hashAlgName).Urn;
if (urn == null)
{
return; //yes... in some cases this somehow happens...
}
urns.Add(urn);
});
if (urns.Count > 0)
{
p.Metadata[BlobStorageServices.MetaKeyNames.Urns] = CSV.FormatLine(urns, false);
sameDataSourceReplicatedDataSourceFetchItem = FindEvidenceItems(urns).FirstOrDefault();
if (sameDataSourceReplicatedDataSourceFetchItem != null)
{
item.DataSourceFetchItemType = DataSourceFetchItem.DataSourceFetchItemTypes.Duplicate;
item.SameDataSourceReplicatedDataSourceFetchItem = sameDataSourceReplicatedDataSourceFetchItem;
return;
}
}
var res = await BlobStorageServices.StoreStreamAsync(
Runner.BlobConfig,
BlobStorageServices.ContainerNames.Secure,
$"{BlobRootPath}{details.Folder.Substring(1)}{details.Name}",
muxer.OpenRead(),
p,
amt => Trace.WriteLine($"Uploading {amt}/{muxer.Length}")
);
item.DataSourceFetchItemProperties = new DataSourceFetchItemProperties();
item.DataSourceFetchItemProperties.Set(p);
item.Url = res.Uri.ToString();
PopulateEvidence(item);
}
}
}
catch (Exception ex)
{
item.DataSourceFetchItemType = DataSourceFetchItem.DataSourceFetchItemTypes.Errored;
item.DataSourceFetchItemProperties.Error = new ExceptionError(ex);
Trace.WriteLine(ex);
}
finally
{
if (item != null)
{
await GdbLocker.GoAsync(async() => {
Gdb.DataSourceFetchItems.Add(item);
await Gdb.SaveChangesAsync();
});
}
}
var ext = Path.GetExtension(details.Name).ToLower();
if (ext == ".pgp" || details.Name.ToLower().Contains(".pgp."))
{
var name = details.Name;
if (name.ToLower().EndsWith(".pgp"))
{
name = name.Left(name.Length - 4);
}
else if (name.ToLower().EndsWith(".pgp.asc"))
{
name = name.Left(name.Length - 8);
}
else if (name.ToLower().Contains(".pgp."))
{
name = new Regex(@"\.pgp\.", RegexOptions.IgnoreCase).Replace(name, ".");
}
await FetchTheItemAsync(
fetch,
new FileDetails(details, name),
DataSourceFetchItem.DataSourceFetchItemTypes.Decrypted,
item,
async _ =>
{
var utfp = Path.GetTempFileName();
using (var st = File.OpenRead(tfn))
{
await Runner.DecryptAsync(st, utfp);
}
return(utfp);
}
);
}
else if (
MimeType.Application.Zip.DoesExtensionMatch(details.Name) &&
DS.DataSourceSettings.DecompressItems &&
dataSourceFetchItemType != DataSourceFetchItem.DataSourceFetchItemTypes.UnpackedRecompressedSingleton)
{
var relUnzipFolder = Path.GetFileNameWithoutExtension(details.Name);
var unzipFolder = Path.Combine(Path.GetDirectoryName(tfn), relUnzipFolder);
using (var st = File.OpenRead(tfn))
{
using (var za = new ZipArchive(st, ZipArchiveMode.Read))
{
if (za.Entries.Count < 2)
{
return;
}
}
}
ZipFile.ExtractToDirectory(tfn, unzipFolder);
await TaskWhenAllOneAtATime(
Directory.GetFiles(unzipFolder, "*.*", SearchOption.AllDirectories).ConvertAll(
unzipped =>
{
string rezipped = unzipped;
bool isRezipped = false;
if (!MimeType.Application.Zip.DoesExtensionMatch(unzipped))
{
rezipped = unzipped + MimeType.Application.Zip.PrimaryFileExtension;
using (var st = File.Create(rezipped))
{
using (var za = new ZipArchive(st, ZipArchiveMode.Create))
{
za.CreateEntryFromFile(unzipped, Path.GetFileName(unzipped));
}
isRezipped = true;
}
}
return(FetchTheItemAsync(
fetch,
new FileDetails(new FileInfo(rezipped), Path.Combine(details.Folder, relUnzipFolder)),
isRezipped ? DataSourceFetchItem.DataSourceFetchItemTypes.UnpackedRecompressedSingleton : DataSourceFetchItem.DataSourceFetchItemTypes.Unpacked,
item,
_ => Task.FromResult(rezipped)
));
}));
Stuff.Noop();
}
}
private async Task GetEm(string folder)
{
var urls = new[] {
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170501_20170531_20170607.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170501_20170531_20170608.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170501_20170531_20170607.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170601_20170630_20170710.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170501_20170531_20170607.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170601_20170630_20170709.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20170601_20170630_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20170701_20170731_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/sfgtestfile.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20170501_20170531_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20170601_20170630_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20170501_20170531_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170601_20170630_20170709.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20140101_20170430_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170501_20170531_20170607.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170501_20170531_20170608.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170601_20170630_20170710.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20170701_20170731_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20170701_20170731_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170601_20170630_20170709.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170601_20170630_20170709.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20170501_20170531_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/sfgtestfile.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20170601_20170630_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20140101_20170430_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20140101_20170430_.txt",
};
var blobConfig = ServiceProvider.GetRequiredService <BSSC>().BlobConfigOptions.Value;
var httpClientFactory = ServiceProvider.GetRequiredService <Utility.IHttpClientFactory>();
int x = 0;
foreach (var u in urls)
{
var uri = new Uri(u);
if (uri.Scheme == BlobStorageServices.AzureBlobServiceProtocol)
{
uri = BlobStorageServices.GetReadonlySharedAccessSignatureUrl(blobConfig, uri);
}
using (var client = httpClientFactory.Create())
{
var parts = uri.LocalPath.Replace("/", "\\").Split(new[] { '\\' }, StringSplitOptions.RemoveEmptyEntries);
string fn = Path.Combine(folder, parts.Skip(5).Format("\\"));
Trace.WriteLine($"download {++x}/{urls.Length} from {u} to {fn}");
using (var st = await client.GetStreamAsync(uri))
{
Directory.CreateDirectory(Path.GetDirectoryName(fn));
using (var fst = File.Create(fn))
{
await st.CopyToAsync(fst);
}
}
}
}
}
