public async Task <List <Partner> > FetchAllPartners(List <OrgFileETag> fileETags, string[] restrictToOrgIds) { await DoTokenCheck(); PartnershipWorkflow workflow = new PartnershipWorkflow(); await workflow.retrieveApiCatalog(_endpoint); workflow.ClearFilesETags(); if (fileETags != null) { foreach (var t in fileETags) { workflow.AddOrgFileETag(t.OrgId, t.Tag, t.CreatedDate); } } bool fetchNewFiles = (fileETags != null); var jdPartnerships = await workflow.getPartnerships(fetchNewFiles, restrictToOrgIds); List <Partner> results = new List <Partner>(); foreach (var p in jdPartnerships) { if (!string.IsNullOrEmpty(p.FromOrg.id)) { Partner dataPartner = new Partner(); dataPartner.Id = p.FromOrg.id; dataPartner.Name = p.FromOrg.name; dataPartner.MyLinkedOrgId = p.ToOrg.id; //dataPartner.PermissionGranted = (p.Permissions != null) ? p.Permissions.Any(x => x.status == "approved" && x.type == "productionAgronomicDetailData") : false; dataPartner.SharedFileCount = p.TotalFileCount; // (p.SharedFiles != null) ? p.SharedFiles.Count(x => x.archived.ToLower() == "false") : 0; dataPartner.PartnershipLink = p.links.Single(l => l.rel == "self").uri; string fileETag = ""; DateTime tagTime = DateTime.Now; workflow.GetOrgETagData(p.FromOrg.id, ref fileETag, ref tagTime); dataPartner.FileETag = fileETag; dataPartner.FilesETagCreatedDate = tagTime; results.Add(dataPartner); } } return(results); }
/// <summary> /// Downloads files for organizations specified in orgIds /// </summary> /// <param name="fileETags">eTags from previous download</param> /// <param name="filePath"></param> /// <param name="orgIds">Only files with an organization id in orgIds will be downloaded</param> /// <param name="downloadedFileIds">List of file ids that have previously been processed</param> /// <param name="callback">Callback executed when a file has been downloaded</param> /// <param name="progressCallback">Executed to update progress</param> /// <param name="errorCallback">Executed when a download error occurs</param> /// <returns></returns> public async Task <List <OrgFileETag> > DownloadOrganizationFiles(List <OrgFileETag> fileETags, string filePath, string[] orgIds, string[] downloadedFileIds, Action <FileDownloadedResult> callback, Action <FileDownloadProgress> progressCallback, Action <FileDownloadedResult> errorCallback) { await DoTokenCheck(); List <OrgFileETag> newETags = new List <OrgFileETag>(); List <OrgFileETag> failedETags = new List <OrgFileETag>(); PartnershipWorkflow p = new PartnershipWorkflow(); await p.retrieveApiCatalog(_endpoint); p.ClearFilesETags(); if (fileETags != null) { foreach (var t in fileETags) { p.AddOrgFileETag(t.OrgId, t.Tag, t.CreatedDate); } } bool fetchNewFiles = (fileETags != null); var partnerships = await p.getPartnerships(fetchNewFiles, orgIds); FileDownloadProgress progressResult = new FileDownloadProgress(); progressResult.CurrentFile = 1; progressResult.TotalFiles = 0; foreach (var partner in partnerships) { if (orgIds.Contains(partner.FromOrg.id) && partner.SharedFiles != null) { progressResult.TotalFiles += partner.SharedFiles.Count(f => !downloadedFileIds.Contains(f.id) && f.type != null && f.type.ToLower() == "hic" && f.archived.ToLower() == "false"); } } progressCallback(progressResult); foreach (var partner in partnerships) { string fileETag = ""; DateTime tagTime = DateTime.Now; p.GetOrgETagData(partner.FromOrg.id, ref fileETag, ref tagTime); newETags.Add(new OrgFileETag { CreatedDate = tagTime, OrgId = partner.FromOrg.id, Tag = fileETag }); if (orgIds.Contains(partner.FromOrg.id) && partner.SharedFiles != null) { foreach (var file in partner.SharedFiles.Where(f => !downloadedFileIds.Contains(f.id) && f.type != null && f.type.ToLower() == "hic" && f.archived.ToLower() == "false")) { string partnerFilePath = filePath.TrimEnd('\\') + "\\" + partner.FromOrg.id + "-" + file.id + "-source\\"; progressCallback(progressResult); if (!(await p.downloadFileInPiecesAndComputeMd5(partnerFilePath, file))) { var originalETag = fileETags.SingleOrDefault(f => f.OrgId == partner.FromOrg.id); //file download failed so add the eTag to the failed list if (originalETag != null) { failedETags.Add(originalETag); } errorCallback(new FileDownloadedResult { Filename = partnerFilePath + file.name, OrganizationID = partner.FromOrg.id, FileIdentifier = file.id }); } else { callback(new FileDownloadedResult { Filename = partnerFilePath + file.name, OrganizationID = partner.FromOrg.id, FileIdentifier = file.id }); } progressResult.CurrentFile++; } } } //if download failed for a certain org, reset the eTag back to the original one to ensure //the download gets retried next time if (failedETags.Count > 0) { foreach (var failedTag in failedETags) { foreach (var newTag in newETags) { if (newTag.OrgId == failedTag.OrgId) { newTag.Tag = failedTag.Tag; } } } } return(newETags); }