public static void DownloadTheResources(string projectId, Uri uri) { try { #region Get Resources ResoucesDetails resources = MongoHelper.GetAllTheResourcesOfWebite(projectId); if (resources == null) { throw new Exception("Resources Object was Null"); } //Get all the selected Domains (If unable to find, then only download the Root Uri resources) List <string> selectedDomains = new List <string>(); try { selectedDomains = MongoHelper.GetSelectedDomainNames(projectId); if (selectedDomains == null) { throw new Exception("Selected Domains is null"); } } catch (Exception ex) { //TODO: Log the Error selectedDomains = new List <string>(); } selectedDomains.Add(uri.Host.ToUpper()); #endregion #region Start Download FileDownloader downloader = new FileDownloader(uri); downloader.Context.DownloadedFileCallBackMethod = (AssetDetails asset, FileType fileType, Byte[] data, string contentType) => SaveInS3AndUpdateDB(projectId, asset, fileType, data, contentType); downloader.Context.NewAssetFoundDetailsCallBackMethod = (AssetDetails asset, FileType type) => NewAssetFound(projectId, asset, type); downloader.Context.Resources.Assets = resources.Assets; downloader.Context.Resources.Scripts = resources.Scripts; downloader.Context.Resources.Styles = resources.Styles; downloader.Context.Resources.SelectedDomains = selectedDomains; downloader.Context.Configuration.UserAgentString = EnvironmentConstants.ApplicationConfiguration.KitsuneUserAgent; downloader.Context.Configuration.ReadAndWriteTimeOut = 60000; downloader.Context.ErrorLogMethod = (LOGTYPE type, string message, Exception innerException) => LogError(type, message, innerException, projectId); downloader.Context.BatchCompletedCallBackMethod = () => Batchcompleted(downloader.Context, projectId); downloader.Execute(); #endregion } catch (Exception ex) { Log.Error(ex, $"Error while downloading the resources for projectId : {projectId}"); throw ex; } }
public static void ReplacePlaceHolder(string projectId, Uri uri) { try { List <FindAndReplace> findAndReplaces = new List <FindAndReplace>(); var projectConfig = APIHelper.GetProjectConfig(projectId); if (projectConfig != null) { var customSourceSyncSettings = projectConfig["custom_source_sync"]; if (customSourceSyncSettings == null) { //Unable to get the customsource settings } else { ProjectCustomSourceSyncSettings customSourceSyncDetails = JsonConvert.DeserializeObject <ProjectCustomSourceSyncSettings>(customSourceSyncSettings.ToString()); if (customSourceSyncDetails != null && customSourceSyncDetails.FindAndReplace != null) { findAndReplaces = customSourceSyncDetails.FindAndReplace; } } } ResoucesDetails resources = MongoHelper.GetAllTheResourcesOfWebite(projectId); List <AssetDetails> allResources = new List <AssetDetails>(); allResources.AddRange(resources.Styles); allResources.AddRange(resources.Scripts); allResources.AddRange(resources.Links); allResources.AddRange(resources.Assets); Parallel.ForEach(resources.Links, new ParallelOptions { MaxDegreeOfParallelism = 20 }, (Link, loopState) => { Replace(Link, allResources, projectId, findAndReplaces, true); Batchcompleted(projectId); if (StopReplacer) { loopState.Stop(); } }); if (StopReplacer) { throw new Exception("Replacer force stop"); } Parallel.ForEach(resources.Styles, new ParallelOptions { MaxDegreeOfParallelism = 20 }, (Style, loopState) => { Replace(Style, allResources, projectId, findAndReplaces, false); Batchcompleted(projectId); if (StopReplacer) { loopState.Stop(); } }); Parallel.ForEach(resources.Scripts, new ParallelOptions { MaxDegreeOfParallelism = 20 }, (Script, loopState) => { Replace(Script, allResources, projectId, findAndReplaces, false); Batchcompleted(projectId); if (StopReplacer) { loopState.Stop(); } }); if (StopReplacer) { throw new Exception("Replacer force stop"); } } catch (Exception ex) { Log.Error(ex, $"ProjectId:{projectId}, Message:Error Replacing PlaceHolder"); if (StopReplacer) { StopReplacer = false; throw ex; } } }