public void WebsiteCrawl()
        {
            List <KrawlerException> Errors = new List <KrawlerException>();
            string url = "https://giphy.com/";
            Uri    uri = new Uri(url);

            Krawler krawler = new Krawler(uri);

            krawler.KrawlContext.ErrorLogMethod = (LOGTYPE x, string y, Exception ex) => Errors.Add(new KrawlerException {
                ErrorType = x, Ex = ex, Message = y
            });
            //krawler.KrawlContext.SaveHtmlMethod = (x, y) => SaveInS3("test1"+x, y);
            krawler.Krawl();
        }
        public static void AnalyseTheWebsite(string projectId, Uri uri, bool IsDeepCrawl)
        {
            try
            {
                string        regexToIgnore               = null;
                string        regexToInclude              = null;
                string        ignore_link_conversion      = null;
                List <string> include_static_asset_folder = null;
                Log.Information("Getting project config");
                var result = APIHelper.GetProjectConfig(projectId);
                if (result != null)
                {
                    var customSourceSyncSettings = result["custom_source_sync"];
                    if (customSourceSyncSettings == null)
                    {
                        //Unable to get the customsource settings
                    }
                    else
                    {
                        Log.Information("ProjectConfig : " + JsonConvert.SerializeObject(customSourceSyncSettings));
                        try
                        {
                            ProjectCustomSourceSyncSettings excludeList = JsonConvert.DeserializeObject <ProjectCustomSourceSyncSettings>(customSourceSyncSettings.ToString());
                            if (!String.IsNullOrEmpty(excludeList.Exclude))
                            {
                                regexToIgnore = excludeList.Exclude;
                            }
                            if (!string.IsNullOrEmpty(excludeList.Include))
                            {
                                regexToInclude = excludeList.Include;
                            }
                            ignore_link_conversion      = excludeList.IgnoreLinkConversion;
                            include_static_asset_folder = excludeList.IncludeStaticAssets;
                            if (excludeList.IncludeStaticAssetApis != null)
                            {
                                var assetList = Utils.GetAllStaticAssetList(excludeList.IncludeStaticAssetApis);
                                if (assetList != null)
                                {
                                    if (include_static_asset_folder == null)
                                    {
                                        include_static_asset_folder = new List <string>();
                                    }
                                    include_static_asset_folder.AddRange(assetList);
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            Log.Error("ProjectCustomSourceSync : " + ex.Message);
                        }
                    }
                }

                Krawler krawler = new Krawler(uri);
                krawler.KrawlContext.ProcessedHtmlCallBackMethod   = (filePath, htmlString) => SaveInS3(projectId, projectId + filePath, htmlString);
                krawler.KrawlContext.UpdatedResoucesCallBackMethod = (resourceObject) => UpdateAnalyseDetails(projectId, resourceObject);
                krawler.KrawlContext.BatchCompletedCallBackMethod  = (CrawledPagesCount) => Batchcompleted(krawler, projectId, CrawledPagesCount);
                krawler.KrawlContext.ErrorLogMethod = (x, y, z) => LogError(x, y, z, projectId);
                krawler.KrawlContext.Configuration.MaxConcurrentThreads = 20;
                krawler.KrawlContext.Configuration.IsDeepCrawl          = IsDeepCrawl;
                krawler.KrawlContext.Configuration.UserAgentString      = EnvironmentConstants.ApplicationConfiguration.KitsuneUserAgent;

                if (!String.IsNullOrEmpty(ignore_link_conversion))
                {
                    krawler.KrawlContext.Resources.IgnoreFileNameChangeRegex = new Regex(ignore_link_conversion);
                }
                krawler.KrawlContext.Resources.ExcludeFilesRegex      = regexToIgnore;
                krawler.KrawlContext.Resources.IncludeFilesRegex      = regexToInclude;
                krawler.KrawlContext.Resources.IncludeStaticAssetList = include_static_asset_folder;
                krawler.Krawl();
            }
            catch (Exception ex)
            {
                Log.Error(ex, $"Error Analysing the Url for projectId : {projectId}");
                //TODO: Throw Error
                throw ex;
            }
        }