/// <summary>
        /// Create new ActivityLog
        /// Pushes the ActivityLogModel to SQS
        /// </summary>
        /// <param name="activityLog"></param>
        public void LogActivity(CreateActivityLog activityLog)
        {
            if (activityLog == null)
            {
                throw new ArgumentNullException(nameof(activityLog));
            }
            if (String.IsNullOrEmpty(activityLog.ResourceId))
            {
                throw new ArgumentNullException(nameof(activityLog.ResourceId));
            }
            if (String.IsNullOrEmpty(activityLog.ActivityId))
            {
                throw new ArgumentNullException(nameof(activityLog.ActivityId));
            }

            ActivityLogSQSModel sqsmodel = new ActivityLogSQSModel
            {
                ResourceId = activityLog.ResourceId,
                ActivityId = activityLog.ActivityId,
                CreatedOn  = activityLog.ActivityCreatedOn,
                Params     = activityLog.Params
            };
            AmazonSQSQueueHandlers <ActivityLogSQSModel> sqsHandler = new AmazonSQSQueueHandlers <ActivityLogSQSModel>(AmazonAWSConstants.ActivitySQSUrl);
            string result = sqsHandler.PushMessageToQueue(sqsmodel, AmazonAWSConstants.SQSAccessKey, AmazonAWSConstants.SQSSecretKey);

            if (result == null)
            {
                throw new Exception($"Error pushing ResourceId : {activityLog.ResourceId} and ActivityId : {activityLog.ActivityId} to SQS");
            }
        }
        public static void OptimisationCompleted(string projectId)
        {
            try
            {
                if (_kitsuneServer == null)
                {
                    InitializeConnection();
                }

                var buildStatsCollection = _kitsuneDatabase.GetCollection <KitsuneBuildStatus>(BuildStatusCollectionName);


                #region Send Migration Report

                var amazonSqsHanlder = new AmazonSQSQueueHandlers <TranspilerFunctionSQSModel>(AmazonAWSConstants.TranspilerSQSUrl);
                var project          = GetProjectDetails(projectId);
                TranspilerFunctionSQSModel sqsModel = new TranspilerFunctionSQSModel()
                {
                    ProjectId = projectId,
                    UserEmail = project.UserEmail
                };
                amazonSqsHanlder.PushMessageToQueue(sqsModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey);

                #endregion
            }
            catch (Exception ex)
            {
                //LOG
            }
        }
예제 #3
0
        static void Main(string[] args)
        {
            Console.WriteLine("Kitsune Compiler Service Started");

            if (isDebug)
            {
                Console.WriteLine("KString service started");
            }
            while (true)
            {
                try
                {
                    var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <KStringQueueModel>(kStringQueueUrl);


                    var task = amazonCompilerSqsQueueHandler.ReceiveMessageFromQueue();
                    try
                    {
                        if (task != null && task.MessageBody != null)
                        {
                            var keywordList = KStringHelper.ExtractKeyword(task.MessageBody.KString);
                            if (keywordList != null && keywordList.Any())
                            {
                                var result = KStringHelper.UpdateKeywordsToKitsuneDB(task.MessageBody.SchemaName, task.MessageBody.UserId, task.MessageBody.KID, task.MessageBody.ReferenceId, task.MessageBody.KString, keywordList);
                                if (!string.IsNullOrEmpty(result))
                                {
                                    Console.WriteLine($"Keyword Extracted : {JsonConvert.SerializeObject(task.MessageBody)}");
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine(String.Format("Error during kstring queue processing, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace));
                    }
                    finally
                    {
                        if (task != null)
                        {
                            amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(task);
                            Console.WriteLine($"Message removed : {task.MessageBody.SchemaName}");
                        }
                    }
                }
                catch (Exception ex)
                {
                    //  EventLogger.LogTrace(String.Format("Error during compilation, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace), null, null);
                    Console.WriteLine(String.Format("Error during kstring processing, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace));
                }
                finally
                {
                }
            }
        }
        public static void TranspilationCompleted(string projectId)
        {
            try
            {
                if (_kitsuneServer == null)
                {
                    InitializeConnection();
                }

                //Update the proejct status to IDLE on transpilation success
                UpdateKitsuneProjectStatus(new UpdateKitsuneProjectStatusRequestModel
                {
                    ProjectId     = projectId,
                    ProjectStatus = ProjectStatus.IDLE
                });

                #region Send Migration Report

                var amazonSqsHanlder             = new AmazonSQSQueueHandlers <ReportGeneratorSQSModel>(AmazonAWSConstants.MigrationReportSQSUrl);
                ReportGeneratorSQSModel sqsModel = new ReportGeneratorSQSModel()
                {
                    ProjectId = projectId
                };
                amazonSqsHanlder.PushMessageToQueue(sqsModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey);

                #endregion

                //Call screenshot API
                CreateAndUpdateDemoScreenShot(projectId);

                //Create Routing Tree
                APIHelper.CreateProjectRoute(projectId, true);

                try
                {
                    ProjectConfigHelper projectConfigHelper = new ProjectConfigHelper(projectId);
                    var isPublish = projectConfigHelper.IsAutoPublishEnabled();
                    //Auto Publish
                    if (isPublish)
                    {
                        //Call publish API
                    }
                }
                catch { }
            }
            catch (Exception ex)
            {
                //LOG
            }
        }
        internal static string PushSitemapGenerationTaskToSQS(SitemapGenerationTaskModel sitemapModel)
        {
            try
            {
                if (sitemapModel == null || String.IsNullOrEmpty(sitemapModel.WebsiteId))
                {
                    return(null);
                }

                if (String.IsNullOrEmpty(sitemapModel.ProjectId))
                {
                    sitemapModel.ProjectId = MongoConnector.GetProjectIdFromWebsiteId(sitemapModel.WebsiteId);
                    if (String.IsNullOrEmpty(sitemapModel.ProjectId))
                    {
                        return(null);
                    }
                }

                if (String.IsNullOrEmpty(sitemapModel.WebsiteUrl))
                {
                    //exclude paths
                    //Handle exception for config
                    // var websiteConfig = MongoConnector.GetProjectConfig(new GetProjectConfigRequestModel() { ProjectId = sitemapModel.ProjectId });
                    sitemapModel.WebsiteUrl = MongoConnector.GetWebsiteUrlFromWebsiteId(sitemapModel.WebsiteId)?.ToLower();
                }

                //To-Do fetch the entire config from db or baseplugin
                var requestModel = new SitemapGenerationTaskQueueModel()
                {
                    param = new SitemapGenerationTaskParameterModel()
                    {
                        access_key = EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey,
                        secret     = EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey,
                        region     = "ap-south-1",
                        acl        = "public-read",
                        bucket     = "kitsune-resource-production",
                        key        = $"{sitemapModel.ProjectId}/websiteresources/{sitemapModel.WebsiteId}/sitemap.xml",
                        url        = sitemapModel.WebsiteUrl
                    },
                    type = 2
                };

                var sqsHanlder = new AmazonSQSQueueHandlers <SitemapGenerationTaskQueueModel>(AmazonAWSConstants.SitemapServiceSQSUrl);
                return(sqsHanlder.PushMessageToQueue(requestModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey,
                                                     EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey, Amazon.RegionEndpoint.APSouth1));
            }
            catch { }
            return(null);
        }
        public static bool UpdateBuildStatus(string user, string projectId, int buildVersion)
        {
            var amazonBuildSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(Program.ServiceConfiguration.GetSection("KitsuneBuildSQSUrl").Value);

            try
            {
                var queueResult = amazonBuildSqsQueueHandler.PushMessageToQueue(new CompilerServiceSQSModel
                {
                    ProjectId    = projectId,
                    BuildVersion = buildVersion
                }, Program.ServiceConfiguration.GetSection("AWSAccessKey").Value, Program.ServiceConfiguration.GetSection("AWSSecretKey").Value, RegionEndpoint.GetBySystemName(Program.ServiceConfiguration.GetSection("AWSRegion").Value));

                if (!string.IsNullOrEmpty(queueResult))
                {
                    var jsonData = JsonConvert.SerializeObject(new { ProjectId = projectId, BuildVersion = buildVersion, Stage = BuildStatus.QueuedBuild });

                    var result = client.PostAsync(string.Format(buildUrl, user), new StringContent(jsonData, Encoding.UTF8, "application/json")).Result;
                    //Add to BuildSQS
                    if (result != null && result.StatusCode == System.Net.HttpStatusCode.OK)
                    {
                        return(true);
                    }
                    else
                    {
                        Log.Error(String.Format("Unable to update the project status : ProjectId {0}", projectId));
                        return(false);
                    }
                }
                else
                {
                    Log.Error(String.Format("Push to build queue failed : ProjectId {0}", projectId));
                    return(false);
                }
            }
            catch (Exception ex)
            {
                Log.Error(String.Format("Push to build queue failed, Error : {0}", ex.Message), projectId, buildVersion.ToString());
                Console.WriteLine(String.Format("Push to build queue failed, Error : {0}", ex.Message), projectId, buildVersion.ToString());
                return(false);
            }
        }
        /// <summary>
        /// Create new ActivityLog
        /// Pushes the ACtivityLogModel to SQS asynchronously
        /// For internal use
        /// </summary>
        /// <param name="activityLog"></param>
        public static async Task LogActivityAync(CreateActivityLog activityLog)
        {
            if (activityLog == null)
            {
                throw new ArgumentNullException(nameof(activityLog));
            }
            if (String.IsNullOrEmpty(activityLog.ResourceId))
            {
                throw new ArgumentNullException(nameof(activityLog.ResourceId));
            }
            if (String.IsNullOrEmpty(activityLog.ActivityId))
            {
                throw new ArgumentNullException(nameof(activityLog.ActivityId));
            }

            ActivityLogSQSModel sqsmodel = new ActivityLogSQSModel
            {
                ResourceId = activityLog.ResourceId,
                ActivityId = activityLog.ActivityId,
                CreatedOn  = activityLog.ActivityCreatedOn,
                Params     = activityLog.Params
            };

            try
            {
                AmazonSQSQueueHandlers <ActivityLogSQSModel> sqsHandler = new AmazonSQSQueueHandlers <ActivityLogSQSModel>(AmazonAWSConstants.ActivitySQSUrl);
                var result = await sqsHandler.PushMessageToQueueAsync(sqsmodel, AmazonAWSConstants.SQSAccessKey, AmazonAWSConstants.SQSSecretKey);

                if (!result.Equals(HttpStatusCode.OK))
                {
                    throw new Exception($"Error pushing ResourceId : {activityLog.ResourceId} and ActivityId : {activityLog.ActivityId} to SQS");
                }
            }
            catch (Exception ex)
            {
                //TODO
                //Log Exception
            }
        }
        static void Main(string[] args)
        {
#if DEBUG
            var builder = new ConfigurationBuilder()
                          .SetBasePath(Directory.GetCurrentDirectory())
                          .AddJsonFile("appsettings.json", true, true)
                          .AddEnvironmentVariables();
#else
            var builder = new ConfigurationBuilder()
                          .AddJsonFile($"appsettings.Production.json", true, true)
                          .AddEnvironmentVariables();
#endif
            ServiceConfiguration = builder.Build();
            string myIP = null;
            try
            {
                string hostName = Dns.GetHostName(); // Retrive the Name of HOST
                                                     // Get the IP
                myIP = Dns.GetHostEntry(hostName)?.AddressList?.Where(x => x.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)?.FirstOrDefault()?.ToString();
            }
            catch
            {
            }
            Logger.InitLogger(ServiceConfiguration.GetSection("AWSAccessKey").Value,
                              ServiceConfiguration.GetSection("AWSSecretKey").Value,
                              ServiceConfiguration.GetSection("CloudWatchLogGroup").Value,
                              myIP);

            Console.WriteLine("Started");
            Log.Information("Compiler service started");
            Console.WriteLine(ServiceConfiguration.GetSection("KitsuneCompilerSQSUrl").Value);
            Log.Information(ServiceConfiguration.GetSection("KitsuneCompilerSQSUrl").Value);



            var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(ServiceConfiguration.GetSection("KitsuneCompilerSQSUrl").Value);
            AmazonAWSHelpers.Models.AmazonSQSMessageQueueModel <CompilerServiceSQSModel> task = null;
            string projectId    = string.Empty;
            int    buildVersion = 0;
            string user         = string.Empty;
            var    errors       = new List <BuildError>();
            List <CompileResult> buildStatus = null;
            while (true)
            {
                try
                {
                    Console.WriteLine($"Polling from the queue : {DateTime.UtcNow.ToString()}");
                    Log.Information($"Polling from the queue : {DateTime.UtcNow.ToString()}");

                    task = amazonCompilerSqsQueueHandler.ReceiveMessageFromQueue(ServiceConfiguration.GetSection("AWSAccessKey").Value, ServiceConfiguration.GetSection("AWSSecretKey").Value,
                                                                                 RegionEndpoint.GetBySystemName(ServiceConfiguration.GetSection("AWSRegion").Value));

                    if (task != null && task.MessageBody != null)
                    {
                        Console.Clear();
                        projectId    = task.MessageBody.ProjectId;
                        buildVersion = task.MessageBody.BuildVersion;
                        user         = task.MessageBody.UserEmail;


                        //Update the kitsune project status to building
                        if (APIHelpers.UpdateProjectStatus(user, projectId, buildVersion.ToString()))
                        {
                            Console.WriteLine(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion), projectId, buildVersion.ToString());
                            Log.Information(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion), projectId, buildVersion.ToString());

                            buildStatus = new BuildAndRunHelper().BuildProject(user, projectId, bool.Parse(ServiceConfiguration.GetSection("_isDev").Value), _currentCompilerVersion);
                            if (buildStatus != null && buildStatus.Any())
                            {
                                errors = new List <BuildError>();
                                foreach (var error in buildStatus)
                                {
                                    if (!error.Success)
                                    {
                                        errors.AddRange(error.ErrorMessages.Select(x => new BuildError
                                        {
                                            Column          = x.LinePosition,
                                            Line            = x.LineNumber,
                                            Message         = x.Message,
                                            ErrorStackTrace = x.Message,
                                            SourceMethod    = "KitsuneCompiler",
                                            SourcePath      = error.PageName
                                        }));
                                    }
                                }
                                APIHelpers.UpdateProjectErrorStatus(user, projectId, buildVersion, errors);
                                Console.WriteLine(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
                                Log.Error(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
                                foreach (var err in errors)
                                {
                                    Console.WriteLine(JsonConvert.SerializeObject(err));
                                    Log.Error(JsonConvert.SerializeObject(err));
                                }
                                buildStatus = null;
                            }
                            else
                            {
                                //UPDATE the project status
                                if (APIHelpers.UpdateBuildStatus(user, projectId, buildVersion))
                                {
                                    Console.WriteLine(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
                                    Log.Information(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
                                }
                            }
                        }
                        amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(task, ServiceConfiguration.GetSection("AWSAccessKey").Value, ServiceConfiguration.GetSection("AWSSecretKey").Value, RegionEndpoint.GetBySystemName(ServiceConfiguration.GetSection("AWSRegion").Value));
                        Console.WriteLine(string.Format("Message removed : {0}", task?.MessageBody?.ProjectId));
                        Log.Information(string.Format("Message removed : {0}", task?.MessageBody?.ProjectId));
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(String.Format("Error during compilation, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace));
                    Log.Error(String.Format("Error during compilation, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace));
                }
            }
        }
        private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
        {
            context.Logger.LogLine($"Processed message {message.Body}");
            context.Logger.LogLine(Environment.GetEnvironmentVariable("SQS_AWS_REGION"));
            context.Logger.LogLine(Directory.GetCurrentDirectory() + "/Files");

            try
            {
                var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(Environment.GetEnvironmentVariable("SQS_URL"));

                var sqsModel = JsonConvert.DeserializeObject <CompilerServiceSQSModel>(message.Body);

                var errors = new List <BuildError>();
                List <CompileResult> buildStatus = null;

                var projectId    = sqsModel.ProjectId;
                var buildVersion = sqsModel.BuildVersion;
                var user         = sqsModel.UserEmail;


                //Update the kitsune project status to building
                if (APIHelpers.UpdateProjectStatus(user, projectId, buildVersion.ToString()))
                {
                    context.Logger.LogLine(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion));

                    buildStatus = new BuildAndRunHelper().BuildProject(user, projectId, bool.Parse(Environment.GetEnvironmentVariable("IS_DEVELOPMENT_VERSION")), 1);
                    if (buildStatus != null && buildStatus.Any())
                    {
                        errors = new List <BuildError>();
                        foreach (var error in buildStatus)
                        {
                            if (!error.Success)
                            {
                                errors.AddRange(error.ErrorMessages.Select(x => new BuildError
                                {
                                    Column          = x.LinePosition,
                                    Line            = x.LineNumber,
                                    Message         = x.Message,
                                    ErrorStackTrace = x.Message,
                                    SourceMethod    = "KitsuneCompiler",
                                    SourcePath      = error.PageName
                                }));
                            }
                        }
                        APIHelpers.UpdateProjectErrorStatus(user, projectId, buildVersion, errors);
                        Console.WriteLine(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
                        Log.Error(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
                        foreach (var err in errors)
                        {
                            Console.WriteLine(JsonConvert.SerializeObject(err));
                            Log.Error(JsonConvert.SerializeObject(err));
                        }
                        buildStatus = null;
                    }
                    else
                    {
                        //UPDATE the project status
                        if (APIHelpers.UpdateBuildStatus(user, projectId, buildVersion))
                        {
                            Console.WriteLine(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
                            Log.Information(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
                        }
                    }
                }

                var response = amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(new AmazonSQSMessageQueueModel <CompilerServiceSQSModel>
                {
                    MessageBody       = JsonConvert.DeserializeObject <CompilerServiceSQSModel>(message.Body),
                    MessageId         = message.MessageId,
                    ReceiptHandle     = message.ReceiptHandle,
                    MessageAttributes = message.Attributes
                });

                context.Logger.LogLine(response);
            }
            catch { }

            await Task.CompletedTask;
        }
예제 #10
0
        public static void Process()
        {
            try
            {
                while (true)
                {
                    string projectId = String.Empty;
                    KitsuneKrawlerStatusCompletion stage = KitsuneKrawlerStatusCompletion.Error;
                    try
                    {
                        var amazonSqsQueueHandler = new AmazonSQSQueueHandlers <KrawlSQSModel>(EnvironmentConstants.ApplicationConfiguration.CrawlerSQSUrl);
                        //var task = amazonSqsQueueHandler.ReceiveMessageFromQueue(awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                        var task = new AmazonAWSHelpers.Models.AmazonSQSMessageQueueModel <KrawlSQSModel>()
                        {
                            MessageBody = new KrawlSQSModel
                            {
                                ProjectId = "5ce4ef18abc486000121acb8",
                                ReCrawl   = true
                            }
                        };
                        if (task != null)
                        {
                            projectId = task.MessageBody.ProjectId;
                            if (!String.IsNullOrEmpty(projectId))
                            {
                                try
                                {
                                    #region Initiate Logger

                                    Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup, projectId);

                                    #endregion

                                    #region Before Process

                                    try
                                    {
                                        ServiceInformationHelper serviceInfo = new ServiceInformationHelper();
                                        Log.Information($"ProjectId : {projectId}, IP: {serviceInfo.GetInstancePrivateIpAddress()}");
                                    }
                                    catch { }

                                    Uri uri = null;
                                    //Get the Details from DB
                                    var crawlDetails = MongoHelper.GetCrawlingDetails(projectId);
                                    if (crawlDetails == null)
                                    {
                                        throw new Exception("CrawlDetails was null");
                                    }
                                    if (!Uri.TryCreate(crawlDetails.Url, UriKind.Absolute, out uri))
                                    {
                                        throw new Exception(String.Format("Error Creating Uri from Url : {0}", crawlDetails.Url));
                                    }
                                    stage = crawlDetails.Stage;

                                    #endregion

                                    #region Process

                                    var isTaskCompleted = false;
                                    Log.Information($"Started, Stage: {stage.ToString()}");
                                    try
                                    {
                                        switch (stage)
                                        {
                                        case KitsuneKrawlerStatusCompletion.Initialising:
                                            InitialiseKrawlerStageHelper.InitialiseKrawler(projectId, uri);
                                            isTaskCompleted = true;
                                            break;

                                        case KitsuneKrawlerStatusCompletion.IdentifyingAllAssetsAndDownloadingWebpage:
                                            MigrationStageHelper.AnalyseTheWebsite(projectId, uri, crawlDetails.CrawlType.Equals(KrawlType.DeepKrawl));
                                            isTaskCompleted = true;
                                            break;

                                        case KitsuneKrawlerStatusCompletion.DownloadingAllStaticAssetsToStorage:
                                            ResourcesStageHelper.DownloadTheResources(projectId, uri);
                                            isTaskCompleted = true;
                                            break;

                                        case KitsuneKrawlerStatusCompletion.UpdatingWebPagesWithNewStaticAssetUri:
                                            PlaceHolderReplacerHelper.ReplacePlaceHolder(projectId, uri);
                                            isTaskCompleted = true;
                                            break;

                                        default:
                                            break;
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError {
                                            ErrorMessage = ex.Message, Stage = stage
                                        });
                                    }
                                    Log.Information($"Completed, Stage: {stage.ToString()}");

                                    #endregion

                                    #region After Process

                                    amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                                    if (isTaskCompleted)
                                    {
                                        stage += 1;
                                        MongoHelper.UpdateCrawlStatsStage(projectId, stage);

                                        //Crawling completed successfully
                                        if (stage == KitsuneKrawlerStatusCompletion.Completed)
                                        {
                                            try
                                            {
                                                APIHelper.KrawlingCompletedUpdateKitsuneProjects(projectId);
                                            }
                                            catch (Exception ex)
                                            {
                                                Log.Error(ex, $"ProjectId:{projectId}, Message:Error updating DB after completion");
                                            }
                                        }

                                        //If need furthur Process again push to sqs
                                        if (stage != KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains &&
                                            stage != KitsuneKrawlerStatusCompletion.Error &&
                                            stage < KitsuneKrawlerStatusCompletion.Completed)
                                        {
                                            amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                                        }

                                        //Event- Analyse Completed (select the domains to download and start next stage)
                                        if (stage == KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains)
                                        {
                                            if (task.MessageBody.ReCrawl)
                                            {
                                                MongoHelper.UpdateCrawlStatsStage(projectId, stage + 1);
                                                amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                                            }
                                            else
                                            {
                                                APIHelper.RegisterAnalyseCompleteEvent(projectId);
                                            }
                                        }
                                    }
                                    else
                                    {
                                        Log.Error($"ProjectId:{projectId}, Message:Error as isTaskCompleted was false for projectId: {projectId}");
                                    }

                                    #endregion
                                }
                                catch (Exception ex)
                                {
                                    //Handle if any exception rises
                                    Log.Error($"ProjectId:{projectId}, Message:Error while Processing the project with Error : {ex.Message}");
                                    MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError {
                                        ErrorMessage = ex.Message, Stage = stage
                                    });
                                    amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                                }
                                Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup);
                            }
                            else
                            {
                                Log.Error($"ProjectId:{projectId}, Message:Error while processing the Service as the projectId was null");
                                amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        //Error picking message from sqs
                        //Error Deleting message from sqs
                        Log.Error(ex, $"ProjectId:{projectId}, Message:Error while processing the Service after getting the value");
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Message:Error while polling from SQS, Exception : {ex.ToString()}");
            }
        }