public static void OptimisationCompleted(string projectId)
        {
            try
            {
                if (_kitsuneServer == null)
                {
                    InitializeConnection();
                }

                var buildStatsCollection = _kitsuneDatabase.GetCollection <KitsuneBuildStatus>(BuildStatusCollectionName);


                #region Send Migration Report

                var amazonSqsHanlder = new AmazonSQSQueueHandlers <TranspilerFunctionSQSModel>(AmazonAWSConstants.TranspilerSQSUrl);
                var project          = GetProjectDetails(projectId);
                TranspilerFunctionSQSModel sqsModel = new TranspilerFunctionSQSModel()
                {
                    ProjectId = projectId,
                    UserEmail = project.UserEmail
                };
                amazonSqsHanlder.PushMessageToQueue(sqsModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey);

                #endregion
            }
            catch (Exception ex)
            {
                //LOG
            }
        }
        /// <summary>
        /// Create new ActivityLog
        /// Pushes the ActivityLogModel to SQS
        /// </summary>
        /// <param name="activityLog"></param>
        public void LogActivity(CreateActivityLog activityLog)
        {
            if (activityLog == null)
            {
                throw new ArgumentNullException(nameof(activityLog));
            }
            if (String.IsNullOrEmpty(activityLog.ResourceId))
            {
                throw new ArgumentNullException(nameof(activityLog.ResourceId));
            }
            if (String.IsNullOrEmpty(activityLog.ActivityId))
            {
                throw new ArgumentNullException(nameof(activityLog.ActivityId));
            }

            ActivityLogSQSModel sqsmodel = new ActivityLogSQSModel
            {
                ResourceId = activityLog.ResourceId,
                ActivityId = activityLog.ActivityId,
                CreatedOn  = activityLog.ActivityCreatedOn,
                Params     = activityLog.Params
            };
            AmazonSQSQueueHandlers <ActivityLogSQSModel> sqsHandler = new AmazonSQSQueueHandlers <ActivityLogSQSModel>(AmazonAWSConstants.ActivitySQSUrl);
            string result = sqsHandler.PushMessageToQueue(sqsmodel, AmazonAWSConstants.SQSAccessKey, AmazonAWSConstants.SQSSecretKey);

            if (result == null)
            {
                throw new Exception($"Error pushing ResourceId : {activityLog.ResourceId} and ActivityId : {activityLog.ActivityId} to SQS");
            }
        }
        public static void TranspilationCompleted(string projectId)
        {
            try
            {
                if (_kitsuneServer == null)
                {
                    InitializeConnection();
                }

                //Update the proejct status to IDLE on transpilation success
                UpdateKitsuneProjectStatus(new UpdateKitsuneProjectStatusRequestModel
                {
                    ProjectId     = projectId,
                    ProjectStatus = ProjectStatus.IDLE
                });

                #region Send Migration Report

                var amazonSqsHanlder             = new AmazonSQSQueueHandlers <ReportGeneratorSQSModel>(AmazonAWSConstants.MigrationReportSQSUrl);
                ReportGeneratorSQSModel sqsModel = new ReportGeneratorSQSModel()
                {
                    ProjectId = projectId
                };
                amazonSqsHanlder.PushMessageToQueue(sqsModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey);

                #endregion

                //Call screenshot API
                CreateAndUpdateDemoScreenShot(projectId);

                //Create Routing Tree
                APIHelper.CreateProjectRoute(projectId, true);

                try
                {
                    ProjectConfigHelper projectConfigHelper = new ProjectConfigHelper(projectId);
                    var isPublish = projectConfigHelper.IsAutoPublishEnabled();
                    //Auto Publish
                    if (isPublish)
                    {
                        //Call publish API
                    }
                }
                catch { }
            }
            catch (Exception ex)
            {
                //LOG
            }
        }
        internal static string PushSitemapGenerationTaskToSQS(SitemapGenerationTaskModel sitemapModel)
        {
            try
            {
                if (sitemapModel == null || String.IsNullOrEmpty(sitemapModel.WebsiteId))
                {
                    return(null);
                }

                if (String.IsNullOrEmpty(sitemapModel.ProjectId))
                {
                    sitemapModel.ProjectId = MongoConnector.GetProjectIdFromWebsiteId(sitemapModel.WebsiteId);
                    if (String.IsNullOrEmpty(sitemapModel.ProjectId))
                    {
                        return(null);
                    }
                }

                if (String.IsNullOrEmpty(sitemapModel.WebsiteUrl))
                {
                    //exclude paths
                    //Handle exception for config
                    // var websiteConfig = MongoConnector.GetProjectConfig(new GetProjectConfigRequestModel() { ProjectId = sitemapModel.ProjectId });
                    sitemapModel.WebsiteUrl = MongoConnector.GetWebsiteUrlFromWebsiteId(sitemapModel.WebsiteId)?.ToLower();
                }

                //To-Do fetch the entire config from db or baseplugin
                var requestModel = new SitemapGenerationTaskQueueModel()
                {
                    param = new SitemapGenerationTaskParameterModel()
                    {
                        access_key = EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey,
                        secret     = EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey,
                        region     = "ap-south-1",
                        acl        = "public-read",
                        bucket     = "kitsune-resource-production",
                        key        = $"{sitemapModel.ProjectId}/websiteresources/{sitemapModel.WebsiteId}/sitemap.xml",
                        url        = sitemapModel.WebsiteUrl
                    },
                    type = 2
                };

                var sqsHanlder = new AmazonSQSQueueHandlers <SitemapGenerationTaskQueueModel>(AmazonAWSConstants.SitemapServiceSQSUrl);
                return(sqsHanlder.PushMessageToQueue(requestModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey,
                                                     EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey, Amazon.RegionEndpoint.APSouth1));
            }
            catch { }
            return(null);
        }
        public static bool UpdateBuildStatus(string user, string projectId, int buildVersion)
        {
            var amazonBuildSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(Program.ServiceConfiguration.GetSection("KitsuneBuildSQSUrl").Value);

            try
            {
                var queueResult = amazonBuildSqsQueueHandler.PushMessageToQueue(new CompilerServiceSQSModel
                {
                    ProjectId    = projectId,
                    BuildVersion = buildVersion
                }, Program.ServiceConfiguration.GetSection("AWSAccessKey").Value, Program.ServiceConfiguration.GetSection("AWSSecretKey").Value, RegionEndpoint.GetBySystemName(Program.ServiceConfiguration.GetSection("AWSRegion").Value));

                if (!string.IsNullOrEmpty(queueResult))
                {
                    var jsonData = JsonConvert.SerializeObject(new { ProjectId = projectId, BuildVersion = buildVersion, Stage = BuildStatus.QueuedBuild });

                    var result = client.PostAsync(string.Format(buildUrl, user), new StringContent(jsonData, Encoding.UTF8, "application/json")).Result;
                    //Add to BuildSQS
                    if (result != null && result.StatusCode == System.Net.HttpStatusCode.OK)
                    {
                        return(true);
                    }
                    else
                    {
                        Log.Error(String.Format("Unable to update the project status : ProjectId {0}", projectId));
                        return(false);
                    }
                }
                else
                {
                    Log.Error(String.Format("Push to build queue failed : ProjectId {0}", projectId));
                    return(false);
                }
            }
            catch (Exception ex)
            {
                Log.Error(String.Format("Push to build queue failed, Error : {0}", ex.Message), projectId, buildVersion.ToString());
                Console.WriteLine(String.Format("Push to build queue failed, Error : {0}", ex.Message), projectId, buildVersion.ToString());
                return(false);
            }
        }
Esempio n. 6
0
        public static void Process()
        {
            try
            {
                while (true)
                {
                    string projectId = String.Empty;
                    KitsuneKrawlerStatusCompletion stage = KitsuneKrawlerStatusCompletion.Error;
                    try
                    {
                        var amazonSqsQueueHandler = new AmazonSQSQueueHandlers <KrawlSQSModel>(EnvironmentConstants.ApplicationConfiguration.CrawlerSQSUrl);
                        //var task = amazonSqsQueueHandler.ReceiveMessageFromQueue(awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                        var task = new AmazonAWSHelpers.Models.AmazonSQSMessageQueueModel <KrawlSQSModel>()
                        {
                            MessageBody = new KrawlSQSModel
                            {
                                ProjectId = "5ce4ef18abc486000121acb8",
                                ReCrawl   = true
                            }
                        };
                        if (task != null)
                        {
                            projectId = task.MessageBody.ProjectId;
                            if (!String.IsNullOrEmpty(projectId))
                            {
                                try
                                {
                                    #region Initiate Logger

                                    Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup, projectId);

                                    #endregion

                                    #region Before Process

                                    try
                                    {
                                        ServiceInformationHelper serviceInfo = new ServiceInformationHelper();
                                        Log.Information($"ProjectId : {projectId}, IP: {serviceInfo.GetInstancePrivateIpAddress()}");
                                    }
                                    catch { }

                                    Uri uri = null;
                                    //Get the Details from DB
                                    var crawlDetails = MongoHelper.GetCrawlingDetails(projectId);
                                    if (crawlDetails == null)
                                    {
                                        throw new Exception("CrawlDetails was null");
                                    }
                                    if (!Uri.TryCreate(crawlDetails.Url, UriKind.Absolute, out uri))
                                    {
                                        throw new Exception(String.Format("Error Creating Uri from Url : {0}", crawlDetails.Url));
                                    }
                                    stage = crawlDetails.Stage;

                                    #endregion

                                    #region Process

                                    var isTaskCompleted = false;
                                    Log.Information($"Started, Stage: {stage.ToString()}");
                                    try
                                    {
                                        switch (stage)
                                        {
                                        case KitsuneKrawlerStatusCompletion.Initialising:
                                            InitialiseKrawlerStageHelper.InitialiseKrawler(projectId, uri);
                                            isTaskCompleted = true;
                                            break;

                                        case KitsuneKrawlerStatusCompletion.IdentifyingAllAssetsAndDownloadingWebpage:
                                            MigrationStageHelper.AnalyseTheWebsite(projectId, uri, crawlDetails.CrawlType.Equals(KrawlType.DeepKrawl));
                                            isTaskCompleted = true;
                                            break;

                                        case KitsuneKrawlerStatusCompletion.DownloadingAllStaticAssetsToStorage:
                                            ResourcesStageHelper.DownloadTheResources(projectId, uri);
                                            isTaskCompleted = true;
                                            break;

                                        case KitsuneKrawlerStatusCompletion.UpdatingWebPagesWithNewStaticAssetUri:
                                            PlaceHolderReplacerHelper.ReplacePlaceHolder(projectId, uri);
                                            isTaskCompleted = true;
                                            break;

                                        default:
                                            break;
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError {
                                            ErrorMessage = ex.Message, Stage = stage
                                        });
                                    }
                                    Log.Information($"Completed, Stage: {stage.ToString()}");

                                    #endregion

                                    #region After Process

                                    amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                                    if (isTaskCompleted)
                                    {
                                        stage += 1;
                                        MongoHelper.UpdateCrawlStatsStage(projectId, stage);

                                        //Crawling completed successfully
                                        if (stage == KitsuneKrawlerStatusCompletion.Completed)
                                        {
                                            try
                                            {
                                                APIHelper.KrawlingCompletedUpdateKitsuneProjects(projectId);
                                            }
                                            catch (Exception ex)
                                            {
                                                Log.Error(ex, $"ProjectId:{projectId}, Message:Error updating DB after completion");
                                            }
                                        }

                                        //If need furthur Process again push to sqs
                                        if (stage != KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains &&
                                            stage != KitsuneKrawlerStatusCompletion.Error &&
                                            stage < KitsuneKrawlerStatusCompletion.Completed)
                                        {
                                            amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                                        }

                                        //Event- Analyse Completed (select the domains to download and start next stage)
                                        if (stage == KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains)
                                        {
                                            if (task.MessageBody.ReCrawl)
                                            {
                                                MongoHelper.UpdateCrawlStatsStage(projectId, stage + 1);
                                                amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                                            }
                                            else
                                            {
                                                APIHelper.RegisterAnalyseCompleteEvent(projectId);
                                            }
                                        }
                                    }
                                    else
                                    {
                                        Log.Error($"ProjectId:{projectId}, Message:Error as isTaskCompleted was false for projectId: {projectId}");
                                    }

                                    #endregion
                                }
                                catch (Exception ex)
                                {
                                    //Handle if any exception rises
                                    Log.Error($"ProjectId:{projectId}, Message:Error while Processing the project with Error : {ex.Message}");
                                    MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError {
                                        ErrorMessage = ex.Message, Stage = stage
                                    });
                                    amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                                }
                                Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup);
                            }
                            else
                            {
                                Log.Error($"ProjectId:{projectId}, Message:Error while processing the Service as the projectId was null");
                                amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        //Error picking message from sqs
                        //Error Deleting message from sqs
                        Log.Error(ex, $"ProjectId:{projectId}, Message:Error while processing the Service after getting the value");
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Message:Error while polling from SQS, Exception : {ex.ToString()}");
            }
        }