public static void OptimisationCompleted(string projectId) { try { if (_kitsuneServer == null) { InitializeConnection(); } var buildStatsCollection = _kitsuneDatabase.GetCollection <KitsuneBuildStatus>(BuildStatusCollectionName); #region Send Migration Report var amazonSqsHanlder = new AmazonSQSQueueHandlers <TranspilerFunctionSQSModel>(AmazonAWSConstants.TranspilerSQSUrl); var project = GetProjectDetails(projectId); TranspilerFunctionSQSModel sqsModel = new TranspilerFunctionSQSModel() { ProjectId = projectId, UserEmail = project.UserEmail }; amazonSqsHanlder.PushMessageToQueue(sqsModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey); #endregion } catch (Exception ex) { //LOG } }
/// <summary> /// Create new ActivityLog /// Pushes the ActivityLogModel to SQS /// </summary> /// <param name="activityLog"></param> public void LogActivity(CreateActivityLog activityLog) { if (activityLog == null) { throw new ArgumentNullException(nameof(activityLog)); } if (String.IsNullOrEmpty(activityLog.ResourceId)) { throw new ArgumentNullException(nameof(activityLog.ResourceId)); } if (String.IsNullOrEmpty(activityLog.ActivityId)) { throw new ArgumentNullException(nameof(activityLog.ActivityId)); } ActivityLogSQSModel sqsmodel = new ActivityLogSQSModel { ResourceId = activityLog.ResourceId, ActivityId = activityLog.ActivityId, CreatedOn = activityLog.ActivityCreatedOn, Params = activityLog.Params }; AmazonSQSQueueHandlers <ActivityLogSQSModel> sqsHandler = new AmazonSQSQueueHandlers <ActivityLogSQSModel>(AmazonAWSConstants.ActivitySQSUrl); string result = sqsHandler.PushMessageToQueue(sqsmodel, AmazonAWSConstants.SQSAccessKey, AmazonAWSConstants.SQSSecretKey); if (result == null) { throw new Exception($"Error pushing ResourceId : {activityLog.ResourceId} and ActivityId : {activityLog.ActivityId} to SQS"); } }
public static void TranspilationCompleted(string projectId) { try { if (_kitsuneServer == null) { InitializeConnection(); } //Update the proejct status to IDLE on transpilation success UpdateKitsuneProjectStatus(new UpdateKitsuneProjectStatusRequestModel { ProjectId = projectId, ProjectStatus = ProjectStatus.IDLE }); #region Send Migration Report var amazonSqsHanlder = new AmazonSQSQueueHandlers <ReportGeneratorSQSModel>(AmazonAWSConstants.MigrationReportSQSUrl); ReportGeneratorSQSModel sqsModel = new ReportGeneratorSQSModel() { ProjectId = projectId }; amazonSqsHanlder.PushMessageToQueue(sqsModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey); #endregion //Call screenshot API CreateAndUpdateDemoScreenShot(projectId); //Create Routing Tree APIHelper.CreateProjectRoute(projectId, true); try { ProjectConfigHelper projectConfigHelper = new ProjectConfigHelper(projectId); var isPublish = projectConfigHelper.IsAutoPublishEnabled(); //Auto Publish if (isPublish) { //Call publish API } } catch { } } catch (Exception ex) { //LOG } }
internal static string PushSitemapGenerationTaskToSQS(SitemapGenerationTaskModel sitemapModel) { try { if (sitemapModel == null || String.IsNullOrEmpty(sitemapModel.WebsiteId)) { return(null); } if (String.IsNullOrEmpty(sitemapModel.ProjectId)) { sitemapModel.ProjectId = MongoConnector.GetProjectIdFromWebsiteId(sitemapModel.WebsiteId); if (String.IsNullOrEmpty(sitemapModel.ProjectId)) { return(null); } } if (String.IsNullOrEmpty(sitemapModel.WebsiteUrl)) { //exclude paths //Handle exception for config // var websiteConfig = MongoConnector.GetProjectConfig(new GetProjectConfigRequestModel() { ProjectId = sitemapModel.ProjectId }); sitemapModel.WebsiteUrl = MongoConnector.GetWebsiteUrlFromWebsiteId(sitemapModel.WebsiteId)?.ToLower(); } //To-Do fetch the entire config from db or baseplugin var requestModel = new SitemapGenerationTaskQueueModel() { param = new SitemapGenerationTaskParameterModel() { access_key = EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, secret = EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey, region = "ap-south-1", acl = "public-read", bucket = "kitsune-resource-production", key = $"{sitemapModel.ProjectId}/websiteresources/{sitemapModel.WebsiteId}/sitemap.xml", url = sitemapModel.WebsiteUrl }, type = 2 }; var sqsHanlder = new AmazonSQSQueueHandlers <SitemapGenerationTaskQueueModel>(AmazonAWSConstants.SitemapServiceSQSUrl); return(sqsHanlder.PushMessageToQueue(requestModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey, Amazon.RegionEndpoint.APSouth1)); } catch { } return(null); }
public static bool UpdateBuildStatus(string user, string projectId, int buildVersion) { var amazonBuildSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(Program.ServiceConfiguration.GetSection("KitsuneBuildSQSUrl").Value); try { var queueResult = amazonBuildSqsQueueHandler.PushMessageToQueue(new CompilerServiceSQSModel { ProjectId = projectId, BuildVersion = buildVersion }, Program.ServiceConfiguration.GetSection("AWSAccessKey").Value, Program.ServiceConfiguration.GetSection("AWSSecretKey").Value, RegionEndpoint.GetBySystemName(Program.ServiceConfiguration.GetSection("AWSRegion").Value)); if (!string.IsNullOrEmpty(queueResult)) { var jsonData = JsonConvert.SerializeObject(new { ProjectId = projectId, BuildVersion = buildVersion, Stage = BuildStatus.QueuedBuild }); var result = client.PostAsync(string.Format(buildUrl, user), new StringContent(jsonData, Encoding.UTF8, "application/json")).Result; //Add to BuildSQS if (result != null && result.StatusCode == System.Net.HttpStatusCode.OK) { return(true); } else { Log.Error(String.Format("Unable to update the project status : ProjectId {0}", projectId)); return(false); } } else { Log.Error(String.Format("Push to build queue failed : ProjectId {0}", projectId)); return(false); } } catch (Exception ex) { Log.Error(String.Format("Push to build queue failed, Error : {0}", ex.Message), projectId, buildVersion.ToString()); Console.WriteLine(String.Format("Push to build queue failed, Error : {0}", ex.Message), projectId, buildVersion.ToString()); return(false); } }
public static void Process() { try { while (true) { string projectId = String.Empty; KitsuneKrawlerStatusCompletion stage = KitsuneKrawlerStatusCompletion.Error; try { var amazonSqsQueueHandler = new AmazonSQSQueueHandlers <KrawlSQSModel>(EnvironmentConstants.ApplicationConfiguration.CrawlerSQSUrl); //var task = amazonSqsQueueHandler.ReceiveMessageFromQueue(awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); var task = new AmazonAWSHelpers.Models.AmazonSQSMessageQueueModel <KrawlSQSModel>() { MessageBody = new KrawlSQSModel { ProjectId = "5ce4ef18abc486000121acb8", ReCrawl = true } }; if (task != null) { projectId = task.MessageBody.ProjectId; if (!String.IsNullOrEmpty(projectId)) { try { #region Initiate Logger Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup, projectId); #endregion #region Before Process try { ServiceInformationHelper serviceInfo = new ServiceInformationHelper(); Log.Information($"ProjectId : {projectId}, IP: {serviceInfo.GetInstancePrivateIpAddress()}"); } catch { } Uri uri = null; //Get the Details from DB var crawlDetails = MongoHelper.GetCrawlingDetails(projectId); if (crawlDetails == null) { throw new Exception("CrawlDetails was null"); } if (!Uri.TryCreate(crawlDetails.Url, UriKind.Absolute, out uri)) { throw new Exception(String.Format("Error Creating Uri from Url : {0}", crawlDetails.Url)); } stage = crawlDetails.Stage; #endregion #region Process var isTaskCompleted = false; Log.Information($"Started, Stage: {stage.ToString()}"); try { switch (stage) { case KitsuneKrawlerStatusCompletion.Initialising: InitialiseKrawlerStageHelper.InitialiseKrawler(projectId, uri); isTaskCompleted = true; break; case KitsuneKrawlerStatusCompletion.IdentifyingAllAssetsAndDownloadingWebpage: MigrationStageHelper.AnalyseTheWebsite(projectId, uri, crawlDetails.CrawlType.Equals(KrawlType.DeepKrawl)); isTaskCompleted = true; break; case KitsuneKrawlerStatusCompletion.DownloadingAllStaticAssetsToStorage: ResourcesStageHelper.DownloadTheResources(projectId, uri); isTaskCompleted = true; break; case KitsuneKrawlerStatusCompletion.UpdatingWebPagesWithNewStaticAssetUri: PlaceHolderReplacerHelper.ReplacePlaceHolder(projectId, uri); isTaskCompleted = true; break; default: break; } } catch (Exception ex) { MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError { ErrorMessage = ex.Message, Stage = stage }); } Log.Information($"Completed, Stage: {stage.ToString()}"); #endregion #region After Process amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); if (isTaskCompleted) { stage += 1; MongoHelper.UpdateCrawlStatsStage(projectId, stage); //Crawling completed successfully if (stage == KitsuneKrawlerStatusCompletion.Completed) { try { APIHelper.KrawlingCompletedUpdateKitsuneProjects(projectId); } catch (Exception ex) { Log.Error(ex, $"ProjectId:{projectId}, Message:Error updating DB after completion"); } } //If need furthur Process again push to sqs if (stage != KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains && stage != KitsuneKrawlerStatusCompletion.Error && stage < KitsuneKrawlerStatusCompletion.Completed) { amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); } //Event- Analyse Completed (select the domains to download and start next stage) if (stage == KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains) { if (task.MessageBody.ReCrawl) { MongoHelper.UpdateCrawlStatsStage(projectId, stage + 1); amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); } else { APIHelper.RegisterAnalyseCompleteEvent(projectId); } } } else { Log.Error($"ProjectId:{projectId}, Message:Error as isTaskCompleted was false for projectId: {projectId}"); } #endregion } catch (Exception ex) { //Handle if any exception rises Log.Error($"ProjectId:{projectId}, Message:Error while Processing the project with Error : {ex.Message}"); MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError { ErrorMessage = ex.Message, Stage = stage }); amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); } Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup); } else { Log.Error($"ProjectId:{projectId}, Message:Error while processing the Service as the projectId was null"); amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); } } } catch (Exception ex) { //Error picking message from sqs //Error Deleting message from sqs Log.Error(ex, $"ProjectId:{projectId}, Message:Error while processing the Service after getting the value"); } } } catch (Exception ex) { Console.WriteLine($"Message:Error while polling from SQS, Exception : {ex.ToString()}"); } }