public static void OptimisationCompleted(string projectId)
{
try
{
if (_kitsuneServer == null)
{
InitializeConnection();
}
var buildStatsCollection = _kitsuneDatabase.GetCollection <KitsuneBuildStatus>(BuildStatusCollectionName);
#region Send Migration Report
var amazonSqsHanlder = new AmazonSQSQueueHandlers <TranspilerFunctionSQSModel>(AmazonAWSConstants.TranspilerSQSUrl);
var project = GetProjectDetails(projectId);
TranspilerFunctionSQSModel sqsModel = new TranspilerFunctionSQSModel()
{
ProjectId = projectId,
UserEmail = project.UserEmail
};
amazonSqsHanlder.PushMessageToQueue(sqsModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey);
#endregion
}
catch (Exception ex)
{
//LOG
}
}
/// <summary>
/// Create new ActivityLog
/// Pushes the ActivityLogModel to SQS
/// </summary>
/// <param name="activityLog"></param>
public void LogActivity(CreateActivityLog activityLog)
{
if (activityLog == null)
{
throw new ArgumentNullException(nameof(activityLog));
}
if (String.IsNullOrEmpty(activityLog.ResourceId))
{
throw new ArgumentNullException(nameof(activityLog.ResourceId));
}
if (String.IsNullOrEmpty(activityLog.ActivityId))
{
throw new ArgumentNullException(nameof(activityLog.ActivityId));
}
ActivityLogSQSModel sqsmodel = new ActivityLogSQSModel
{
ResourceId = activityLog.ResourceId,
ActivityId = activityLog.ActivityId,
CreatedOn = activityLog.ActivityCreatedOn,
Params = activityLog.Params
};
AmazonSQSQueueHandlers <ActivityLogSQSModel> sqsHandler = new AmazonSQSQueueHandlers <ActivityLogSQSModel>(AmazonAWSConstants.ActivitySQSUrl);
string result = sqsHandler.PushMessageToQueue(sqsmodel, AmazonAWSConstants.SQSAccessKey, AmazonAWSConstants.SQSSecretKey);
if (result == null)
{
throw new Exception($"Error pushing ResourceId : {activityLog.ResourceId} and ActivityId : {activityLog.ActivityId} to SQS");
}
}
public static void TranspilationCompleted(string projectId)
{
try
{
if (_kitsuneServer == null)
{
InitializeConnection();
}
//Update the proejct status to IDLE on transpilation success
UpdateKitsuneProjectStatus(new UpdateKitsuneProjectStatusRequestModel
{
ProjectId = projectId,
ProjectStatus = ProjectStatus.IDLE
});
#region Send Migration Report
var amazonSqsHanlder = new AmazonSQSQueueHandlers <ReportGeneratorSQSModel>(AmazonAWSConstants.MigrationReportSQSUrl);
ReportGeneratorSQSModel sqsModel = new ReportGeneratorSQSModel()
{
ProjectId = projectId
};
amazonSqsHanlder.PushMessageToQueue(sqsModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey);
#endregion
//Call screenshot API
CreateAndUpdateDemoScreenShot(projectId);
//Create Routing Tree
APIHelper.CreateProjectRoute(projectId, true);
try
{
ProjectConfigHelper projectConfigHelper = new ProjectConfigHelper(projectId);
var isPublish = projectConfigHelper.IsAutoPublishEnabled();
//Auto Publish
if (isPublish)
{
//Call publish API
}
}
catch { }
}
catch (Exception ex)
{
//LOG
}
}
internal static string PushSitemapGenerationTaskToSQS(SitemapGenerationTaskModel sitemapModel)
{
try
{
if (sitemapModel == null || String.IsNullOrEmpty(sitemapModel.WebsiteId))
{
return(null);
}
if (String.IsNullOrEmpty(sitemapModel.ProjectId))
{
sitemapModel.ProjectId = MongoConnector.GetProjectIdFromWebsiteId(sitemapModel.WebsiteId);
if (String.IsNullOrEmpty(sitemapModel.ProjectId))
{
return(null);
}
}
if (String.IsNullOrEmpty(sitemapModel.WebsiteUrl))
{
//exclude paths
//Handle exception for config
// var websiteConfig = MongoConnector.GetProjectConfig(new GetProjectConfigRequestModel() { ProjectId = sitemapModel.ProjectId });
sitemapModel.WebsiteUrl = MongoConnector.GetWebsiteUrlFromWebsiteId(sitemapModel.WebsiteId)?.ToLower();
}
//To-Do fetch the entire config from db or baseplugin
var requestModel = new SitemapGenerationTaskQueueModel()
{
param = new SitemapGenerationTaskParameterModel()
{
access_key = EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey,
secret = EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey,
region = "ap-south-1",
acl = "public-read",
bucket = "kitsune-resource-production",
key = $"{sitemapModel.ProjectId}/websiteresources/{sitemapModel.WebsiteId}/sitemap.xml",
url = sitemapModel.WebsiteUrl
},
type = 2
};
var sqsHanlder = new AmazonSQSQueueHandlers <SitemapGenerationTaskQueueModel>(AmazonAWSConstants.SitemapServiceSQSUrl);
return(sqsHanlder.PushMessageToQueue(requestModel, EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_AccessKey,
EnvironmentConstants.ApplicationConfiguration.AWSSQSConfiguration.AWS_SecretKey, Amazon.RegionEndpoint.APSouth1));
}
catch { }
return(null);
}
public static bool UpdateBuildStatus(string user, string projectId, int buildVersion)
{
var amazonBuildSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(Program.ServiceConfiguration.GetSection("KitsuneBuildSQSUrl").Value);
try
{
var queueResult = amazonBuildSqsQueueHandler.PushMessageToQueue(new CompilerServiceSQSModel
{
ProjectId = projectId,
BuildVersion = buildVersion
}, Program.ServiceConfiguration.GetSection("AWSAccessKey").Value, Program.ServiceConfiguration.GetSection("AWSSecretKey").Value, RegionEndpoint.GetBySystemName(Program.ServiceConfiguration.GetSection("AWSRegion").Value));
if (!string.IsNullOrEmpty(queueResult))
{
var jsonData = JsonConvert.SerializeObject(new { ProjectId = projectId, BuildVersion = buildVersion, Stage = BuildStatus.QueuedBuild });
var result = client.PostAsync(string.Format(buildUrl, user), new StringContent(jsonData, Encoding.UTF8, "application/json")).Result;
//Add to BuildSQS
if (result != null && result.StatusCode == System.Net.HttpStatusCode.OK)
{
return(true);
}
else
{
Log.Error(String.Format("Unable to update the project status : ProjectId {0}", projectId));
return(false);
}
}
else
{
Log.Error(String.Format("Push to build queue failed : ProjectId {0}", projectId));
return(false);
}
}
catch (Exception ex)
{
Log.Error(String.Format("Push to build queue failed, Error : {0}", ex.Message), projectId, buildVersion.ToString());
Console.WriteLine(String.Format("Push to build queue failed, Error : {0}", ex.Message), projectId, buildVersion.ToString());
return(false);
}
}
public static void Process()
{
try
{
while (true)
{
string projectId = String.Empty;
KitsuneKrawlerStatusCompletion stage = KitsuneKrawlerStatusCompletion.Error;
try
{
var amazonSqsQueueHandler = new AmazonSQSQueueHandlers <KrawlSQSModel>(EnvironmentConstants.ApplicationConfiguration.CrawlerSQSUrl);
//var task = amazonSqsQueueHandler.ReceiveMessageFromQueue(awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
var task = new AmazonAWSHelpers.Models.AmazonSQSMessageQueueModel <KrawlSQSModel>()
{
MessageBody = new KrawlSQSModel
{
ProjectId = "5ce4ef18abc486000121acb8",
ReCrawl = true
}
};
if (task != null)
{
projectId = task.MessageBody.ProjectId;
if (!String.IsNullOrEmpty(projectId))
{
try
{
#region Initiate Logger
Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup, projectId);
#endregion
#region Before Process
try
{
ServiceInformationHelper serviceInfo = new ServiceInformationHelper();
Log.Information($"ProjectId : {projectId}, IP: {serviceInfo.GetInstancePrivateIpAddress()}");
}
catch { }
Uri uri = null;
//Get the Details from DB
var crawlDetails = MongoHelper.GetCrawlingDetails(projectId);
if (crawlDetails == null)
{
throw new Exception("CrawlDetails was null");
}
if (!Uri.TryCreate(crawlDetails.Url, UriKind.Absolute, out uri))
{
throw new Exception(String.Format("Error Creating Uri from Url : {0}", crawlDetails.Url));
}
stage = crawlDetails.Stage;
#endregion
#region Process
var isTaskCompleted = false;
Log.Information($"Started, Stage: {stage.ToString()}");
try
{
switch (stage)
{
case KitsuneKrawlerStatusCompletion.Initialising:
InitialiseKrawlerStageHelper.InitialiseKrawler(projectId, uri);
isTaskCompleted = true;
break;
case KitsuneKrawlerStatusCompletion.IdentifyingAllAssetsAndDownloadingWebpage:
MigrationStageHelper.AnalyseTheWebsite(projectId, uri, crawlDetails.CrawlType.Equals(KrawlType.DeepKrawl));
isTaskCompleted = true;
break;
case KitsuneKrawlerStatusCompletion.DownloadingAllStaticAssetsToStorage:
ResourcesStageHelper.DownloadTheResources(projectId, uri);
isTaskCompleted = true;
break;
case KitsuneKrawlerStatusCompletion.UpdatingWebPagesWithNewStaticAssetUri:
PlaceHolderReplacerHelper.ReplacePlaceHolder(projectId, uri);
isTaskCompleted = true;
break;
default:
break;
}
}
catch (Exception ex)
{
MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError {
ErrorMessage = ex.Message, Stage = stage
});
}
Log.Information($"Completed, Stage: {stage.ToString()}");
#endregion
#region After Process
amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
if (isTaskCompleted)
{
stage += 1;
MongoHelper.UpdateCrawlStatsStage(projectId, stage);
//Crawling completed successfully
if (stage == KitsuneKrawlerStatusCompletion.Completed)
{
try
{
APIHelper.KrawlingCompletedUpdateKitsuneProjects(projectId);
}
catch (Exception ex)
{
Log.Error(ex, $"ProjectId:{projectId}, Message:Error updating DB after completion");
}
}
//If need furthur Process again push to sqs
if (stage != KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains &&
stage != KitsuneKrawlerStatusCompletion.Error &&
stage < KitsuneKrawlerStatusCompletion.Completed)
{
amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
}
//Event- Analyse Completed (select the domains to download and start next stage)
if (stage == KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains)
{
if (task.MessageBody.ReCrawl)
{
MongoHelper.UpdateCrawlStatsStage(projectId, stage + 1);
amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
}
else
{
APIHelper.RegisterAnalyseCompleteEvent(projectId);
}
}
}
else
{
Log.Error($"ProjectId:{projectId}, Message:Error as isTaskCompleted was false for projectId: {projectId}");
}
#endregion
}
catch (Exception ex)
{
//Handle if any exception rises
Log.Error($"ProjectId:{projectId}, Message:Error while Processing the project with Error : {ex.Message}");
MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError {
ErrorMessage = ex.Message, Stage = stage
});
amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
}
Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup);
}
else
{
Log.Error($"ProjectId:{projectId}, Message:Error while processing the Service as the projectId was null");
amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
}
}
}
catch (Exception ex)
{
//Error picking message from sqs
//Error Deleting message from sqs
Log.Error(ex, $"ProjectId:{projectId}, Message:Error while processing the Service after getting the value");
}
}
}
catch (Exception ex)
{
Console.WriteLine($"Message:Error while polling from SQS, Exception : {ex.ToString()}");
}
}