static void Main(string[] args)
{
Console.WriteLine("Kitsune Compiler Service Started");
if (isDebug)
{
Console.WriteLine("KString service started");
}
while (true)
{
try
{
var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <KStringQueueModel>(kStringQueueUrl);
var task = amazonCompilerSqsQueueHandler.ReceiveMessageFromQueue();
try
{
if (task != null && task.MessageBody != null)
{
var keywordList = KStringHelper.ExtractKeyword(task.MessageBody.KString);
if (keywordList != null && keywordList.Any())
{
var result = KStringHelper.UpdateKeywordsToKitsuneDB(task.MessageBody.SchemaName, task.MessageBody.UserId, task.MessageBody.KID, task.MessageBody.ReferenceId, task.MessageBody.KString, keywordList);
if (!string.IsNullOrEmpty(result))
{
Console.WriteLine($"Keyword Extracted : {JsonConvert.SerializeObject(task.MessageBody)}");
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(String.Format("Error during kstring queue processing, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace));
}
finally
{
if (task != null)
{
amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(task);
Console.WriteLine($"Message removed : {task.MessageBody.SchemaName}");
}
}
}
catch (Exception ex)
{
// EventLogger.LogTrace(String.Format("Error during compilation, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace), null, null);
Console.WriteLine(String.Format("Error during kstring processing, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace));
}
finally
{
}
}
}
static void Main(string[] args)
{
#if DEBUG
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", true, true)
.AddEnvironmentVariables();
#else
var builder = new ConfigurationBuilder()
.AddJsonFile($"appsettings.Production.json", true, true)
.AddEnvironmentVariables();
#endif
ServiceConfiguration = builder.Build();
string myIP = null;
try
{
string hostName = Dns.GetHostName(); // Retrive the Name of HOST
// Get the IP
myIP = Dns.GetHostEntry(hostName)?.AddressList?.Where(x => x.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)?.FirstOrDefault()?.ToString();
}
catch
{
}
Logger.InitLogger(ServiceConfiguration.GetSection("AWSAccessKey").Value,
ServiceConfiguration.GetSection("AWSSecretKey").Value,
ServiceConfiguration.GetSection("CloudWatchLogGroup").Value,
myIP);
Console.WriteLine("Started");
Log.Information("Compiler service started");
Console.WriteLine(ServiceConfiguration.GetSection("KitsuneCompilerSQSUrl").Value);
Log.Information(ServiceConfiguration.GetSection("KitsuneCompilerSQSUrl").Value);
var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(ServiceConfiguration.GetSection("KitsuneCompilerSQSUrl").Value);
AmazonAWSHelpers.Models.AmazonSQSMessageQueueModel <CompilerServiceSQSModel> task = null;
string projectId = string.Empty;
int buildVersion = 0;
string user = string.Empty;
var errors = new List <BuildError>();
List <CompileResult> buildStatus = null;
while (true)
{
try
{
Console.WriteLine($"Polling from the queue : {DateTime.UtcNow.ToString()}");
Log.Information($"Polling from the queue : {DateTime.UtcNow.ToString()}");
task = amazonCompilerSqsQueueHandler.ReceiveMessageFromQueue(ServiceConfiguration.GetSection("AWSAccessKey").Value, ServiceConfiguration.GetSection("AWSSecretKey").Value,
RegionEndpoint.GetBySystemName(ServiceConfiguration.GetSection("AWSRegion").Value));
if (task != null && task.MessageBody != null)
{
Console.Clear();
projectId = task.MessageBody.ProjectId;
buildVersion = task.MessageBody.BuildVersion;
user = task.MessageBody.UserEmail;
//Update the kitsune project status to building
if (APIHelpers.UpdateProjectStatus(user, projectId, buildVersion.ToString()))
{
Console.WriteLine(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion), projectId, buildVersion.ToString());
Log.Information(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion), projectId, buildVersion.ToString());
buildStatus = new BuildAndRunHelper().BuildProject(user, projectId, bool.Parse(ServiceConfiguration.GetSection("_isDev").Value), _currentCompilerVersion);
if (buildStatus != null && buildStatus.Any())
{
errors = new List <BuildError>();
foreach (var error in buildStatus)
{
if (!error.Success)
{
errors.AddRange(error.ErrorMessages.Select(x => new BuildError
{
Column = x.LinePosition,
Line = x.LineNumber,
Message = x.Message,
ErrorStackTrace = x.Message,
SourceMethod = "KitsuneCompiler",
SourcePath = error.PageName
}));
}
}
APIHelpers.UpdateProjectErrorStatus(user, projectId, buildVersion, errors);
Console.WriteLine(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
Log.Error(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
foreach (var err in errors)
{
Console.WriteLine(JsonConvert.SerializeObject(err));
Log.Error(JsonConvert.SerializeObject(err));
}
buildStatus = null;
}
else
{
//UPDATE the project status
if (APIHelpers.UpdateBuildStatus(user, projectId, buildVersion))
{
Console.WriteLine(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
Log.Information(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
}
}
}
amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(task, ServiceConfiguration.GetSection("AWSAccessKey").Value, ServiceConfiguration.GetSection("AWSSecretKey").Value, RegionEndpoint.GetBySystemName(ServiceConfiguration.GetSection("AWSRegion").Value));
Console.WriteLine(string.Format("Message removed : {0}", task?.MessageBody?.ProjectId));
Log.Information(string.Format("Message removed : {0}", task?.MessageBody?.ProjectId));
}
}
catch (Exception ex)
{
Console.WriteLine(String.Format("Error during compilation, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace));
Log.Error(String.Format("Error during compilation, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace));
}
}
}
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
{
context.Logger.LogLine($"Processed message {message.Body}");
context.Logger.LogLine(Environment.GetEnvironmentVariable("SQS_AWS_REGION"));
context.Logger.LogLine(Directory.GetCurrentDirectory() + "/Files");
try
{
var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(Environment.GetEnvironmentVariable("SQS_URL"));
var sqsModel = JsonConvert.DeserializeObject <CompilerServiceSQSModel>(message.Body);
var errors = new List <BuildError>();
List <CompileResult> buildStatus = null;
var projectId = sqsModel.ProjectId;
var buildVersion = sqsModel.BuildVersion;
var user = sqsModel.UserEmail;
//Update the kitsune project status to building
if (APIHelpers.UpdateProjectStatus(user, projectId, buildVersion.ToString()))
{
context.Logger.LogLine(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion));
buildStatus = new BuildAndRunHelper().BuildProject(user, projectId, bool.Parse(Environment.GetEnvironmentVariable("IS_DEVELOPMENT_VERSION")), 1);
if (buildStatus != null && buildStatus.Any())
{
errors = new List <BuildError>();
foreach (var error in buildStatus)
{
if (!error.Success)
{
errors.AddRange(error.ErrorMessages.Select(x => new BuildError
{
Column = x.LinePosition,
Line = x.LineNumber,
Message = x.Message,
ErrorStackTrace = x.Message,
SourceMethod = "KitsuneCompiler",
SourcePath = error.PageName
}));
}
}
APIHelpers.UpdateProjectErrorStatus(user, projectId, buildVersion, errors);
Console.WriteLine(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
Log.Error(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
foreach (var err in errors)
{
Console.WriteLine(JsonConvert.SerializeObject(err));
Log.Error(JsonConvert.SerializeObject(err));
}
buildStatus = null;
}
else
{
//UPDATE the project status
if (APIHelpers.UpdateBuildStatus(user, projectId, buildVersion))
{
Console.WriteLine(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
Log.Information(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
}
}
}
var response = amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(new AmazonSQSMessageQueueModel <CompilerServiceSQSModel>
{
MessageBody = JsonConvert.DeserializeObject <CompilerServiceSQSModel>(message.Body),
MessageId = message.MessageId,
ReceiptHandle = message.ReceiptHandle,
MessageAttributes = message.Attributes
});
context.Logger.LogLine(response);
}
catch { }
await Task.CompletedTask;
}
public static void Process()
{
try
{
while (true)
{
string projectId = String.Empty;
KitsuneKrawlerStatusCompletion stage = KitsuneKrawlerStatusCompletion.Error;
try
{
var amazonSqsQueueHandler = new AmazonSQSQueueHandlers <KrawlSQSModel>(EnvironmentConstants.ApplicationConfiguration.CrawlerSQSUrl);
//var task = amazonSqsQueueHandler.ReceiveMessageFromQueue(awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
var task = new AmazonAWSHelpers.Models.AmazonSQSMessageQueueModel <KrawlSQSModel>()
{
MessageBody = new KrawlSQSModel
{
ProjectId = "5ce4ef18abc486000121acb8",
ReCrawl = true
}
};
if (task != null)
{
projectId = task.MessageBody.ProjectId;
if (!String.IsNullOrEmpty(projectId))
{
try
{
#region Initiate Logger
Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup, projectId);
#endregion
#region Before Process
try
{
ServiceInformationHelper serviceInfo = new ServiceInformationHelper();
Log.Information($"ProjectId : {projectId}, IP: {serviceInfo.GetInstancePrivateIpAddress()}");
}
catch { }
Uri uri = null;
//Get the Details from DB
var crawlDetails = MongoHelper.GetCrawlingDetails(projectId);
if (crawlDetails == null)
{
throw new Exception("CrawlDetails was null");
}
if (!Uri.TryCreate(crawlDetails.Url, UriKind.Absolute, out uri))
{
throw new Exception(String.Format("Error Creating Uri from Url : {0}", crawlDetails.Url));
}
stage = crawlDetails.Stage;
#endregion
#region Process
var isTaskCompleted = false;
Log.Information($"Started, Stage: {stage.ToString()}");
try
{
switch (stage)
{
case KitsuneKrawlerStatusCompletion.Initialising:
InitialiseKrawlerStageHelper.InitialiseKrawler(projectId, uri);
isTaskCompleted = true;
break;
case KitsuneKrawlerStatusCompletion.IdentifyingAllAssetsAndDownloadingWebpage:
MigrationStageHelper.AnalyseTheWebsite(projectId, uri, crawlDetails.CrawlType.Equals(KrawlType.DeepKrawl));
isTaskCompleted = true;
break;
case KitsuneKrawlerStatusCompletion.DownloadingAllStaticAssetsToStorage:
ResourcesStageHelper.DownloadTheResources(projectId, uri);
isTaskCompleted = true;
break;
case KitsuneKrawlerStatusCompletion.UpdatingWebPagesWithNewStaticAssetUri:
PlaceHolderReplacerHelper.ReplacePlaceHolder(projectId, uri);
isTaskCompleted = true;
break;
default:
break;
}
}
catch (Exception ex)
{
MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError {
ErrorMessage = ex.Message, Stage = stage
});
}
Log.Information($"Completed, Stage: {stage.ToString()}");
#endregion
#region After Process
amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
if (isTaskCompleted)
{
stage += 1;
MongoHelper.UpdateCrawlStatsStage(projectId, stage);
//Crawling completed successfully
if (stage == KitsuneKrawlerStatusCompletion.Completed)
{
try
{
APIHelper.KrawlingCompletedUpdateKitsuneProjects(projectId);
}
catch (Exception ex)
{
Log.Error(ex, $"ProjectId:{projectId}, Message:Error updating DB after completion");
}
}
//If need furthur Process again push to sqs
if (stage != KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains &&
stage != KitsuneKrawlerStatusCompletion.Error &&
stage < KitsuneKrawlerStatusCompletion.Completed)
{
amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
}
//Event- Analyse Completed (select the domains to download and start next stage)
if (stage == KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains)
{
if (task.MessageBody.ReCrawl)
{
MongoHelper.UpdateCrawlStatsStage(projectId, stage + 1);
amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
}
else
{
APIHelper.RegisterAnalyseCompleteEvent(projectId);
}
}
}
else
{
Log.Error($"ProjectId:{projectId}, Message:Error as isTaskCompleted was false for projectId: {projectId}");
}
#endregion
}
catch (Exception ex)
{
//Handle if any exception rises
Log.Error($"ProjectId:{projectId}, Message:Error while Processing the project with Error : {ex.Message}");
MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError {
ErrorMessage = ex.Message, Stage = stage
});
amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
}
Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup);
}
else
{
Log.Error($"ProjectId:{projectId}, Message:Error while processing the Service as the projectId was null");
amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey);
}
}
}
catch (Exception ex)
{
//Error picking message from sqs
//Error Deleting message from sqs
Log.Error(ex, $"ProjectId:{projectId}, Message:Error while processing the Service after getting the value");
}
}
}
catch (Exception ex)
{
Console.WriteLine($"Message:Error while polling from SQS, Exception : {ex.ToString()}");
}
}