static void Main(string[] args) { Console.WriteLine("Kitsune Compiler Service Started"); if (isDebug) { Console.WriteLine("KString service started"); } while (true) { try { var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <KStringQueueModel>(kStringQueueUrl); var task = amazonCompilerSqsQueueHandler.ReceiveMessageFromQueue(); try { if (task != null && task.MessageBody != null) { var keywordList = KStringHelper.ExtractKeyword(task.MessageBody.KString); if (keywordList != null && keywordList.Any()) { var result = KStringHelper.UpdateKeywordsToKitsuneDB(task.MessageBody.SchemaName, task.MessageBody.UserId, task.MessageBody.KID, task.MessageBody.ReferenceId, task.MessageBody.KString, keywordList); if (!string.IsNullOrEmpty(result)) { Console.WriteLine($"Keyword Extracted : {JsonConvert.SerializeObject(task.MessageBody)}"); } } } } catch (Exception ex) { Console.WriteLine(String.Format("Error during kstring queue processing, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace)); } finally { if (task != null) { amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(task); Console.WriteLine($"Message removed : {task.MessageBody.SchemaName}"); } } } catch (Exception ex) { // EventLogger.LogTrace(String.Format("Error during compilation, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace), null, null); Console.WriteLine(String.Format("Error during kstring processing, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace)); } finally { } } }
static void Main(string[] args) { #if DEBUG var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", true, true) .AddEnvironmentVariables(); #else var builder = new ConfigurationBuilder() .AddJsonFile($"appsettings.Production.json", true, true) .AddEnvironmentVariables(); #endif ServiceConfiguration = builder.Build(); string myIP = null; try { string hostName = Dns.GetHostName(); // Retrive the Name of HOST // Get the IP myIP = Dns.GetHostEntry(hostName)?.AddressList?.Where(x => x.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)?.FirstOrDefault()?.ToString(); } catch { } Logger.InitLogger(ServiceConfiguration.GetSection("AWSAccessKey").Value, ServiceConfiguration.GetSection("AWSSecretKey").Value, ServiceConfiguration.GetSection("CloudWatchLogGroup").Value, myIP); Console.WriteLine("Started"); Log.Information("Compiler service started"); Console.WriteLine(ServiceConfiguration.GetSection("KitsuneCompilerSQSUrl").Value); Log.Information(ServiceConfiguration.GetSection("KitsuneCompilerSQSUrl").Value); var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(ServiceConfiguration.GetSection("KitsuneCompilerSQSUrl").Value); AmazonAWSHelpers.Models.AmazonSQSMessageQueueModel <CompilerServiceSQSModel> task = null; string projectId = string.Empty; int buildVersion = 0; string user = string.Empty; var errors = new List <BuildError>(); List <CompileResult> buildStatus = null; while (true) { try { Console.WriteLine($"Polling from the queue : {DateTime.UtcNow.ToString()}"); Log.Information($"Polling from the queue : {DateTime.UtcNow.ToString()}"); task = amazonCompilerSqsQueueHandler.ReceiveMessageFromQueue(ServiceConfiguration.GetSection("AWSAccessKey").Value, ServiceConfiguration.GetSection("AWSSecretKey").Value, RegionEndpoint.GetBySystemName(ServiceConfiguration.GetSection("AWSRegion").Value)); if (task != null && task.MessageBody != null) { Console.Clear(); projectId = task.MessageBody.ProjectId; buildVersion = task.MessageBody.BuildVersion; user = task.MessageBody.UserEmail; //Update the kitsune project status to building if (APIHelpers.UpdateProjectStatus(user, projectId, buildVersion.ToString())) { Console.WriteLine(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion), projectId, buildVersion.ToString()); Log.Information(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion), projectId, buildVersion.ToString()); buildStatus = new BuildAndRunHelper().BuildProject(user, projectId, bool.Parse(ServiceConfiguration.GetSection("_isDev").Value), _currentCompilerVersion); if (buildStatus != null && buildStatus.Any()) { errors = new List <BuildError>(); foreach (var error in buildStatus) { if (!error.Success) { errors.AddRange(error.ErrorMessages.Select(x => new BuildError { Column = x.LinePosition, Line = x.LineNumber, Message = x.Message, ErrorStackTrace = x.Message, SourceMethod = "KitsuneCompiler", SourcePath = error.PageName })); } } APIHelpers.UpdateProjectErrorStatus(user, projectId, buildVersion, errors); Console.WriteLine(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion)); Log.Error(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion)); foreach (var err in errors) { Console.WriteLine(JsonConvert.SerializeObject(err)); Log.Error(JsonConvert.SerializeObject(err)); } buildStatus = null; } else { //UPDATE the project status if (APIHelpers.UpdateBuildStatus(user, projectId, buildVersion)) { Console.WriteLine(string.Format("Compilation done successful"), projectId, buildVersion.ToString()); Log.Information(string.Format("Compilation done successful"), projectId, buildVersion.ToString()); } } } amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(task, ServiceConfiguration.GetSection("AWSAccessKey").Value, ServiceConfiguration.GetSection("AWSSecretKey").Value, RegionEndpoint.GetBySystemName(ServiceConfiguration.GetSection("AWSRegion").Value)); Console.WriteLine(string.Format("Message removed : {0}", task?.MessageBody?.ProjectId)); Log.Information(string.Format("Message removed : {0}", task?.MessageBody?.ProjectId)); } } catch (Exception ex) { Console.WriteLine(String.Format("Error during compilation, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace)); Log.Error(String.Format("Error during compilation, ErrorMessage : {0}, StackTrace : {1}", ex.Message, ex.StackTrace)); } } }
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context) { context.Logger.LogLine($"Processed message {message.Body}"); context.Logger.LogLine(Environment.GetEnvironmentVariable("SQS_AWS_REGION")); context.Logger.LogLine(Directory.GetCurrentDirectory() + "/Files"); try { var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(Environment.GetEnvironmentVariable("SQS_URL")); var sqsModel = JsonConvert.DeserializeObject <CompilerServiceSQSModel>(message.Body); var errors = new List <BuildError>(); List <CompileResult> buildStatus = null; var projectId = sqsModel.ProjectId; var buildVersion = sqsModel.BuildVersion; var user = sqsModel.UserEmail; //Update the kitsune project status to building if (APIHelpers.UpdateProjectStatus(user, projectId, buildVersion.ToString())) { context.Logger.LogLine(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion)); buildStatus = new BuildAndRunHelper().BuildProject(user, projectId, bool.Parse(Environment.GetEnvironmentVariable("IS_DEVELOPMENT_VERSION")), 1); if (buildStatus != null && buildStatus.Any()) { errors = new List <BuildError>(); foreach (var error in buildStatus) { if (!error.Success) { errors.AddRange(error.ErrorMessages.Select(x => new BuildError { Column = x.LinePosition, Line = x.LineNumber, Message = x.Message, ErrorStackTrace = x.Message, SourceMethod = "KitsuneCompiler", SourcePath = error.PageName })); } } APIHelpers.UpdateProjectErrorStatus(user, projectId, buildVersion, errors); Console.WriteLine(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion)); Log.Error(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion)); foreach (var err in errors) { Console.WriteLine(JsonConvert.SerializeObject(err)); Log.Error(JsonConvert.SerializeObject(err)); } buildStatus = null; } else { //UPDATE the project status if (APIHelpers.UpdateBuildStatus(user, projectId, buildVersion)) { Console.WriteLine(string.Format("Compilation done successful"), projectId, buildVersion.ToString()); Log.Information(string.Format("Compilation done successful"), projectId, buildVersion.ToString()); } } } var response = amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(new AmazonSQSMessageQueueModel <CompilerServiceSQSModel> { MessageBody = JsonConvert.DeserializeObject <CompilerServiceSQSModel>(message.Body), MessageId = message.MessageId, ReceiptHandle = message.ReceiptHandle, MessageAttributes = message.Attributes }); context.Logger.LogLine(response); } catch { } await Task.CompletedTask; }
public static void Process() { try { while (true) { string projectId = String.Empty; KitsuneKrawlerStatusCompletion stage = KitsuneKrawlerStatusCompletion.Error; try { var amazonSqsQueueHandler = new AmazonSQSQueueHandlers <KrawlSQSModel>(EnvironmentConstants.ApplicationConfiguration.CrawlerSQSUrl); //var task = amazonSqsQueueHandler.ReceiveMessageFromQueue(awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); var task = new AmazonAWSHelpers.Models.AmazonSQSMessageQueueModel <KrawlSQSModel>() { MessageBody = new KrawlSQSModel { ProjectId = "5ce4ef18abc486000121acb8", ReCrawl = true } }; if (task != null) { projectId = task.MessageBody.ProjectId; if (!String.IsNullOrEmpty(projectId)) { try { #region Initiate Logger Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup, projectId); #endregion #region Before Process try { ServiceInformationHelper serviceInfo = new ServiceInformationHelper(); Log.Information($"ProjectId : {projectId}, IP: {serviceInfo.GetInstancePrivateIpAddress()}"); } catch { } Uri uri = null; //Get the Details from DB var crawlDetails = MongoHelper.GetCrawlingDetails(projectId); if (crawlDetails == null) { throw new Exception("CrawlDetails was null"); } if (!Uri.TryCreate(crawlDetails.Url, UriKind.Absolute, out uri)) { throw new Exception(String.Format("Error Creating Uri from Url : {0}", crawlDetails.Url)); } stage = crawlDetails.Stage; #endregion #region Process var isTaskCompleted = false; Log.Information($"Started, Stage: {stage.ToString()}"); try { switch (stage) { case KitsuneKrawlerStatusCompletion.Initialising: InitialiseKrawlerStageHelper.InitialiseKrawler(projectId, uri); isTaskCompleted = true; break; case KitsuneKrawlerStatusCompletion.IdentifyingAllAssetsAndDownloadingWebpage: MigrationStageHelper.AnalyseTheWebsite(projectId, uri, crawlDetails.CrawlType.Equals(KrawlType.DeepKrawl)); isTaskCompleted = true; break; case KitsuneKrawlerStatusCompletion.DownloadingAllStaticAssetsToStorage: ResourcesStageHelper.DownloadTheResources(projectId, uri); isTaskCompleted = true; break; case KitsuneKrawlerStatusCompletion.UpdatingWebPagesWithNewStaticAssetUri: PlaceHolderReplacerHelper.ReplacePlaceHolder(projectId, uri); isTaskCompleted = true; break; default: break; } } catch (Exception ex) { MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError { ErrorMessage = ex.Message, Stage = stage }); } Log.Information($"Completed, Stage: {stage.ToString()}"); #endregion #region After Process amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); if (isTaskCompleted) { stage += 1; MongoHelper.UpdateCrawlStatsStage(projectId, stage); //Crawling completed successfully if (stage == KitsuneKrawlerStatusCompletion.Completed) { try { APIHelper.KrawlingCompletedUpdateKitsuneProjects(projectId); } catch (Exception ex) { Log.Error(ex, $"ProjectId:{projectId}, Message:Error updating DB after completion"); } } //If need furthur Process again push to sqs if (stage != KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains && stage != KitsuneKrawlerStatusCompletion.Error && stage < KitsuneKrawlerStatusCompletion.Completed) { amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); } //Event- Analyse Completed (select the domains to download and start next stage) if (stage == KitsuneKrawlerStatusCompletion.IdentifyingExternalDomains) { if (task.MessageBody.ReCrawl) { MongoHelper.UpdateCrawlStatsStage(projectId, stage + 1); amazonSqsQueueHandler.PushMessageToQueue(task.MessageBody, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); } else { APIHelper.RegisterAnalyseCompleteEvent(projectId); } } } else { Log.Error($"ProjectId:{projectId}, Message:Error as isTaskCompleted was false for projectId: {projectId}"); } #endregion } catch (Exception ex) { //Handle if any exception rises Log.Error($"ProjectId:{projectId}, Message:Error while Processing the project with Error : {ex.Message}"); MongoHelper.UpdateCrawlErrorMessage(projectId, new KrawlError { ErrorMessage = ex.Message, Stage = stage }); amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); } Logger.InitLogger(awsCloudWatchConfig.AWSAccessKey, awsCloudWatchConfig.AWSSecretKey, LogGroup); } else { Log.Error($"ProjectId:{projectId}, Message:Error while processing the Service as the projectId was null"); amazonSqsQueueHandler.DeleteMessageFromQueue(task, awsSQSConfig.AWSAccessKey, awsSQSConfig.AWSSecretKey); } } } catch (Exception ex) { //Error picking message from sqs //Error Deleting message from sqs Log.Error(ex, $"ProjectId:{projectId}, Message:Error while processing the Service after getting the value"); } } } catch (Exception ex) { Console.WriteLine($"Message:Error while polling from SQS, Exception : {ex.ToString()}"); } }