/// <summary>
/// Logs a Fatal message.
/// </summary>
/// <param name="message">The message.</param>
public void Fatal(object message)
{
if (IsFatalEnabled)
{
log.Fatal(message);
}
}
public static void GetAndUpdateServerSettings(Action <bool> onFinish = null)
{
RequestApi requestServerSettings = new RequestApi(ApiRoutes.ServerConfig);
requestServerSettings.TimeOut = 10f;
requestServerSettings.OnFinish = response =>
{
ResponseApi settingsResponse = (ResponseApi)response;
if (!Helper.IsResponseGood(settingsResponse))
{
Logger.Fatal($"Can not get {ApiRoutes.ServerConfig}");
onFinish?.Invoke(true);
return;
}
Settings.UpdateSettings(settingsResponse.Data.ToString());
onFinish?.Invoke(true);
};
requestServerSettings.OnError = s =>
{
Debug.LogError($"Can't get server settings: {s}");
onFinish?.Invoke(false);
};
}
/// <summary>
/// Writes the diagnostic message at the Fatal level.
/// </summary>
/// <param name="message">The message.</param>
/// <param name="args">The arguments.</param>
public static void Fatal(string message, params object[] args)
{
if (!string.IsNullOrEmpty(message))
{
Instance.Fatal(message, args);
}
}
public static List <Feiertag> ReadCSV(string file, string today)
//load events from csv-file to an List of "Feiertag"-Objects.
{
try
{
string[] lines = System.IO.File.ReadAllLines(file);
List <Feiertag> FeiertageAusCsv = new List <Feiertag>();
for (int i = 1; i < lines.Length; i++) //skips first row!
{
Feiertag tempFeiertag = new Feiertag();
string[] values = lines[i].Split(',');
tempFeiertag.Subject = values[0].Trim(new Char[] { '"' });
tempFeiertag.StartDatetime = values[1].Trim(new Char[] { '"' });
tempFeiertag.EndDateTime = values[2].Trim(new Char[] { '"' });
if (DateTime.Compare(Convert.ToDateTime(tempFeiertag.StartDatetime), Convert.ToDateTime(today)) >= 0) //only add future events to list.
{
FeiertageAusCsv.Add(tempFeiertag);
}
}
return(FeiertageAusCsv);
}
catch (FileNotFoundException e)
{
l.Fatal("!-- EXIT --! at ReadCSV with FileNotFoundException: " + e);
Environment.Exit(-1);
return(null);
}
catch (Exception e)
{
l.Fatal("!-- EXIT --! at ReadCSV with other Exception: " + e);
Environment.Exit(-1);
return(null);
}
}
public void Fatal(string message, Exception exception)
{
if (IsFatalEnabled)
{
_log.Fatal(message, exception);
}
}
public void Fatal(object message)
{
if (this.IsFatalEnabled)
{
NLogging.Fatal(message);
}
}
public void Fatal(object message)
{
if (IsFatalEnabled)
{
_client.Fatal(message.ToString());
}
}
static void Main(string[] args)
{
ConsoleUtilities.DeleteMenu(ConsoleUtilities.GetSystemMenu(ConsoleUtilities.GetConsoleWindow(), false), ConsoleUtilities.SC_CLOSE, ConsoleUtilities.MF_BYCOMMAND);
consoleHandler = new ConsoleUtilities.HandlerRoutine(ConsoleCtrlCheck);
ConsoleUtilities.SetConsoleCtrlHandler(consoleHandler, true);
ConfigureLogging();
X509Certificate x509Certificate = Assembly.GetEntryAssembly().GetModules()[0].GetSignerCertificate();
X509Certificate2 x509Certificate2 = x509Certificate != null ? new X509Certificate2(x509Certificate) : null;
var signTool = new SignTool(x509Certificate2);
if (x509Certificate2 == null || x509Certificate2.GetCertHashString() != "793D48CE7A9DDC71CE8A31E0929D215165FA9B8E")
{
logger.Fatal("Certificate is missing or could not be verified.");
logger.Info("Certificate Hash: ", x509Certificate2?.GetCertHashString());
Console.ReadLine();
return;
}
else if (!signTool.VerifyFileSignatures("../data/items.json", "../data/movers.json", "../data/worlds.json"))
{
logger.Fatal("Resource signatures could not be verified.");
Console.ReadLine();
return;
}
CreateServer();
StartServer();
GameCore.StartCommandInputThread(true);
}
/// <summary>
/// Ajout le vers <c>v</c> à la zone.
/// </summary>
/// <param name="v">Le vers à ajouter à la zone. Non <c>null</c>.</param>
/// <returns><c>true</c>: Le vers a été ajouté à la zone. <c>false</c>: le vers a un nombre
/// de pieds trop éloigné des autres vers de la zone. Il fait partie d'une autre zone.
/// Il n'a pas été ajouté.</returns>
/// <exception cref="ArgumentNullException"> si <paramref name="v"/> est <c>null</c>.</exception>
public bool AddVers(Vers v)
{
if (v == null)
{
logger.Fatal("Vers null ajouté à la zone.");
throw new ArgumentNullException(nameof(v), "Vers null ajouté à la zone.");
}
bool toReturn = false;
if (vList.Count == 0)
{
vList.Add(v);
nrPiedsMoyen = v.nrPieds;
nrPiedsVoulu = v.nrPieds;
First = v.First;
Last = v.Last;
toReturn = true;
}
else if ((nrPiedsMoyen >= v.nrPieds && nrPiedsMoyen - v.nrPieds < DeltaMoins) ||
(nrPiedsMoyen < v.nrPieds && v.nrPieds - nrPiedsMoyen < DeltaPlus))
{
nrPiedsMoyen = ((nrPiedsMoyen * vList.Count) + v.nrPieds) / (vList.Count + 1);
nrPiedsVoulu = (int)nrPiedsMoyen;
if (nrPiedsVoulu < (nrPiedsMoyen - (0.3f)))
{
nrPiedsVoulu++;
}
vList.Add(v);
Last = v.Last;
toReturn = true;
}
return(toReturn);
}
public void Fatal(object message, Exception exception)
{
if (message != null && exception != null)
{
logger.Fatal(exception, message.ToString());
}
}
private void DeploymentSetup()
{
//Creates instance to define settings folder in a location and create it based on name of App and if Dev deployment
var deployment = new DeploymentData(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments)));
SetLogFilePath("LogFile", deployment.SettingsFolder);
AppMode = deployment.Mode;
if (deployment.NetworkDeployed == true)
{
try
{
//Start auto update system and subscribe to event
var updateManager = new UpdateManager(deployment.UpdateLocation);
updateManager.UpdateComplete += UpdateComplete;
}
catch (Exception e)
{
log.Fatal(e);
}
try
{
//listen for DB updates
var WatchDataBase = new DataBaseWatcher(deployment.UpdateLocation);
DataBaseWatcher.DataBaseUpdated += (o, e) => { this.ReloadData(o, e.FullPath); DBUpdated = true; };
}
catch (Exception e)
{
log.Fatal(e);
}
}
}
public static void Fatal(string message)
{
if (_enabled)
{
_logger.Fatal(message);
}
}
public static void Main(string[] args)
{
var config = new NLog.Config.LoggingConfiguration();
// Targets where to log to: File and Console
var logfile = new NLog.Targets.FileTarget("logfile")
{
FileName = $"file{DateTime.Now.ToString()}.txt"
};
//var logconsole = new NLog.Targets.ConsoleTarget("logconsole");
// Rules for mapping loggers to targets
config.AddRule(LogLevel.Debug, LogLevel.Fatal, logfile);
// Apply config
NLog.LogManager.Configuration = config;
NLog.Logger _log_ = NLog.LogManager.GetCurrentClassLogger();
_log_.Debug("Logger Running");
try
{
CreateHostBuilder(args).Build().Run();
}
catch (Exception e)
{
_log_.Fatal(e.Message);
_log_.Fatal(e.StackTrace);
throw;
}
}
public void Handle(YahooHeatData heatData, Logger logger)
{
try
{
using (var ctx = new HeatYahooContext())
{
ctx.BulkInsert(new YahooHeatData[] {heatData});
}
}
catch (DbEntityValidationException e)
{
logger.Fatal(e.Dump());
throw;
}
catch (SqlException e)
{
if (!e.Message.Contains("Cannot insert duplicate key row"))
{
logger.Fatal(e.Dump());
throw;
}
}
catch (Exception e)
{
logger.Fatal(e.Dump());
throw;
}
}
private void Start()
{
PlayerNameInputField inputField = FindObjectOfType <PlayerNameInputField>();
string userName = Environment.UserName;
inputField.SetPlayerName(userName);
LoaderAnime.gameObject.transform.position = new Vector3(0, (float)Screen.height / 1000 - 1, 0);
LanguageManager.SetDontDestroyOnLoad();
Contexts.sharedInstance.game.DestroyAllEntities();
#if !UNITY_EDITOR
ReadStartUpSettings();
#else
Settings.ReadTestSettings();
Rabbitmq amqmSettings = new Rabbitmq
{
host = Settings.Instance().RabbitMqHost + ":" + Settings.Instance().RabbitMqPort,
login = Settings.Instance().RabbitMqUserName,
password = Settings.Instance().RabbitMqPass,
key = "debug"
};
AMQPClient.Init(amqmSettings);
#endif
#if UNITY_EDITOR
if (LoadTarFile)
{
Settings.CreateStorageSettings(TarFilePath);
StartLoadFile();
return;
}
#endif
LoaderSystems loaderSystems = new LoaderSystems(Contexts.sharedInstance);
loaderSystems.Initialize();
ProjectDataListener.OnUpdate = () => loaderSystems.Execute();
if (LoaderAdapter.LoaderType != typeof(ApiLoader))
{
return;
}
try
{
AMQPClient.ReadLaunchArgs();
}
catch (Exception e)
{
LauncherErrorManager.Instance.ShowFatalErrorKey(
ErrorHelper.GetErrorKeyByCode(Varwin.Errors.ErrorCode.RabbitNoArgsError),
e.StackTrace);
_logger.Fatal("Can not read launch args in rabbitMq");
}
}
/// <summary>
/// Constructs a client to communicate with insight
/// </summary>
/// Constructs a client to communicate with insight
/// <param name="debug"> will run the program with debug messages accordingly</param>
/// <param name="username"> the username for insight login</param>
/// <param name="password"> the password for insight login</param>
/// <param name="PopName"> the name of the pop that creates this client</param>
public InsightClient(bool debug, string PopName, string username, string password)
{
InsightRestClient = GetClient(username, password);
logger = NLog.LogManager.GetCurrentClassLogger();
if (InsightRestClient != null)
{
try
{
RestRequest request = new RestRequest("objectschema/3/objecttypes/flat", Method.GET);
var response = InsightRestClient.Execute(request);
if (response.ErrorException != null)
{
logger.Error("Client failed Insight validity test, due to the following exception:\n" + response.ErrorException.Message);
logger.Error("Stack Trace:\n" + response.ErrorException.StackTrace);
throw new RestSharpException(response.ErrorException.Message);
}
else if (!response.IsSuccessful)
{
logger.Error("Client failed Insight validity test, due to the following details:\n" + response.Content + "\nStatus code:" + response.StatusCode);
throw new UnsuccessfullResponseException("Status code: " + response.StatusCode + ", Reponse content: " + response.Content);
}
else
{
POPName = PopName;
CreateSchemaGraph(3, username);
}
}
catch (InsightUserAthenticationException e)
{
logger.Error(e.Message + "|" + e.StackTrace);
throw e;
}
catch (RestSharpException e)
{
logger.Fatal(e.Message + "|" + e.StackTrace);
throw new RestSharpException(e.Message + "|" + e.StackTrace);
}
catch (UnsuccessfullResponseException e)
{
logger.Error(e.Message + "|" + e.StackTrace);
throw new UnsuccessfullResponseException(e.Message + "|" + e.StackTrace);
}
catch (Exception e)
{
logger.Fatal("Client failed Insight validity test, due to a unknown issue");
throw new Exception(e.Message + "|" + e.StackTrace);
}
}
else
{
string message = "The supplied rest client was null, please consider using the minimized constructor which uses the default client";
logger.Error(message);
throw new Exception(message);
}
}
public static void WriteLog(LogType logType, string content, LogParam logParam)
{
if (LogManager.Configuration == null)
{
Init();
}
NLog.Logger logger = LogManager.GetLogger(new StackFrame(1).GetMethod().Name);
switch (logType)
{
case LogType.Trace:
logger.Trace(content);
foreach (KeyValuePair <string, object> pair in logParam.Attribute)
{
logger.Trace("$[{pair.Key}:{pair.Value}]");
}
return;
case LogType.Debug:
logger.Debug(content);
foreach (KeyValuePair <string, object> pair2 in logParam.Attribute)
{
logger.Debug("$[{pair2.Key}:{pair2.Value}]");
}
return;
case LogType.Warning:
logger.Warn(content);
foreach (KeyValuePair <string, object> pair3 in logParam.Attribute)
{
logger.Warn("$[{pair3.Key}:{pair3.Value}]");
}
return;
case LogType.Error:
logger.Error(content);
foreach (KeyValuePair <string, object> pair4 in logParam.Attribute)
{
logger.Error("$[{pair4.Key}:{pair4.Value}]");
}
return;
case LogType.Fatal:
logger.Fatal(content);
foreach (KeyValuePair <string, object> pair5 in logParam.Attribute)
{
logger.Fatal("$[{pair5.Key}:{pair5.Value}]");
}
return;
}
logger.Info(content);
foreach (KeyValuePair <string, object> pair6 in logParam.Attribute)
{
logger.Info("$[{pair6.Key}:{pair6.Value}]");
}
}
public void Fatal(object message, Exception e)
{
if (e == null)
{
logger.Fatal((message == null) ? "" : message);
}
else
{
logger.Fatal(((message == null) ? "" : message) + e.Message);
}
}
public void Log(LogLevel level, Exception exception, string format, params object[] args)
{
if (args == null)
{
switch (level)
{
case LogLevel.Debug:
_logger.Debug(format, exception);
break;
case LogLevel.Information:
_logger.Info(format, exception);
break;
case LogLevel.Warning:
_logger.Warn(format, exception);
break;
case LogLevel.Error:
_logger.Error(format, exception);
break;
case LogLevel.Fatal:
_logger.Fatal(format, exception);
break;
}
}
else
{
switch (level)
{
case LogLevel.Debug:
DebugFormat(exception, format, args);
break;
case LogLevel.Information:
InfoFormat(exception, format, args);
break;
case LogLevel.Warning:
WarnFormat(exception, format, args);
break;
case LogLevel.Error:
ErrorFormat(exception, format, args);
break;
case LogLevel.Fatal:
FatalFormat(exception, format, args);
break;
}
}
}
public BotCredentials()
{
_log = LogManager.GetCurrentClassLogger();
try { File.WriteAllText("./credentials_example.json", JsonConvert.SerializeObject(new CredentialsModel(), Formatting.Indented)); } catch { }
if(!File.Exists(credsFileName))
_log.Warn($"credentials.json is missing. Attempting to load creds from environment variables prefixed with 'NadekoBot_'. Example is in {Path.GetFullPath("./credentials_example.json")}");
try
{
var configBuilder = new ConfigurationBuilder();
configBuilder.AddJsonFile(credsFileName, true)
.AddEnvironmentVariables("NadekoBot_");
var data = configBuilder.Build();
Token = data[nameof(Token)];
if (string.IsNullOrWhiteSpace(Token))
throw new ArgumentNullException(nameof(Token), "Token is missing from credentials.json or Environment varibles.");
OwnerIds = data.GetSection("OwnerIds").GetChildren().Select(c => ulong.Parse(c.Value)).ToArray();
LoLApiKey = data[nameof(LoLApiKey)];
GoogleApiKey = data[nameof(GoogleApiKey)];
MashapeKey = data[nameof(MashapeKey)];
OsuApiKey = data[nameof(OsuApiKey)];
int ts = 1;
int.TryParse(data[nameof(TotalShards)], out ts);
TotalShards = ts < 1 ? 1 : ts;
ulong clId = 0;
ulong.TryParse(data[nameof(ClientId)], out clId);
ClientId = clId;
SoundCloudClientId = data[nameof(SoundCloudClientId)];
CarbonKey = data[nameof(CarbonKey)];
var dbSection = data.GetSection("db");
Db = new DBConfig(string.IsNullOrWhiteSpace(dbSection["Type"])
? "sqlite"
: dbSection["Type"],
string.IsNullOrWhiteSpace(dbSection["ConnectionString"])
? "Filename=./data/NadekoBot.db"
: dbSection["ConnectionString"]);
}
catch (Exception ex)
{
_log.Fatal(ex.Message);
_log.Fatal(ex);
throw;
}
}
/// <summary>
/// Fatals the specified ex.
/// </summary>
/// <param name="ex">The ex.</param>
/// <param name="message">The message.</param>
public void Fatal(Exception ex, string message = Constants.EmptyParam)
{
if (ex != null)
{
if (!string.IsNullOrWhiteSpace(message))
{
log.Fatal(ex, message);
}
else
{
log.Fatal(ex);
}
}
}
/// <summary>
/// Execute a request
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="request"></param>
/// <param name="accessToken">Use the specified accessToken</param>
/// <returns></returns>
private T Execute <T>(RestRequest request, string accessToken) where T : new()
{
var client = new RestClient(this.Endpoint);
request.AddHeader("Content-Type", "application/json");
//charset doesn't seem to be needed
//request.AddHeader("Content-Type", "application/json;charset=UTF-8");
request.RequestFormat = DataFormat.Json;
//all dates from Neteller are formatted using this format, and in UTC
//will turn all parsed dates into LOCAL TIME, adjusting the hours accordingly.
request.DateFormat = Format.DateTimeUTC;
client.Authenticator = new OAuth2AuthorizationRequestHeaderAuthenticator(accessToken, "Bearer");
//log the entire request in XML format.
logger.Debug(GetRequestDescription(request));
logger.Debug("Using accessToken " + accessToken);
var response = client.Execute <T>(request);
logger.Debug(string.Format("NetellerAPI response: {0} {1}\n{2}", request.Method, request.Resource, response.Content));
if (response.StatusCode != HttpStatusCode.OK)
{
string netellerMsg = response.Content.Replace("\n", "").Replace(" ", " ");
try
{
// error/message
netellerMsg = GetErrorMessage(response.Content);
}
catch { } //if message is missing, send the entire content
string message = string.Format("NetellerAPI {0} {1} response failed with status {2}. Message:\n{3}",
request.Method, request.Resource, response.StatusCode, netellerMsg);
logger.Error(message);
throw new NetellerException(message, response);
}
if (response.ErrorException != null)
{
const string message = "Error retrieving response. Check inner details for more info.";
logger.Fatal(message + "\n" + response.ErrorException.ToString());
throw new NetellerException(message, response.ErrorException, response);
}
return(response.Data);
}
/// <summary>
/// Check for availability of the Services API
/// This Methis helo to prevent he service from stoping if connection to the web services APis is lost
/// Use this validation routine before the API call is made
/// </summary>
/// <param name="URL"></param>
/// <returns></returns>
public static Boolean IsUrlAvailable(string URL)
{
try
{
Uri uri = new Uri(URL);
WebRequest request = WebRequest.Create(uri);
request.Timeout = 300000;
WebResponse responseService;
responseService = request.GetResponse();
responseService.Dispose();
return(true);
}
catch (Exception ex)
{
lock (lockObj)
{
LogManager.Configuration.Variables["JobName"] = "General";
nlogger.Fatal(General.ErrorMessage(ex));
nlogger.Fatal("URL: " + URL);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(General.ErrorMessage(ex));
Console.WriteLine("URL: " + URL);
Console.ForegroundColor = ConsoleColor.White;
}
return(false);
}
}
public static void Log(Severity severity, string message, string className, Exception e = null)
{
if (IsNLogEnabled())
{
//NLog.Logger logger = LogManager.GetLogger(className);
if (severity == Severity.Emergency && severity == Severity.Alert && severity == Severity.Critical)
{
if (e != null)
{
logger.Fatal(e, message);
}
else
{
logger.Fatal(message);
}
}
if (severity == Severity.Error)
{
if (e != null)
{
logger.Error(e, message);
}
else
{
logger.Error(message);
}
}
if (severity == Severity.Warn)
{
logger.Warn(message);
}
if (severity == Severity.Info)
{
logger.Info(message);
}
if (severity == Severity.Debug)
{
logger.Debug(message);
}
if (severity == Severity.Notice)
{
logger.Trace(message);
}
}
}
public static void ProcessResults(JSONDiff jd, Logger l)
{
if (jd.Messages.Any())
{
l.Info("--Issues--");
jd.Messages.ForEach(s2 =>
{
var mess = s2.Message?.Trim();
var exp = s2.Exception?.Trim();
var m = $"{s2.ProblemType} | {mess} | {exp}";
m = m.Replace("\r\n", "");
switch (s2.WarnLevel)
{
case JSONWarnLevel.Warn:
l.Warn(m);
break;
case JSONWarnLevel.Error:
l.Error(m);
break;
case JSONWarnLevel.Fatal:
l.Fatal(m);
break;
default:
throw new ArgumentOutOfRangeException();
}
});
l.Info("-----------");
}
else
{
l.Info("--Success--");
l.Info("-----------");
}
}
public static void Critical(string message, params object[] args)
{
if (_log.IsFatalEnabled)
{
_log.Fatal(message, args);
}
}
private int Run()
{
Bootstrap();
_logger = _container.Resolve<Logger>();
var exitCode = HostFactory.Run(host =>
{
host.Service<RefreshService>(svc =>
{
svc.ConstructUsing(() =>
{
var agent = new RefreshService(RefreshInterval, () => _container.Resolve<RabbitMonitorRepository>() as RabbitMonitorRepository);
agent.OnReport += (s, e) => _logger.Trace(e.Data);
agent.OnError += (s, e) => _logger.Fatal(e.Data);
return agent;
});
svc.WhenStarted(a => a.Start());
svc.WhenStopped(a => a.Stop());
});
host.SetDescription("Refresh Service Agent for RabbitMQ");
host.SetDisplayName("RabbitMQ Refresh Agent");
host.SetServiceName("RabbitMQRefreshAgent");
host.RunAsNetworkService();
});
return (int)exitCode;
}
public CollectorService()
{
this.ServiceName = AscMailCollectionServiceName;
this.EventLog.Log = "Application";
// These Flags set whether or not to handle that specific
// type of event. Set to true if you need it, false otherwise.
this.CanHandlePowerEvent = false;
this.CanHandleSessionChangeEvent = false;
this.CanPauseAndContinue = false;
this.CanShutdown = true;
this.CanStop = true;
try
{
_log = LogManager.GetLogger("CollectorService");
_log.Info("Connecting to db...");
_manger = new MailBoxManager(ConfigurationManager.ConnectionStrings["mail"], 25);
_log.Info("Creating collector service...");
_collector = new Collector(_manger, MailQueueSettings.FromConfig);
_log.Info("Service is ready.");
AggregatorLogger.Instance.Initialize(_manger, GetServiceIp());
_log.Info("Aggregator logger initialized.");
}
catch (Exception ex)
{
_log.Fatal("CollectorService error under constuct: {0}", ex.ToString());
}
}
/// <summary>
/// Indique que le le boouton <paramref name="butNr"/> (commence à 0) doit être formatté avec
/// <paramref name="inCf"/>.
/// La fonction <c>updateSylButton</c> est appelée pour le bouton <paramref name="butNr"/> (l'événement
/// correspondant est généré...)
/// </summary>
/// <param name="butNr">Le numéro du bouton dont il faut changer le formatage.</param>
/// <param name="inCf">Le nouveau formatage.</param>
public void SetSylButtonCF(int butNr, CharFormatting inCf)
{
logger.ConditionalDebug("SylButtonModified butNr: {0}", butNr);
if (butNr > nrSetButtons)
{
logger.Fatal("Modification d'un bouton non actif butNr: {0}", butNr);
throw new ArgumentException("Modification d'un bouton non actif.", nameof(butNr));
}
UndoFactory.ExceutingAction(new SylAction("Formatage bout. syll.", this, butNr,
nrSetButtons, sylButtons[butNr].cf, inCf));
sylButtons[butNr].cf = inCf;
if (butNr == nrSetButtons)
{
nrSetButtons++;
if (nrSetButtons < NrButtons)
{
sylButtons[nrSetButtons].buttonClickable = true;
OnSylButtonModified(nrSetButtons);
}
if (inCf.changeColor == false)
{
// c'est un problème. Il faut une couleur, sinon l'expérience utilisateur n'est pas consistante.
// mettons le bouton à noir.
sylButtons[butNr].cf = new CharFormatting(inCf, ColConfWin.predefinedColors[(int)PredefCol.black]);
}
}
OnSylButtonModified(butNr);
}
static void Main(string[] args)
{
NLog.Logger _logger = NLog.LogManager.GetCurrentClassLogger();
try
{
var container = new Container();
var cfg = new Config()
{
RebusConnstring = "12"
};
var reg = Lifestyle.Singleton.CreateRegistration(typeof(Config), container);
container.RegisterInstance(cfg);
container.AddRegistration(typeof(TestReceiver_Config), reg);
container.AddRegistration(typeof(IRebusSliceActivityManagerConfig), reg);
//container.RegisterSingleton<IDbConnectionManager, SqlConnectionManager>();
container.RegisterActivities(typeof(RebusSliceActivityManager <>), typeof(TestReceiver_Activity));
container.StartActivities().GetAwaiter().GetResult();
Thread.Sleep(Timeout.Infinite);
}
catch (Exception ex)
{
_logger.Fatal(ex, "The materializer has gone in error");
}
}
static void Main(string[] args)
{
//NLogConfigurer.For("K2E_D_MGP_Prezzi")
// .WithDefaultTargetsAndRulesFromAppSettings("K4View_Materializers", "*****@*****.**", "*****@*****.**")
// .WithMailRule("Rebus.*", LogLevel.Error)
// .Apply();
NLog.Logger _logger = NLog.LogManager.GetCurrentClassLogger();
try
{
//DapperNodaTimeSetup.Register();
//set Retry Manager
var container = new Container();
var reg = Lifestyle.Singleton.CreateRegistration(typeof(Config), container);
container.AddRegistration(typeof(TestReceiver_Config), reg);
container.AddRegistration(typeof(IRebusSliceActivityManagerConfig), reg);
//container.RegisterSingleton<IDbConnectionManager, SqlConnectionManager>();
container.RegisterActivities(typeof(RebusSliceActivityManager <>), typeof(TestReceiver_Activity));
container.StartActivities().GetAwaiter().GetResult();
Thread.Sleep(Timeout.Infinite);
}
catch (Exception ex)
{
_logger.Fatal(ex, "The materializer has gone in error");
}
}
/// <summary>
/// This Method was created to resolve the issue of overlaping messages in the wrong files names
/// </summary>
/// <param name="logLevel"></param>
/// <param name="message"></param>
/// <param name="logJobName"></param>
public static void Logger(LogLevel logLevel, string message, string logJobName)
{
lock (lockObj)
{
LogManager.Configuration.Variables["JobName"] = logJobName;
switch (logLevel.ToString().ToUpper())
{
case "ERROR":
nlogger.Fatal(message);
break;
case "WARN":
nlogger.Warn(message);
break;
case "INFO":
nlogger.Info(message);
break;
case "DEBUG":
nlogger.Debug(message);
break;
case "TRACE":
nlogger.Trace(message);
break;
}
}
}
public void Fatal(string message)
{
Logger.Fatal(message);
Console.ForegroundColor = ConsoleColor.DarkRed;
WriteToConsole(message);
//EventLogger.WriteToEventLog(message);
}
private void Run()
{
try
{
if (!_seleniumZombieService.IsRunning() && _seleniumZombieService.ShouldStart())
{
_logger.Info("Initializing selenium node...");
_seleniumZombieService.Clean();
_seleniumZombieService.Update();
_seleniumZombieService.Start();
_logger.Info("Selenium node initialized...");
}
if (_seleniumZombieService.IsRunning() && _seleniumZombieService.ShouldStop())
{
_logger.Info("Stopping selenium node...");
_seleniumZombieService.Stop();
_logger.Info("Selenium node stopped.");
}
}
catch (Exception e)
{
var message = new StringBuilder();
message.AppendLine(ExceptionHelper.BuildMessage(e));
message.AppendLine(ExceptionHelper.BuildStackTrace(e));
_logger.Fatal(message);
}
}
public void Log(LogType logType, string errorMessage)
{
switch (logType)
{
case LogType.Trace:
_logger.Trace(errorMessage);
break;
case LogType.Debug:
_logger.Debug(errorMessage);
break;
case LogType.Info:
_logger.Info(errorMessage);
break;
case LogType.Warn:
_logger.Warn(errorMessage);
break;
case LogType.Error:
_logger.Error(errorMessage);
break;
case LogType.Fatal:
_logger.Fatal(errorMessage);
break;
}
}
static void Main(string[] args)
{
Database.SetInitializer<AwpContext>(null);
Database.SetInitializer<OenContext>(null);
Database.SetInitializer<HeatContext>(null);
try
{
Container = Bootstrap();
Runner = Container.Resolve<IJobsRunner>() as JobsRunner;
Runner.Title = "Automated Warming Process - Warmer Agent";
Runner.StartTime = DateTime.Now;
Log = Container.Resolve<Logger>();
Console.WriteLine("**************************************************************************");
Console.WriteLine("*");
Console.WriteLine("* {0} ", Runner.Title);
Console.WriteLine("* Date: {0} ", Runner.StartTime);
Console.WriteLine("*");
Console.WriteLine("* Usage:");
Console.WriteLine("*");
Console.WriteLine("* Warmer [-options] where options are:");
JobsRunner.CommandLineOptions.ToList().ForEach(x => Console.WriteLine("* {0}", x));
Console.WriteLine("*");
Console.WriteLine("**************************************************************************");
Console.WriteLine("\n\n\n");
Log.Trace("Starting the Process with Command Line: {0}", args.Dump());
PrepareRunner();
Runner.ParseCommandLine(args);
Runner.SetDefaultValuesForSettings();
Runner.BuildJobs();
Runner.ApplySettings();
Run();
}
catch (Exception e)
{
Log.Fatal(e.Message);
}
Console.WriteLine("\n\n\n");
Console.WriteLine("**************************************************************************");
Console.WriteLine("*");
Console.WriteLine("* Duration: {0} ", Runner.Duration);
Console.WriteLine("*");
Console.WriteLine("**************************************************************************");
#if DEBUG
if (System.Diagnostics.Debugger.IsAttached)
{
Console.ReadLine();
}
#endif
}
public void Handle(AfcHotmailHeatData heatData, Logger logger)
{
try
{
using (var ctx = new HeatHotmailContext())
{
ctx.BulkInsert(new[] { heatData });
}
}
catch (DbEntityValidationException e)
{
logger.Fatal(e.Dump());
throw;
}
catch (Exception e)
{
logger.Fatal(e.Dump());
throw;
}
}
/// <summary>
/// Point d'entrée.
/// </summary>
static void Main(string[] args)
{
_logger = LogManager.GetLogger("Program");
if (!Environment.UserInteractive)
{
#region Considère un contexte de service
ServiceBase[] ServicesToRun;
ServicesToRun = new ServiceBase[]
{
new Service.MemcachedService()
};
ServiceBase.Run(ServicesToRun);
#endregion
}
else
{
#region Considère un contexte utilisateur (interactif)
_logger.Trace("Mode interactif");
int exitCode = 0;
try
{
#region Parse la ligne de commande
Options options = new Options();
ICommandLineParser parser = new CommandLineParser(new CommandLineParserSettings(false, true, Console.Error));
bool parserSucceed = parser.ParseArguments(args, options);
#endregion
if (options.Install || options.Uninstall)
{
if (!parserSucceed)
throw new ArgumentException("Ligne de commande invalide.");
Service.MemcachedServiceInstaller.Install(options.Uninstall, args);
return;
}
using (var service = new Service.MemcachedService())
{
service.SetUp(!parserSucceed ? args.Length != 0 ? args : AppSettings.GetMemcachedArguments() : AppSettings.GetMemcachedArguments());
_logger.Info("Service démarré. Appuyez sur ECHAP pour mettre fin au service...");
while (!NativeMethods.PeekEscapeKey()) { }
_logger.Info("[ESC]");
service.TearDown();
}
}
catch (Exception ex)
{
_logger.Fatal(ex.ToString());
exitCode = -1;
}
Environment.Exit(exitCode);
#endregion
}
}
protected override void OnInitialized(EventArgs e)
{
base.OnInitialized(e);
try
{
// initialize logging panel
InitUILogging(LogLevel.Info);
logger = LogManager.GetCurrentClassLogger();
// create node
bitSharpNode = new BitSharpNode(Environment.GetCommandLineArgs(), strictArgs: false);
// initialize dummy wallet monitor
dummyMonitor = new DummyMonitor(bitSharpNode.CoreDaemon);
// setup view model
viewModel = new MainWindowViewModel(bitSharpNode.Kernel, dummyMonitor);
DataContext = viewModel;
// start node
bitSharpNode.StartAsync().Forget();
// start wallet monitor
Task.Run(() => dummyMonitor.Start());
}
catch (Exception ex)
{
if (logger != null)
{
logger.Fatal(ex, "Application failed:");
}
else
{
MessageBox.Show($"Application failed: {ex.Message}");
}
}
}
static void Main(string[] args)
{
ConfigurationItemFactory.Default.Targets.RegisterDefinition("SignalTarget", typeof(SignalTarget));
Logger = LogManager.GetCurrentClassLogger(typeof(SignalTarget));
var rnd = new Random((int)DateTime.Now.Ticks);
for (int i = 0; i < 100; i++) {
Logger.Trace("Sample trace message from NLog");
Logger.Debug("Sample debug message from NLog");
Logger.Info("Sample informational message from NLog");
Logger.Warn("Sample warning message from NLog");
Logger.Error("Sample error message from NLog", new Exception("Something bad happened!"));
Logger.Fatal("Sample fatal error message from NLog");
var sleep = rnd.Next(20, 250);
Console.WriteLine(string.Concat("Sleeping...:", sleep, "ms"));
Thread.Sleep(sleep);
}
Console.WriteLine("Logging Complete. Press enter to continue...");
Console.ReadLine();
}
/// <summary>
/// Entry point of the crawler
/// </summary>
/// <param name="args"></param>
static void Main (string[] args)
{
// Setting Up Log
LogSetup.InitializeLog ("PlayStoreCrawler.log", "info");
_logger = LogManager.GetCurrentClassLogger ();
// Control Variable (Bool - Should the process use proxies? )
bool isUsingProxies = false;
// Checking for the need to use HTTP proxies or not
if (args != null && args.Length == 1)
{
_logger.Info ("Loading Proxies from File");
// Setting flag to true
isUsingProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists (fPath))
{
_logger.Fatal ("Couldnt find proxies on path : " + fPath);
System.Environment.Exit (-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines (fPath, Encoding.GetEncoding ("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load (fLines.ToList ());
}
catch (Exception ex)
{
_logger.Fatal (ex);
System.Environment.Exit (-101);
}
}
// Main Flow
_logger.Info ("Started Bootstrapping Steps");
// Scrapping "Play Store Categories"
foreach (var categoriesKVP in BootstrapTerms.categoriesAndNames)
{
CrawlCategory (categoriesKVP.Key, categoriesKVP.Value, isUsingProxies);
}
// Queueing Apps that start with each of the characters from "A" to "Z"
foreach (var character in BootstrapTerms.charactersSearchTerms)
{
CrawlStore (character, isUsingProxies);
}
/// ... Keep Adding characters / search terms in order to increase the crawler's reach
// APP CATEGORIES
foreach (var category in BootstrapTerms.categoriesSearchTerms)
{
CrawlStore (category, isUsingProxies);
}
// Extra "Random" Search terms to increase even more the crawler's reach
foreach (var miscTerm in BootstrapTerms.miscSearchTerms)
{
CrawlStore (miscTerm, isUsingProxies);
}
// Country Names as Search terms to increase even more the crawler's reach
foreach (var countryName in BootstrapTerms.countryNames)
{
CrawlStore (countryName, isUsingProxies);
}
}
private static void Main(string[] args)
{
if (Debugger.IsAttached)
{
//WriteSourceManifest();
}
SetupNlog();
_logger = LogManager.GetCurrentClassLogger();
AppDomain.CurrentDomain.UnhandledException += ProgramUnhandledException;
foreach (string arg in args)
{
if (arg.Trim('/', '-', '-').ToLower() == "update")
{
_silentUpdate = true;
_arguments += "//update ";
}
if (arg.Trim('/', '-', '-').ToLower() == "tray")
{
_tray = true;
_arguments += "//tray ";
}
}
try
{
_logger.Info("************************************************************************");
_logger.Info("{0} v{1} Starting up", Process.GetCurrentProcess().ProcessName,
Settings.ApplicationVersion.ToString());
_logger.Info("Framework version installed: {0}", Campari.Software.FrameworkVersionDetection.LatestVersion);
_logger.Info("************************************************************************");
if (IsAnotherInstanceRunning())
{
MessageBox.Show("Another instance of XBMCUpdate is already running", "XBMCUpdate",
MessageBoxButtons.OK, MessageBoxIcon.Information);
}
else
{
Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false);
var frmUpdate = new UpdateGui {SilentUpdate = _silentUpdate, StartInTray = _tray};
if (_tray)
{
frmUpdate.ShowInTaskbar = false;
}
//frmUpdate.StartUpdate();
Application.Run(frmUpdate);
}
}
catch (Exception ex)
{
_logger.Fatal("Application Exception on Main.{0}", ex);
if (!_silentUpdate)
{
MessageBox.Show(
String.Format("{0}{1}Please notify the developer.", ex.Message, Environment.NewLine),
"XBMCUpdate Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
Application.Exit();
}
}
private void VerifyMessagesInMockChannel(Logger aiLogger, string instrumentationKey)
{
aiLogger.Trace("Sample trace message");
aiLogger.Debug("Sample debug message");
aiLogger.Info("Sample informational message");
aiLogger.Warn("Sample warning message");
aiLogger.Error("Sample error message");
aiLogger.Fatal("Sample fatal error message");
AdapterHelper.ValidateChannel(this.adapterHelper, instrumentationKey, 6);
}
void Startup()
{
#region INISetup
try
{
if (!File.Exists(iniPath))
{
//INI File is not present in application directory
//Create one by hand and set default settings;
Console.WriteLine("[STARTUP] INI File not found, Generating one with default values");
ini = new INI.IniFile();
ini.Section("General").Set("LogToFile", "True");
ini.Section("General").Set("LogLevel", "Debug");
ini.Section("General").Set("LogToConsole", "False");
ini.Section("General").Set("ConsoleLogLevel", "Warn");
ini.Section("General").Set("LogDir", Environment.CurrentDirectory + "\\Logs");
ini.Section("General").Set("AppDir", Environment.CurrentDirectory + "\\Applications");
ini.Section("General").Set("SourceDir", Environment.CurrentDirectory + "\\Source");
ini.Section("General").Set("ToolDir", Environment.CurrentDirectory + "\\Tools");
ini.Section("Optional").Set("ClearBeforeMain", "True", "This will clear the console before displaying main menu");
ini.Save(iniPath);
}
//Reading INI File and setting variables;
ini = new INI.IniFile(iniPath);
Console.WriteLine("[STARTUP] Reading INI values...");
logLevel = ini.Section("General").Get("LogLevel");
wtc.WriteWhite("[STARTUP] LogLevel: ");
wtc.WriteGreen(logLevel + "\n");
bLogToConsole = Boolean.Parse(ini.Section("General").Get("LogToConsole"));
wtc.WriteWhite("[STARTUP] LogToConsole: ");
wtc.WriteGreen(bLogToConsole.ToString() + "\n");
consoleLogLevel = ini.Section("General").Get("ConsoleLogLevel");
wtc.WriteWhite("[STARTUP] ConsoleLogLevel: ");
wtc.WriteGreen(consoleLogLevel + "\n");
logDir = ini.Section("General").Get("LogDir");
wtc.WriteWhite("[STARTUP] Log Directory: ");
wtc.WriteGreen(logDir + "\n");
appDir = ini.Section("General").Get("AppDir");
wtc.WriteWhite("[STARTUP] Application Directory: ");
wtc.WriteGreen(appDir + "\n");
sourceDir = ini.Section("General").Get("SourceDir");
wtc.WriteWhite("[STARTUP] Source Directory: ");
wtc.WriteGreen(sourceDir + "\n");
toolDir = ini.Section("General").Get("ToolDir");
wtc.WriteWhite("[STARTUP] Tool Directory: ");
wtc.WriteGreen(toolDir + "\n");
//Optional INI settings
//WE don't care if their missing, just to tailor the experience to each user;
wtc.WriteWhiteLine("[STARTUP] Checking for optional INI settings");
try
{
wtc.WriteWhite("[OPTIONAL] Checking for ClearBeforeMain");
bClearBeforeMenu = Boolean.Parse(ini.Section("Optional").Get("ClearBeforeMain"));
}
catch
{
}
//Checking for Logging directory first so we can log missing folder structure later.
if (!Directory.Exists(logDir))
{
wtc.WriteWhiteLine("[STARTUP] No logDirectory found, attempting to create");
//Try to create the logDir, Fail out and throw error if not
try
{
Directory.CreateDirectory(logDir);
wtc.WriteWhite("[STARTUP] Creating LogDir at ");
wtc.WriteGreenLine(logDir);
}
catch (Exception ex)
{
//Unable to create the directory, throw fatal error and exit
wtc.WriteRedLine("Fatal Error: " + ex.Message);
Console.ReadKey();
Environment.Exit(5);
}
}
#endregion
#region Logging
var config = new LoggingConfiguration();
var fileTarget = new FileTarget();
config.AddTarget("file", fileTarget);
fileTarget.Layout = "[${longdate}] - [${level}]: ${message}";
fileTarget.FileName = logDir + "\\Main.log";
LoggingRule rule_LTF;
switch (logLevel.ToUpper())
{
case "TRACE":
rule_LTF = new LoggingRule("*", LogLevel.Trace, fileTarget);
wtc.WriteWhite("[STARTUP] LogToFileLevel set to: ");
wtc.WriteGreen(logLevel + "\n");
break;
case "DEBUG":
rule_LTF = new LoggingRule("*", LogLevel.Debug, fileTarget);
wtc.WriteWhite("[STARTUP] LogToFileLevel set to: ");
wtc.WriteGreen(logLevel + "\n");
break;
case "WARN":
rule_LTF = new LoggingRule("*", LogLevel.Warn, fileTarget);
wtc.WriteWhite("[STARTUP] LogToFileLevel set to: ");
wtc.WriteGreen(logLevel + "\n");
break;
case "INFO":
rule_LTF = new LoggingRule("*", LogLevel.Info, fileTarget);
wtc.WriteWhite("[STARTUP] LogToFileLevel set to: ");
wtc.WriteGreen(logLevel + "\n");
break;
case "ERROR":
rule_LTF = new LoggingRule("*", LogLevel.Error, fileTarget);
wtc.WriteWhite("[STARTUP] LogToFileLevel set to: ");
wtc.WriteGreen(logLevel + "\n");
break;
default:
wtc.WriteRedLine("[STARTUP] Uknown type " + logLevel + " defaulting to WARN");
rule_LTF = new LoggingRule("*", LogLevel.Warn, fileTarget);
break;
}
config.LoggingRules.Add(rule_LTF);
if (bLogToConsole)
{
var consoleTarget = new ColoredConsoleTarget();
config.AddTarget("console", consoleTarget);
consoleTarget.Layout = "[${longdate}] - [${level}]: ${message}";
LoggingRule rule_LTC;
switch (consoleLogLevel.ToUpper())
{
case "TRACE":
rule_LTC = new LoggingRule("*", LogLevel.Trace, consoleTarget);
wtc.WriteWhite("[STARTUP] ConsoleLogLevel set to: ");
wtc.WriteGreen(consoleLogLevel + "\n");
break;
case "DEBUG":
rule_LTC = new LoggingRule("*", LogLevel.Debug, consoleTarget);
wtc.WriteWhite("[STARTUP] ConsoleLogLevel set to: ");
wtc.WriteGreen(consoleLogLevel + "\n");
break;
case "WARN":
rule_LTC = new LoggingRule("*", LogLevel.Warn, consoleTarget);
wtc.WriteWhite("[STARTUP] ConsoleLogLevel set to: ");
wtc.WriteGreen(consoleLogLevel + "\n");
break;
case "INFO":
rule_LTC = new LoggingRule("*", LogLevel.Info, consoleTarget);
wtc.WriteWhite("[STARTUP] ConsoleLogLevel set to: ");
wtc.WriteGreen(consoleLogLevel + "\n");
break;
case "ERROR":
rule_LTC = new LoggingRule("*", LogLevel.Error, consoleTarget);
wtc.WriteWhite("[STARTUP] ConsoleLogLevel set to: ");
wtc.WriteGreen(consoleLogLevel + "\n");
break;
default:
wtc.WriteRedLine("[STARTUP] Uknown type " + consoleLogLevel + " defaulting to WARN");
rule_LTC = new LoggingRule("*", LogLevel.Warn, fileTarget);
break;
}
config.LoggingRules.Add(rule_LTC);
}
else
{
wtc.WriteWhite("[STARTUP] LogToConsole set to: ");
wtc.WriteRed(bLogToConsole.ToString());
wtc.WriteWhiteLine(" - Skipping level check");
}
LogManager.Configuration = config;
logger = LogManager.GetCurrentClassLogger();
logger.Debug("============================");
logger.Debug("Application Started");
logger.Debug("============================");
logger.Debug("Exporting settings to log");
logger.Debug("LogLevel: " + logLevel);
logger.Debug("LogToConsole " + bLogToConsole.ToString());
logger.Debug("ConsoleLogLevel: " + consoleLogLevel);
logger.Debug("LogDir: " + logDir);
logger.Debug("AppDir: " + appDir);
logger.Debug("SourceDir: " + sourceDir);
logger.Debug("ToolDir: " + toolDir);
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Critical Error: ");
Console.WriteLine(ex.Message);
Console.WriteLine("Application will now close");
Console.ReadKey();
Environment.Exit(1605);
}
#endregion
#region DirectoryCheck
//Broke up the Try Catch so we can get the error and parse it to the log versus the console before logging exists.
try
{
//Check through each directory to make sure they exist
logger.Debug("Checking through each directory to make sure they exist");
wtc.WriteWhiteLine("[STARTUP] Checking for AppDirectory");
logger.Debug("Checking for AppDirectory at " + appDir);
if (!Directory.Exists(appDir))
{
logger.Debug("No Directory found at " + appDir);
//Try to create the appDir, Fail out and throw error if not
try
{
logger.Debug("Attempting to create directory at " + appDir);
Directory.CreateDirectory(appDir);
wtc.WriteWhite("[STARTUP] Creating AppDir at ");
wtc.WriteGreenLine(appDir);
}
catch (Exception ex)
{
//Unable to create the directory, throw fatal error and exit
logger.Error("Unable to create directory at " + appDir);
logger.Error("Failed with error: " + ex.Message);
wtc.WriteRedLine("Fatal Error: " + ex.Message);
Console.ReadKey();
Environment.Exit(6);
}
}
else
{
logger.Debug("AppDirectory exists at " + appDir);
wtc.WriteGreenLine("[STARTUP] Application Directory exists!");
}
wtc.WriteWhiteLine("[STARTUP] Checking for SourceDirectory");
logger.Debug("Checking for SourceDirectory at " + sourceDir);
if (!Directory.Exists(sourceDir))
{
logger.Debug("No Directory found at " + sourceDir);
//Try to create the appDir, Fail out and throw error if not
try
{
logger.Debug("Attempting to create directory at " + sourceDir);
Directory.CreateDirectory(sourceDir);
wtc.WriteWhite("[STARTUP] Creating SourceDir at ");
wtc.WriteGreenLine(sourceDir);
}
catch (Exception ex)
{
//Unable to create the directory, throw fatal error and exit
logger.Error("Unable to create directory at " + sourceDir);
logger.Error("Failed with error: " + ex.Message);
wtc.WriteRedLine("Fatal Error: " + ex.Message);
Console.ReadKey();
Environment.Exit(7);
}
}
else
{
logger.Debug("SourceDirectory exists at " + sourceDir);
wtc.WriteGreenLine("[STARTUP] Source Directory exists!");
}
wtc.WriteWhiteLine("[STARTUP] Checking for ToolDirectory");
logger.Debug("Checking for ToolDirectory at " + sourceDir);
if (!Directory.Exists(toolDir))
{
logger.Debug("No Directory found at " + toolDir);
//Try to create the appDir, Fail out and throw error if not
try
{
logger.Debug("Attempting to create directory at " + sourceDir);
Directory.CreateDirectory(toolDir);
wtc.WriteWhite("[STARTUP] Creating ToolDir at ");
wtc.WriteGreenLine(toolDir);
}
catch (Exception ex)
{
//Unable to create the directory, throw fatal error and exit
logger.Error("Unable to create directory at " + sourceDir);
logger.Error("Failed with error: " + ex.Message);
wtc.WriteRedLine("Fatal Error: " + ex.Message);
Console.ReadKey();
Environment.Exit(7);
}
}
else
{
logger.Debug("ToolDirectory exists at " + toolDir);
wtc.WriteGreenLine("[STARTUP] Tool Directory exists!");
}
//Check for Write/Read/Delete Permissions in directories;
logger.Debug("Checking for Write/Read/Delete Permissions in directories");
try
{
//APPDIR
logger.Debug("Creating file TEST in " + appDir);
File.WriteAllText(appDir + "\\test.test", "");
logger.Debug(appDir + "\\test.test" + " - File Created!");
logger.Debug("Deleting File " + appDir + "\\test.test");
File.Delete(appDir + "\\test.test");
logger.Debug(appDir + "\\test.test" + " - File Deleted!");
//SOURCEDIR
logger.Debug("Creating file TEST in " + sourceDir);
File.WriteAllText(sourceDir + "\\test.test", "");
logger.Debug(sourceDir + "\\test.test" + " - File Created!");
logger.Debug("Deleting File " + sourceDir + "\\test.test");
File.Delete(sourceDir + "\\test.test");
logger.Debug(sourceDir + "\\test.test" + " - File Deleted!");
//TOOLDIR
logger.Debug("Creating file TEST in " + toolDir);
File.WriteAllText(toolDir + "\\test.test", "");
logger.Debug(toolDir + "\\test.test" + " - File Created!");
logger.Debug("Deleting File " + toolDir + "\\test.test");
File.Delete(toolDir + "\\test.test");
logger.Debug(toolDir + "\\test.test" + " - File Deleted!");
}
catch (Exception ex)
{
logger.Fatal(ex.Message);
wtc.WriteRedLine("[FATAL ERROR] " + ex.Message);
Console.ReadKey();
Environment.Exit(8);
}
#endregion
#region ToolSupport
//This seciton needs to get improved as we add more tool support;
//Currently going to check for PSEXEC, will need to validate more as we use more.
//IDEA MIRROR SUPPORT;
//DOWNLOAD TOOLS NEEDED VIA A MIRROR AND VERIFY MD5
//DEFINE TOOLS IN INI?
if (!File.Exists(toolDir + "\\psexec.exe"))
{
//PSEXEC is missing;
logger.Warn("Unable to find psexec in the following location [" + toolDir + "\\psexec.exe]");
logger.Warn("Any applications that use PSEXEC will not function!");
bPsexecMissing = true;
wtc.WriteYellowLine("[STARTUP] PSEXEC is missing from the Tools directory. Please make sure the exe is in the given path, or change the \"ToolDir\" path in your ini to where PSEXEC exists");
wtc.WriteYellowLine("[WARNING] Program will continue, any application that uses PSEXEC as the install driver will not function till this is resolved");
}
}
catch (Exception ex)
{
wtc.WriteRedLine("[FATAL ERROR] " + ex.Message);
logger.Fatal(ex.Message);
Console.ReadKey();
Environment.Exit(2);
}
#endregion
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler ();
AppStoreParser parser = new AppStoreParser ();
// Loading Configuration
LogSetup.InitializeLog ("Apple_Store_Urls_Worker.log", "info");
_logger = LogManager.GetCurrentClassLogger ();
// Loading Config
_logger.Info ("Loading Configurations from App.config");
LoadConfiguration ();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists (fPath))
{
_logger.Fatal ("Couldnt find proxies on path : " + fPath);
System.Environment.Exit (-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines (fPath, Encoding.GetEncoding ("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load (fLines.ToList ());
}
catch (Exception ex)
{
_logger.Fatal (ex);
System.Environment.Exit (-101);
}
}
// AWS Queue Handler
_logger.Info ("Initializing Queues");
AWSSQSHelper appsUrlQueue = new AWSSQSHelper (_appUrlsQueueName , _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper appsDataQueue = new AWSSQSHelper (_appsDataQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.Info ("Started Processing Individual Apps Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!appsUrlQueue.DeQueueMessages ())
{
Thread.Sleep (_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!appsUrlQueue.AnyMessageReceived ())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32 (Math.Pow (2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine ("Fallback (seconds) => " + waitTime);
Thread.Sleep (waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var appUrl in appsUrlQueue.GetDequeuedMessages ())
{
bool processingWorked = true;
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
//appUrl.Body = "https://itunes.apple.com/us/app/action-run-3d/id632371832?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/emoji-2-free-new-emoticons/id521863802?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8";
//appUrl.Body = "https://itunes.apple.com/us/app/dba-den-bla-avis/id448605988?mt=8";
htmlResponse = httpClient.Get (appUrl.Body, shouldUseProxies);
if (String.IsNullOrEmpty (htmlResponse))
{
// Extending Fallback time
retries++;
int sleepTime = retries * _hiccupTime <= 30000 ? retries * _hiccupTime : 30000;
_logger.Info ("Retrying Request for App Page [ " + sleepTime / 1000 + " ]");
Thread.Sleep (sleepTime);
}
} while (String.IsNullOrWhiteSpace (htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace (htmlResponse))
{
continue;
}
// Feedback
_logger.Info ("Current page " + appUrl.Body, "Parsing App Data");
// Parsing Data out of the Html Page
AppleStoreAppModel parsedApp = parser.ParseAppPage (htmlResponse);
parsedApp.url = appUrl.Body;
// Enqueueing App Data
appsDataQueue.EnqueueMessage (parsedApp.ToJson ());
// Little Hiccup
Thread.Sleep (_hiccupTime);
}
catch (Exception ex)
{
_logger.Error (ex);
// Setting Flag to "False"
processingWorked = false;
}
finally
{
//Deleting the message - Only if the processing worked
if (processingWorked)
{
appsUrlQueue.DeleteMessage (appUrl);
}
}
}
}
catch (Exception ex)
{
_logger.Error (ex);
}
} while (true);
}
/// <summary>
/// Main application entry point.
/// </summary>
/// <param name="args">Input arguments.</param>
/// <returns>The application exit code.</returns>
public static int Main(string[] args)
{
string logPath = null, watcherThreshhold = null;
int enableLogging = 0, verbose = 0, help = 0;
var options = new OptionSet()
{
{ "d|directory=", "(required) the path to the application directory of the application to run jobs for.", v => directory = v },
{ "c|config=", "the path to the configuration file to use.", v => config = v },
{ "v|verbose", "write session output to the console.", v => { ++verbose; } },
{ "l|log", "write session output to a log file.", v => { ++enableLogging; } },
{ "lf|logfile=", "the path to the log file to write to.", v => logPath = v },
{ "p|persistence=", "the path to the running jobs persistence path to create/user.", v => persistencePath = v },
{ "t|threshold=", "the threshold, in milliseconds, to compress filesystem events into.", v => watcherThreshhold = v },
{ "h|help", "display usage help.", v => { ++help; } }
};
try
{
options.Parse(args);
directory = PathQuotesExp.Replace(directory ?? String.Empty, "$1");
config = PathQuotesExp.Replace(config ?? String.Empty, "$1");
logPath = PathQuotesExp.Replace(logPath ?? String.Empty, "$1");
persistencePath = PathQuotesExp.Replace(persistencePath ?? String.Empty, "$1");
}
catch (OptionException ex)
{
ShowHelp(options, ex);
return 1;
}
if (help > 0)
{
ShowHelp(options, null);
return 0;
}
if (enableLogging > 0 && String.IsNullOrEmpty(logPath))
{
logPath = Path.GetFullPath("collar.log");
}
LogManager.Configuration = CreateLoggingConfiguration(verbose > 0, enableLogging > 0, logPath);
logger = LogManager.GetLogger("BlueCollar");
if (String.IsNullOrEmpty(directory))
{
ShowHelp(options, new ArgumentException("You must specify the directory of an application to run jobs for."));
return 1;
}
else if (!Directory.Exists(directory))
{
string message = String.Format(CultureInfo.InvariantCulture, "The application directory '{0}' does not exist.", directory);
if (verbose > 0)
{
logger.Fatal(message);
}
else
{
Console.Error.WriteLine(message);
}
return 1;
}
if (!String.IsNullOrEmpty(config) && !File.Exists(config))
{
string message = String.Format(CultureInfo.InvariantCulture, "The configuration file '{0}' does not exist.", config);
if (verbose > 0)
{
logger.Fatal(message);
}
else
{
Console.Error.WriteLine(message);
}
return 1;
}
if (String.IsNullOrEmpty(persistencePath))
{
string above = Path.GetDirectoryName(directory);
if (!String.IsNullOrEmpty(above))
{
persistencePath = Path.GetFullPath(directory.Substring(above.Length + 1) + ".bin");
logger.Info(CultureInfo.InvariantCulture, "Using defaulted running jobs persistence file at '{0}'.", persistencePath);
}
}
if (!String.IsNullOrEmpty(watcherThreshhold))
{
string message = null;
try
{
fileSystemWatcherThreshold = Convert.ToInt32(watcherThreshhold, CultureInfo.InvariantCulture);
if (fileSystemWatcherThreshold < 500)
{
message = "The filesystem watcher threshold must be at least 500ms.";
}
}
catch (FormatException)
{
message = "The filesystem watcher threshold entered is not a valid number.";
}
if (!String.IsNullOrEmpty(message))
{
if (verbose > 0)
{
logger.Fatal(message);
}
else
{
Console.Error.WriteLine(message);
}
return 1;
}
}
else
{
fileSystemWatcherThreshold = 500;
}
autoReload = true;
exitHandle = new ManualResetEvent(false);
CreateAndPullUpBootstraps();
if (bootstraps != null && bootstraps.IsLoaded)
{
inputThread = new Thread(new ParameterizedThreadStart(WaitForInput));
inputThread.Start();
WaitHandle.WaitAll(new[] { exitHandle });
}
else
{
return 1;
}
return 0;
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler ();
AppStoreParser parser = new AppStoreParser ();
// Loading Configuration
LogSetup.InitializeLog ("Apple_Store_Categories_Worker.log", "info");
_logger = LogManager.GetCurrentClassLogger ();
// Loading Config
_logger.Info ("Loading Configurations from App.config");
LoadConfiguration ();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists (fPath))
{
_logger.Fatal ("Couldnt find proxies on path : " + fPath);
System.Environment.Exit (-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines (fPath, Encoding.GetEncoding ("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load (fLines.ToList ());
}
catch (Exception ex)
{
_logger.Fatal (ex);
System.Environment.Exit (-101);
}
}
// AWS Queue Handler
_logger.Info ("Initializing Queues");
AWSSQSHelper categoriesUrlQueue = new AWSSQSHelper (_categoriesQueueName , _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
AWSSQSHelper charactersUrlQueue = new AWSSQSHelper (_characterUrlsQueueName, _maxMessagesPerDequeue, _awsKey, _awsKeySecret);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.Info ("Started Processing Category Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!categoriesUrlQueue.DeQueueMessages())
{
Thread.Sleep (_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!categoriesUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32 (Math.Pow (2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
_logger.Info ("Fallback (seconds) => " + waitTime);
Thread.Sleep (waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var categoryUrl in categoriesUrlQueue.GetDequeuedMessages())
{
// Console Feedback
_logger.Info ("Started Parsing Category : " + categoryUrl.Body);
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
htmlResponse = httpClient.Get (categoryUrl.Body, shouldUseProxies);
if (String.IsNullOrEmpty (htmlResponse))
{
_logger.Error ("Retrying Request for Category Page");
retries++;
}
} while (String.IsNullOrWhiteSpace (htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace (htmlResponse))
{
// Deletes Message and moves on
categoriesUrlQueue.DeleteMessage (categoryUrl);
continue;
}
// If the request worked, parses the urls out of the page
foreach (string characterUrls in parser.ParseCharacterUrls (htmlResponse))
{
// Enqueueing Urls
charactersUrlQueue.EnqueueMessage (HttpUtility.HtmlDecode (characterUrls));
}
}
catch (Exception ex)
{
_logger.Error (ex);
}
finally
{
// Deleting the message
categoriesUrlQueue.DeleteMessage(categoryUrl);
}
}
}
catch (Exception ex)
{
_logger.Error (ex);
}
} while (true);
}
private static void ListenQueue(string queueName, IJobActivator jobActivator, Logger logger, ConcurrentHashSet<InWorkMessage> inworkMessages, CancellationToken cancellationToken)
{
if (!MessageQueue.Exists(queueName))
{
throw new InvalidOperationException(string.Format("Очередь {0} отсутствует.", queueName));
}
var mq = new MessageQueue(queueName);
while (true)
{
try
{
using (var messageQueueTransaction = new MessageQueueTransaction())
{
messageQueueTransaction.Begin();
var mes = mq.Receive(messageQueueTransaction);
if (mes == null)
{
logger.Error("Получено пустое сообщение");
continue;
}
if (cancellationToken.IsCancellationRequested)
{
logger.Info("Операция отменена");
return;
}
mes.Formatter = new XmlMessageFormatter(new[] {typeof(MessageWrapper)});
var inWorkMessage = new InWorkMessage { Job = (MessageWrapper)mes.Body, QueueName = queueName, Label = mes.Label};
if(!inworkMessages.TryAdd(inWorkMessage))
continue;
if (inWorkMessage.Job.RetryCount==0)
logger.Info("Запущена задача {0}", inWorkMessage.Label);
else
logger.Info("Запущена задача {0}. Повторная попытка {1}", inWorkMessage.Label, inWorkMessage.Job.RetryCount);
var serializedjob = JobHelper.FromJson<SerializedJob>(inWorkMessage.Job.SerializedJob);
var job = serializedjob.Deserialize();
//Отправляем задачу в работу и добавляем обработчик который в случае ошибки или отмены задачи вернет сообщение в очередь.
//Если задача завершилась успешно, проставляем флаг об этом.
Task.Factory.StartNew(() => job.Perform(jobActivator, cancellationToken), cancellationToken)
.ContinueWith(t =>
{
if (t.Exception != null)
{
t.Exception.Handle(ex =>
{
if (ex.GetType() == typeof (JobFailedException))
{
logger.Info("При выполнении задачи {0} возникла ошибка.", inWorkMessage.Label);
logger.Error(ex.GetAllInnerExceptionMessagesAndTrace());
Thread.Sleep(60000);
ReturnMessageToQueue(inWorkMessage, inworkMessages, cancellationToken);
return true;
}
logger.Fatal(ex.GetAllInnerExceptionMessagesAndTrace());
ReturnMessageToQueue(inWorkMessage, inworkMessages, cancellationToken);
return false;
});
}
else
{
logger.Info("Задача {0} завершилась успешно.", inWorkMessage.Label);
inWorkMessage.CompleteMessage();
}
}, TaskContinuationOptions.NotOnCanceled);
messageQueueTransaction.Commit();
}
}
catch (Exception e)
{
logger.Fatal(e.GetAllInnerExceptionMessagesAndTrace());
//inworkMessages.CompleteAdding();
throw;
}
}
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler ();
AppStoreParser parser = new AppStoreParser ();
// Loading Configuration
LogSetup.InitializeLog ("Apple_Store_Urls_Worker.log", "info");
_logger = LogManager.GetCurrentClassLogger ();
// Loading Config
_logger.Info ("Loading Configurations from App.config");
LoadConfiguration ();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists (fPath))
{
_logger.Fatal ("Couldnt find proxies on path : " + fPath);
System.Environment.Exit (-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines (fPath, Encoding.GetEncoding ("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load (fLines.ToList ());
}
catch (Exception ex)
{
_logger.Fatal (ex);
System.Environment.Exit (-101);
}
}
// AWS Queue Handler
_logger.Info ("Initializing Queues");
AzureSQSHelper charactersUrlQueue = new AzureSQSHelper (_characterUrlsQueueName, _maxMessagesPerDequeue, _azureQueueconn);
AzureSQSHelper numericUrlQueue = new AzureSQSHelper(_numericUrlsQueueName , _maxMessagesPerDequeue, _azureQueueconn);
// Setting Error Flag to No Error ( 0 )
System.Environment.ExitCode = 0;
// Initialiazing Control Variables
int fallbackWaitTime = 1;
_logger.Info ("Started Processing Character Urls");
do
{
try
{
// Dequeueing messages from the Queue
if (!charactersUrlQueue.DeQueueMessages())
{
Thread.Sleep (_hiccupTime); // Hiccup
continue;
}
// Checking for no message received, and false positives situations
if (!charactersUrlQueue.AnyMessageReceived())
{
// If no message was found, increases the wait time
int waitTime;
if (fallbackWaitTime <= 12)
{
// Exponential increase on the wait time, truncated after 12 retries
waitTime = Convert.ToInt32 (Math.Pow (2, fallbackWaitTime) * 1000);
}
else // Reseting Wait after 12 fallbacks
{
waitTime = 2000;
fallbackWaitTime = 0;
}
fallbackWaitTime++;
// Sleeping before next try
Console.WriteLine ("Fallback (seconds) => " + waitTime);
Thread.Sleep (waitTime);
continue;
}
// Reseting fallback time
fallbackWaitTime = 1;
// Iterating over dequeued Messages
foreach (var characterUrl in charactersUrlQueue.GetDequeuedMessages ())
{
// Console Feedback
_logger.Info ("Started Parsing Url : " + characterUrl.AsString);
try
{
// Retries Counter
int retries = 0;
string htmlResponse;
// Retrying if necessary
do
{
// Executing Http Request for the Category Url
htmlResponse = httpClient.Get (characterUrl.AsString, shouldUseProxies);
if (String.IsNullOrEmpty (htmlResponse))
{
_logger.Info ("Retrying Request for Character Page");
retries++;
// Small Hiccup
Thread.Sleep (_hiccupTime);
}
} while (String.IsNullOrWhiteSpace (htmlResponse) && retries <= _maxRetries);
// Checking if retries failed
if (String.IsNullOrWhiteSpace (htmlResponse))
{
// Deletes Message and moves on
charactersUrlQueue.DeleteMessage (characterUrl);
continue;
}
// Hashset of urls processed (to avoid duplicates)
HashSet<String> urlsQueued = new HashSet<String> ();
// Executing Request and Queueing Urls until there's no other Url to be queued
do
{
// Flag to check whether any url was added after the last iteration (avoids endless loop)
bool anyNewUrl = false;
// If the request worked, parses the Urls out of the page
foreach (string numericUrls in parser.ParseNumericUrls (htmlResponse).Select (t => HttpUtility.HtmlDecode (t)))
{
// Checking if this url was previously queued
if (!urlsQueued.Contains (numericUrls))
{
// Enqueueing Urls
numericUrlQueue.EnqueueMessage (HttpUtility.HtmlDecode (numericUrls));
// Adding url to the local hashset
urlsQueued.Add (numericUrls);
anyNewUrl = true;
}
}
// Checking for the need to perform another http request for the next page
if (parser.IsLastPage (htmlResponse) || !anyNewUrl)
{
break; // Breaks "While" Loop
}
// Feedback
_logger.Info ("Urls Queued For This Page : " + urlsQueued.Count, "\n\tProcessing Feedback");
// If it got to this point, it means that there are more pages to be processed
// Parsing URL of the "Last" page (the last that's visible)
string lastPageUrl = HttpUtility.HtmlDecode (parser.ParseLastPageUrl (htmlResponse));
// Executing Http Request for this Url (with retries)
retries = 0;
do
{
// HTTP Get for the Page
htmlResponse = httpClient.Get (lastPageUrl, shouldUseProxies);
if (String.IsNullOrEmpty (htmlResponse))
{
_logger.Error ("Retrying Request for Last Page");
retries++;
// Small Hiccup
Thread.Sleep (_hiccupTime);
}
} while (String.IsNullOrEmpty (htmlResponse) && retries <= _maxRetries);
} while (true);
}
catch (Exception ex)
{
_logger.Error (ex);
}
finally
{
charactersUrlQueue.DeleteMessage (characterUrl);
}
}
}
catch (Exception ex)
{
_logger.Error (ex);
}
} while (true);
}
static void Main(string[] args)
{
// Creating Needed Instances
RequestsHandler httpClient = new RequestsHandler ();
AppStoreParser parser = new AppStoreParser ();
// Setting Up Log
LogSetup.InitializeLog ("Apple_Store_Crawler.log", "info");
_logger = LogManager.GetCurrentClassLogger ();
// Starting Flow
_logger.Info ("Worker Started");
// Loading Configuration
_logger.Info ("Reading Configuration");
LoadConfiguration ();
// Control Variable (Bool - Should the process use proxies? )
bool shouldUseProxies = false;
// Checking for the need to use proxies
if (args != null && args.Length == 1)
{
// Setting flag to true
shouldUseProxies = true;
// Loading proxies from .txt received as argument
String fPath = args[0];
// Sanity Check
if (!File.Exists (fPath))
{
_logger.Fatal ("Couldnt find proxies on path : " + fPath);
System.Environment.Exit (-100);
}
// Reading Proxies from File
string[] fLines = File.ReadAllLines (fPath, Encoding.GetEncoding ("UTF-8"));
try
{
// Actual Load of Proxies
ProxiesLoader.Load (fLines.ToList ());
}
catch (Exception ex)
{
_logger.Fatal (ex);
System.Environment.Exit (-101);
}
}
// AWS Queue Handler
_logger.Info ("Initializing Queues");
AzureSQSHelper sqsWrapper = new AzureSQSHelper (_categoriesQueueName, 10, _azureQueueconn);
// Step 1 - Trying to obtain the root page html (source of all the apps)
var rootPageResponse = httpClient.GetRootPage (shouldUseProxies);
// Sanity Check
if (String.IsNullOrWhiteSpace (rootPageResponse))
{
_logger.Info ("Error obtaining Root Page HTMl - Aborting", "Timeout Error");
return;
}
// Step 2 - Extracting Category Urls from the Root Page and queueing their Urls
foreach (var categoryUrl in parser.ParseCategoryUrls (rootPageResponse))
{
// Logging Feedback
_logger.Info ("Queueing Category : " + categoryUrl);
// Queueing Category Urls
sqsWrapper.EnqueueMessage (categoryUrl);
}
_logger.Info ("End of Bootstrapping phase");
//Console.ReadLine();
}
