public static FlexibleOptions Initialize (string[] args, bool thrownOnError, InitializationOptions options = null) { InitOptions = options; DefaultProgramInitialization (); ProgramOptions = CheckCommandLineParams (args, thrownOnError); if (ProgramOptions.Get<bool> ("help", false) || ProgramOptions.Get<bool> ("h", false)) { show_help (""); CloseApplication (0, true); } // display program initialization header if (!Console.IsOutputRedirected) { ConsoleUtils.DisplayHeader ( typeof(ConsoleUtils).Namespace.Replace (".SimpleHelpers", ""), "options: " + (ProgramOptions == null ? "none" : "\n# " + String.Join ("\n# ", ProgramOptions.Options.Select (i => i.Key + "=" + i.Value)))); } else { var logger = GetLogger (); if (logger.IsDebugEnabled) { logger.Debug ("options: " + (ProgramOptions == null ? "none" : "\n# " + String.Join ("\n# ", ProgramOptions.Options.Select (i => i.Key + "=" + i.Value)))); } } return ProgramOptions; }
/// <summary> /// Main program entry point. /// </summary> static void Main(string[] args) { // set error exit code System.Environment.ExitCode = -50; try { // load configurations ProgramOptions = ConsoleUtils.Initialize(args, true); // start execution Execute(ProgramOptions); // check before ending for waitForKeyBeforeExit option if (ProgramOptions.Get("waitForKeyBeforeExit", false)) { ConsoleUtils.WaitForAnyKey(); } } catch (Exception ex) { LogManager.GetCurrentClassLogger().Fatal(ex); // check before ending for waitForKeyBeforeExit option if (ProgramOptions.Get("waitForKeyBeforeExit", false)) { ConsoleUtils.WaitForAnyKey(); } ConsoleUtils.CloseApplication(-60, true); } // set success exit code ConsoleUtils.CloseApplication(0, false); }
private static List <string> ParseArgumentAsList(FlexibleOptions options, string key) { // first check if we have a json array var list = options.Get <string[]> (key, null); // fallback to a csv string if (list == null) { list = (options.Get(key, "") ?? "").Split(',', ';'); } return(list.Select(i => i.Trim()).Where(i => !String.IsNullOrEmpty(i)).ToList()); }
public void Merge_WithComplexPriority_ShouldWork() { var opt1 = new FlexibleOptions(); opt1.Set("k1", "v1"); opt1.Set("k2", "v2"); opt1.Set("k3", "v3"); opt1.SetAlias("k3", "3", "key3"); var opt2 = new FlexibleOptions(); opt2.Set("k2", "v2.2"); var opt3 = new FlexibleOptions(); opt3.Set("k3", "v3.3"); // opt1 has lower priority var merge = FlexibleOptions.Merge(opt1, opt2, opt3); Assert.NotNull(merge); Assert.Equal(opt1.Get("k1"), merge.Get("k1")); Assert.Equal(opt2.Get("k2"), merge.Get("k2")); Assert.Equal(opt3.Get("k3"), merge.Get("k3")); // alias Assert.Equal(opt3.Get("k3"), merge.Get("key3")); Assert.Equal(opt3.Get("k3"), merge.Get("3")); // opt1 has higher priority merge = FlexibleOptions.Merge(opt2, opt3, opt1); Assert.NotNull(merge); Assert.Equal(opt1.Get("k1"), merge.Get("k1")); Assert.NotEqual(opt2.Get("k2"), merge.Get("k2")); Assert.NotEqual(opt3.Get("k3"), merge.Get("k3")); Assert.Equal(opt1.Get("k2"), merge.Get("k2")); Assert.Equal(opt1.Get("k3"), merge.Get("k3")); // alias Assert.Equal(opt1.Get("k3"), merge.Get("key3")); Assert.Equal(opt1.Get("k3"), merge.Get("3")); // opt1 has higher priority but with null options merge = FlexibleOptions.Merge(null, opt2, null, opt3, opt1, null); Assert.NotNull(merge); Assert.Equal(opt1.Get("k1"), merge.Get("k1")); Assert.NotEqual(opt2.Get("k2"), merge.Get("k2")); Assert.NotEqual(opt3.Get("k3"), merge.Get("k3")); Assert.Equal(opt1.Get("k2"), merge.Get("k2")); Assert.Equal(opt1.Get("k3"), merge.Get("k3")); // alias Assert.Equal(opt1.Get("k3"), merge.Get("key3")); Assert.Equal(opt1.Get("k3"), merge.Get("3")); }
public void SetAlias_AllowCaseInsensitiveGet_ShouldWork() { var expected = "v3"; var opt1 = new FlexibleOptions(); opt1.Set("k3", expected); opt1.SetAlias("k3", "3", "key3"); Assert.Equal(expected, opt1.Get("K3")); Assert.Equal(expected, opt1.Get("KEY3")); }
public void SetAlias_AllowGet_ShouldWork() { var expected = "v3"; var opt1 = new FlexibleOptions (); opt1.Set ("k3", expected); opt1.SetAlias ("k3", "3", "key3"); Assert.Equal (expected, opt1.Get ("k3")); Assert.Equal (expected, opt1.Get ("key3")); Assert.Equal (expected, opt1.Get ("3")); Assert.True (String.IsNullOrEmpty (opt1.Get ("KEY"))); }
public void SetAlias_AllowGet_ShouldWork() { var expected = "v3"; var opt1 = new FlexibleOptions(); opt1.Set("k3", expected); opt1.SetAlias("k3", "3", "key3"); Assert.Equal(expected, opt1.Get("k3")); Assert.Equal(expected, opt1.Get("key3")); Assert.Equal(expected, opt1.Get("3")); Assert.True(String.IsNullOrEmpty(opt1.Get("KEY"))); }
public void Get_CaseInsensitive_ShouldWork () { var expected = "v1"; var opt1 = new FlexibleOptions (); opt1.Set ("key1", expected); Assert.Equal (expected, opt1.Get ("KEY1")); }
public void Get_CaseInsensitive_ShouldWork() { var expected = "v1"; var opt1 = new FlexibleOptions(); opt1.Set("key1", expected); Assert.Equal(expected, opt1.Get("KEY1")); }
public void Get_AsDateTime_ShouldWork() { // datetime without milliseconds var expected = new DateTime(2015, 01, 02, 03, 04, 05); var opt1 = new FlexibleOptions(); opt1.Set("dtAsDateTime", expected); Assert.Equal(expected, opt1.Get <DateTime> ("dtAsDateTime", DateTime.MinValue)); opt1.Set("dtAsString", expected.ToString(System.Globalization.CultureInfo.InvariantCulture)); Assert.Equal(expected, opt1.Get <DateTime> ("dtAsString", DateTime.MinValue)); opt1.Set("dtAsISOString", expected.ToString("o")); Assert.Equal(expected, opt1.Get <DateTime> ("dtAsISOString", DateTime.MinValue)); opt1.Set("dtAsDate", expected.ToString("yyyyMMdd")); Assert.Equal(expected.Date, opt1.Get <DateTime> ("dtAsDate", DateTime.MinValue)); }
public void Get_AsDateTime_ShouldWork () { // datetime without milliseconds var expected = new DateTime (2015, 01, 02, 03, 04, 05); var opt1 = new FlexibleOptions (); opt1.Set ("dtAsDateTime", expected); Assert.Equal (expected, opt1.Get<DateTime> ("dtAsDateTime", DateTime.MinValue)); opt1.Set ("dtAsString", expected.ToString (System.Globalization.CultureInfo.InvariantCulture)); Assert.Equal (expected, opt1.Get<DateTime> ("dtAsString", DateTime.MinValue)); opt1.Set ("dtAsISOString", expected.ToString ("o")); Assert.Equal (expected, opt1.Get<DateTime> ("dtAsISOString", DateTime.MinValue)); opt1.Set ("dtAsDate", expected.ToString ("yyyyMMdd")); Assert.Equal (expected.Date, opt1.Get<DateTime> ("dtAsDate", DateTime.MinValue)); }
/// <summary> /// Main program entry point. /// </summary> static void Main(string[] args) { // set error exit code System.Environment.ExitCode = -50; try { // load configurations ProgramOptions = ConsoleUtils.Initialize(args, true); CancellationTokenSource cts = new CancellationTokenSource(); System.Console.CancelKeyPress += (s, e) => { logger.Debug("User requested exit (Ctrl + C), exiting..."); e.Cancel = true; cts.Cancel(); }; // start execution ExecuteAsync(ProgramOptions, cts.Token).Wait(); // check before ending for waitForKeyBeforeExit option if (ProgramOptions.Get("waitForKeyBeforeExit", false)) { ConsoleUtils.WaitForAnyKey(); } } catch (Exception ex) { LogManager.GetCurrentClassLogger().Fatal(ex); // check before ending for waitForKeyBeforeExit option if (ProgramOptions != null && ProgramOptions.Get("waitForKeyBeforeExit", false)) { ConsoleUtils.WaitForAnyKey(); } ConsoleUtils.CloseApplication(-60, true); } // set success exit code ConsoleUtils.CloseApplication(0, false); }
/// <summary> /// Main program entry point. /// </summary> static void Main (string[] args) { // set error exit code System.Environment.ExitCode = -50; try { // load configurations ProgramOptions = ConsoleUtils.Initialize (args, true); CancellationTokenSource cts = new CancellationTokenSource (); System.Console.CancelKeyPress += (s, e) => { logger.Debug ("User requested exit (Ctrl + C), exiting..."); e.Cancel = true; cts.Cancel (); }; // start execution ExecuteAsync (ProgramOptions, cts.Token).Wait (); // check before ending for waitForKeyBeforeExit option if (ProgramOptions.Get ("waitForKeyBeforeExit", false)) ConsoleUtils.WaitForAnyKey (); } catch (Exception ex) { LogManager.GetCurrentClassLogger ().Fatal (ex); // check before ending for waitForKeyBeforeExit option if (ProgramOptions != null && ProgramOptions.Get ("waitForKeyBeforeExit", false)) ConsoleUtils.WaitForAnyKey (); ConsoleUtils.CloseApplication (-60, true); } // set success exit code ConsoleUtils.CloseApplication (0, false); }
private static void Execute(FlexibleOptions options) { logger.Debug("Start"); // Parsing Arguments - Sanity Check ParseArguments(options); logger.Debug("Opening connections..."); CheckConnections(options); // Reaching Databases MongoServer sourceDatabase = MongoDbContext.GetServer(options.Get("source")); MongoServer targetDatabase = MongoDbContext.GetServer(options.Get("target")); // process list logger.Debug("Start migrating data..."); Migrator.DatabaseCopy(sourceDatabase, targetDatabase, _sourceDatabases, _targetDatabases, _collections, _insertBatchSize, _copyIndexes, _dropCollections, _threads, options); System.Threading.Thread.Sleep(1000); logger.Debug("Done migrating data!"); }
public void Merge_WithComplexPriority_ShouldWork () { var opt1 = new FlexibleOptions (); opt1.Set ("k1", "v1"); opt1.Set ("k2", "v2"); opt1.Set ("k3", "v3"); opt1.SetAlias ("k3", "3", "key3"); var opt2 = new FlexibleOptions (); opt2.Set ("k2", "v2.2"); var opt3 = new FlexibleOptions (); opt3.Set ("k3", "v3.3"); // opt1 has lower priority var merge = FlexibleOptions.Merge (opt1, opt2, opt3); Assert.NotNull (merge); Assert.Equal (opt1.Get ("k1"), merge.Get ("k1")); Assert.Equal (opt2.Get ("k2"), merge.Get ("k2")); Assert.Equal (opt3.Get ("k3"), merge.Get ("k3")); // alias Assert.Equal (opt3.Get ("k3"), merge.Get ("key3")); Assert.Equal (opt3.Get ("k3"), merge.Get ("3")); // opt1 has higher priority merge = FlexibleOptions.Merge (opt2, opt3, opt1); Assert.NotNull (merge); Assert.Equal (opt1.Get ("k1"), merge.Get ("k1")); Assert.NotEqual (opt2.Get ("k2"), merge.Get ("k2")); Assert.NotEqual (opt3.Get ("k3"), merge.Get ("k3")); Assert.Equal (opt1.Get ("k2"), merge.Get ("k2")); Assert.Equal (opt1.Get ("k3"), merge.Get ("k3")); // alias Assert.Equal (opt1.Get ("k3"), merge.Get ("key3")); Assert.Equal (opt1.Get ("k3"), merge.Get ("3")); // opt1 has higher priority but with null options merge = FlexibleOptions.Merge (null, opt2, null, opt3, opt1, null); Assert.NotNull (merge); Assert.Equal (opt1.Get ("k1"), merge.Get ("k1")); Assert.NotEqual (opt2.Get ("k2"), merge.Get ("k2")); Assert.NotEqual (opt3.Get ("k3"), merge.Get ("k3")); Assert.Equal (opt1.Get ("k2"), merge.Get ("k2")); Assert.Equal (opt1.Get ("k3"), merge.Get ("k3")); // alias Assert.Equal (opt1.Get ("k3"), merge.Get ("key3")); Assert.Equal (opt1.Get ("k3"), merge.Get ("3")); }
private static void Execute(FlexibleOptions options) { logger.Info ("Start"); // Sanity Checks string inputPath = options.Get("inputFile"); string outputPath = options.Get("outputFile"); if (String.IsNullOrEmpty(inputPath)) { logger.Fatal ("Parameter {--inputfile} is mandatory"); System.Environment.Exit (-101); } if (String.IsNullOrEmpty (inputPath)) { logger.Fatal ("Parameter {--outputFile} is mandatory"); System.Environment.Exit (-102); } if (!File.Exists(inputPath)) { logger.Fatal ("Parameter {--inputfile} should point to an existing file"); System.Environment.Exit (-103); } // Control Variables int processed = 0, errors = 0; // Processing Urls using(StreamReader fReader = new StreamReader (inputPath)) { using (StreamWriter fWriter = new StreamWriter (outputPath)) { fWriter.AutoFlush = true; string fLine; using (WebRequests client = new WebRequests ()) { // Configuring HTTP Client client.Referer = "http://sortfolio.com/"; client.Host = "sortfolio.com"; client.Encoding = "gzip, deflate, sdch"; client.Headers.Add ("Upgrade-Insecure-Requests", "1"); while ((fLine = fReader.ReadLine ()) != null) { // Processing Only "Listings" if (fLine.IndexOf ("/listings/") >= 0) { try { // Reaching Listing Page string url = fLine.Split (';')[0]; string htmlReponse = client.Get (url); // Control Variable Inc. processed++; // Sanity Check if (String.IsNullOrEmpty (htmlReponse)) { logger.Error ("Failed to read response for {0}", url); errors++; continue; } // Parsing Data HtmlDocument map = new HtmlDocument (); map.LoadHtml (htmlReponse); var tmpNode = map.DocumentNode.SelectSingleNode ("//script[contains(@id,'mail_to')]"); if (tmpNode == null) { continue; } string email = tmpNode.InnerText; // Extracting actual email data out of decoded string email = email.Replace ("eval(decodeURIComponent('", String.Empty).Replace ("'", String.Empty).Replace (")", String.Empty); email = System.Web.HttpUtility.UrlDecode (email); Regex emailRegex = new Regex (@"Node\('(.+?)'\)\);"); email = emailRegex.Match (email).Groups[1].Value; tmpNode = map.DocumentNode.SelectSingleNode ("//*[@id='content']/div/div[2]/p/strong"); string budget = tmpNode == null ? String.Empty : tmpNode.InnerText; tmpNode = map.DocumentNode.SelectSingleNode ("//span[@itemprop='addressLocality']"); string headQuarter = tmpNode == null ? String.Empty : tmpNode.InnerText; tmpNode = map.DocumentNode.SelectSingleNode ("//a[@class='website']"); string website = tmpNode == null ? String.Empty : tmpNode.InnerText; // Writing to output file fWriter.WriteLine ("{0};{1};{2};{3};{4}", url, headQuarter, website, email, budget); logger.Info ("Processed: {0}", url); if (processed % 100 == 0) { logger.Info ("\tProgress:{0} - {1}", processed, errors); } } catch(Exception ex) { logger.Error (ex); errors++; } } } } } } logger.Info ("End"); Console.ReadLine (); }
private static void Execute(FlexibleOptions options) { Logger.Info("Start"); // Args Sanity Check if (options.Options == null || options.Options.Count == 0) { Console.WriteLine("No arguments received."); System.Environment.Exit(-101); } // Prompts for user Input Console.WriteLine("Is the configuration correct ? Y/N"); var key = Console.ReadKey().Key; // Checking Key if (key == ConsoleKey.N) // N = "NO" { Console.WriteLine(" => 'NO' : Aborting"); System.Environment.Exit(-102); } else if (key != ConsoleKey.Y) // Anything other than "N" and "Y" is an error. { Console.WriteLine(" => 'Wrong Key Pressed' : Expected either 'Y' or 'N'"); System.Environment.Exit(-102); } Console.WriteLine(" => Proceeding with Export."); // Sanity Check of Config and Arguments if (!ValidateConfig(options)) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Missing MongoDB Configuration Parameter. (Server, Database, Collection and Credentials are Mandatory)"); Console.ForegroundColor = ConsoleColor.White; System.Environment.Exit(-103); } // Creating instance of MongoDB String sourceConnString = MongoDbContext.BuildConnectionString(options["sourceUsername"], options["sourcePassword"], options["sourceServer"], options["authDatabaseName"]); // Reaching Databases MongoDatabase sourceDatabase = MongoDbContext.GetServer(sourceConnString).GetDatabase(options["sourceDatabaseName"]); // Assembling "Query" to MongoDB, if any query text was provided QueryDocument query = String.IsNullOrWhiteSpace(options["mongoQuery"]) ? null : new QueryDocument(QueryDocument.Parse(options["mongoQuery"])); // Checking if the provided Collection Exists if (!sourceDatabase.CollectionExists(options["collection"])) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Collection [ " + options["collection"] + " ] does not exists on the specified database"); Console.ForegroundColor = ConsoleColor.White; System.Environment.Exit(-104); } if (options["format"].ToUpper() == "CSV") { // Loading Export Configuration from XML File if (!JsonToCSV.LoadExportLayout(options["layoutFile"])) { // Error Checking Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Error Loading Export Layout"); Console.WriteLine("Message : " + JsonToCSV.errorMessage); Console.ForegroundColor = ConsoleColor.White; System.Environment.Exit(-105); } } // Setting up MongoDB Cursor MongoCursor cursor = sourceDatabase.GetCollection <BsonDocument> (options["collection"]).Find(query); cursor.SetFlags(QueryFlags.NoCursorTimeout); // Checking for the need to apply limit int _limit = options.Get <int>("limit", -1); if (_limit != -1) { cursor.SetLimit(_limit); } // Counters int recordsProcessed = 0; // JSON Settings to keep the "JSON" output as "Strict" var jsonSettings = new JsonWriterSettings() { OutputMode = JsonOutputMode.Strict }; // File Writer using (StreamWriter fWriter = new StreamWriter(options["outputFile"], false, Encoding.UTF8)) { // Auto Flush fWriter.AutoFlush = true; // Output File Line string fileLine = String.Empty; // Should we add headers to the output CSV file? if (options["format"].ToUpper() == "CSV" && options.Get <bool>("addHeader", false)) { // Writing Headers fWriter.WriteLine(JsonToCSV.Fields); } // Iterating over documents found using the query foreach (BsonDocument document in cursor) { // Picking which export method will be used if (options["format"].ToUpper() == "CSV") { // Extracting data from it fileLine = JsonToCSV.BsonToCSV(document); } else { fileLine = document.ToJson(jsonSettings); } // Checking for errors if (String.IsNullOrWhiteSpace(fileLine)) { continue; } // Writing to output csv fWriter.WriteLine(fileLine.Replace(System.Environment.NewLine, "<br>")); // Counting if (recordsProcessed++ % 100 == 0) { Console.WriteLine("Processed : " + recordsProcessed); } } } Logger.Info("End"); }
public void SetAlias_AllowCaseInsensitiveGet_ShouldWork () { var expected = "v3"; var opt1 = new FlexibleOptions (); opt1.Set ("k3", expected); opt1.SetAlias ("k3", "3", "key3"); Assert.Equal (expected, opt1.Get ("K3")); Assert.Equal (expected, opt1.Get ("KEY3")); }
/// <summary> /// Parses out the Arguments received from the "CLI" /// </summary> /// <param name="args">Array of arguments received from the "CLI"</param> private static void ParseArguments(FlexibleOptions options) { // parse arguments _sourceUri = options["source"]; _sourceServer = options["source-server"]; _sourceUsername = options["source-username"]; _sourcePassword = options["source-password"]; _sourceAuthDatabase = options.Get("source-auth-database", options["auth-database-name-source"]); _targetUri = options["target"]; _targetServer = options["target-server"]; _targetUsername = options["target-username"]; _targetPassword = options["target-password"]; _targetAuthDatabase = options.Get("target-auth-database", options["auth-database-name-target"]); _insertBatchSize = options.Get("batch-size", options.Get("insert-batch-size", -1)); _threads = options.Get("threads", 1); _copyIndexes = options.Get("copy-indexes", false); _dropCollections = options.Get("drop-collections", false); _skipExisting = options.Get("skip-existing", false); _skipCount = options.Get("skip-count", false); _eraseObjectId = options.Get("refresh-object-id", false); // check parameter databases _sourceDatabases = ParseArgumentAsList(options, "databases").Concat(ParseArgumentAsList(options, "source-database")).Distinct().ToList(); _targetDatabases = ParseArgumentAsList(options, "target-database"); // check collections parameter _collections = ParseArgumentAsList(options, "collections"); _targetCollection = options.Get("target-collection"); // Sanity Checks if (String.IsNullOrEmpty(_sourceUri) && String.IsNullOrEmpty(_sourceServer)) { logger.Error("No source mongodb server connection information provided: use the argument 'source' to provide a mongodb uri with the connection information"); ConsoleUtils.CloseApplication(-101, true); } if (String.IsNullOrEmpty(_targetUri) && String.IsNullOrEmpty(_targetServer)) { logger.Error("No source mongodb server connection information provided: use the argument 'target' to provide a mongodb uri with the connection information"); ConsoleUtils.CloseApplication(-102, true); } if (_sourceDatabases.Count == 0) { logger.Error("No database selected: use the argument 'databases' to provide a list of databases"); ConsoleUtils.CloseApplication(-103, true); } // if no target database is provided, lets use the sourcedatabases if (_targetDatabases.Count == 0) { _targetDatabases = null; } else { // if we have target database names: // 1. sourceDatabase cannot contain wildcard if (_sourceDatabases.Any(i => SharedMethods.HasWildcard(i))) { logger.Error("Wildcard cannot be used in source database names if a list of target databases is provided!"); ConsoleUtils.CloseApplication(-104, true); } // 2. sourceDatabase cannot contain wildcard if (_sourceDatabases.Any(i => SharedMethods.HasWildcard(i))) { logger.Error("Wildcard cannot be used in target database names!"); ConsoleUtils.CloseApplication(-105, true); } // 3. check for database mapping discrepancy if (_sourceDatabases.Count != _targetDatabases.Count) { logger.Error("Different number of source and target databases detected: use the argument 'databases' and 'targetDatabases' to provide a list of databases"); ConsoleUtils.CloseApplication(-106, true); } if (_sourceDatabases.Any(i => i.IndexOf('=') > 0)) { logger.Error("Invalid use of target database parameter: if the argument 'databases' has a key=value format to indicate the target database name, the argument 'targetDatabases' cannot be used."); ConsoleUtils.CloseApplication(-107, true); } } // Collections Check // If we received a LIST of source collections but also received a TARGET collection, we should halt. // Only a collection-by-collection copy should be make when picking a target collection if (_collections != null && _collections.Count > 1 && !String.IsNullOrWhiteSpace(_targetCollection)) { logger.Error("Arguments 'collections' and 'targetCollection' are exclusive. When providing a 'targetCollection' you can only copy a SINGLE collection to it at a time."); } }
/// <summary> /// Does all the heavy lifting of Dequeueing messages and queueing it to be /// recorded on mongodb. /// </summary> /// <param name="threadContext"></param> public void ThreadPoolCallback(Object threadContext) { // Thread ID _id = Convert.ToString((int)threadContext); // Control Variables int logThreshold = _programOptions.Get <int>("LogThreshold", 10000); int messagesPerDequeue = _programOptions.Get <int>("MessagesPerDequeue", 10); int movedMessages = 0; _logger.Info("Started Worker : " + _id); AWSSQSHelper sourceQueueHandler = new AWSSQSHelper(_programOptions["SourceQueue"], messagesPerDequeue, _programOptions["SourceAWSAccessKey"], _programOptions["SourceAWSSecretKey"]); AWSSQSHelper targetQueueHandler = new AWSSQSHelper(_programOptions["TargetQueue"], messagesPerDequeue, _programOptions["TargetAWSAccessKey"], _programOptions["TargetAWSSecretKey"]); // Local Buffer of SQS Messages List <Message> sqsBuffer = new List <Message> (); // Iterating over Dequeued Messages foreach (var sqsMessage in sourceQueueHandler.GetMessages()) { // Feeding local buffer of messages to be deleted sqsBuffer.Add(sqsMessage); // Incrementing counter of moved messages movedMessages++; // Enqueueing and Deleting messages every 1000 if (sqsBuffer.Count == 1000) { _logger.Info(String.Format("Enqueueing Messages to target - Worker {0}", _id)); targetQueueHandler.EnqueueMessages(sqsBuffer.Select(t => t.Body).ToList()); _logger.Info(String.Format("Deleting Messages from source - Worker {0}", _id)); sourceQueueHandler.DeleteMessages(sqsBuffer); sqsBuffer.Clear(); } // Checking for the need to log progress if (movedMessages % logThreshold == 0) { _logger.Info(String.Format("Moved {0} messages - Thread {1}", movedMessages, _id)); } } // Enqueuing and Removing Remainder of buffer _logger.Info("Enqueueing remainder of buffered messages to target SQS Queue"); targetQueueHandler.EnqueueMessages(sqsBuffer.Select(t => t.Body).ToList()); _logger.Info("Deleting remainder of buffered messages from source SQS Queue"); sourceQueueHandler.DeleteMessages(sqsBuffer); sqsBuffer.Clear(); // Finished Processing _logger.Info(String.Format("Finished Moving SQS Messages. Halting {0}", _id)); // Setting up "Done Event" to signal Thread Pool _doneEvent.Set(); }
private static void CreateCollection(MongoCollection <BsonDocument> sourceCollection, MongoCollection <BsonDocument> targetCollection, FlexibleOptions options) { if (targetCollection.Exists()) { return; } List <string> config = new List <string> (); if (!String.IsNullOrWhiteSpace(options.Get("collection-wt-configString"))) { config.AddRange(options.Get("collection-wt-configString", "").Split(',').Select(i => i.Trim()).Where(i => !String.IsNullOrEmpty(i))); } if (options.HasOption("collection-wt-block-compressor") && valid_wt_compressors.Contains(options.Get("collection-wt-block-compressor", "invalid"))) { config.RemoveAll(i => i.StartsWith("block_compressor=", StringComparison.OrdinalIgnoreCase)); config.Add("block_compressor=" + options.Get("collection-wt-block-compressor", "").ToLowerInvariant()); } if (!String.IsNullOrWhiteSpace(options.Get("collection-wt-allocation"))) { // Mongodb version 3.0.4 defaults to: "allocation_size=4KB,internal_page_max=4KB,leaf_page_max=32KB,leaf_value_max=1MB" if (options.Get("collection-wt-allocation") == "2x") { config.RemoveAll(i => i.StartsWith("allocation_size=", StringComparison.OrdinalIgnoreCase) || i.StartsWith("leaf_page_max=", StringComparison.OrdinalIgnoreCase) || i.StartsWith("internal_page_max=", StringComparison.OrdinalIgnoreCase)); config.Add("allocation_size=8KB"); config.Add("leaf_page_max=64KB"); config.Add("internal_page_max=8KB"); } else if (options.Get("collection-wt-allocation") == "4x") { config.RemoveAll(i => i.StartsWith("allocation_size=", StringComparison.OrdinalIgnoreCase) || i.StartsWith("leaf_page_max=", StringComparison.OrdinalIgnoreCase) || i.StartsWith("internal_page_max=", StringComparison.OrdinalIgnoreCase)); config.Add("allocation_size=16KB"); config.Add("leaf_page_max=64KB"); config.Add("internal_page_max=16KB"); } else if (options.Get("collection-wt-allocation") == "8x") { config.RemoveAll(i => i.StartsWith("allocation_size=", StringComparison.OrdinalIgnoreCase) || i.StartsWith("leaf_page_max=", StringComparison.OrdinalIgnoreCase) || i.StartsWith("internal_page_max=", StringComparison.OrdinalIgnoreCase)); config.Add("allocation_size=32KB"); config.Add("leaf_page_max=128KB"); config.Add("internal_page_max=32KB"); } } // apply configuration if (config.Count > 0) { try { var storageEngineDoc = new BsonDocument("wiredTiger", new BsonDocument("configString", String.Join(",", config))); targetCollection.Database.CreateCollection(targetCollection.Name, CollectionOptions.SetStorageEngineOptions(storageEngineDoc)); } catch (Exception ex) { NLog.LogManager.GetLogger("CreateCollection").Error(ex); } } }
private static bool HasCollectionCreationOptions(FlexibleOptions options) { return((options.HasOption("collection-wt-block-compressor") && valid_wt_compressors.Contains(options.Get("collection-wt-block-compressor", "invalid"))) || (!String.IsNullOrEmpty(options.Get("collection-wt-allocation")))); }
/// <summary> /// Log initialization. /// </summary> internal static void InitializeLog (string logFileName = null, string logLevel = null, InitializationOptions initOptions = null, FlexibleOptions appOptions = null) { // default parameters initialization from config file if (String.IsNullOrEmpty (logFileName)) logFileName = _logFileName ?? System.Configuration.ConfigurationManager.AppSettings["logFilename"]; if (String.IsNullOrEmpty (logFileName)) logFileName = ("${basedir}/log/" + typeof (ConsoleUtils).Namespace.Replace (".SimpleHelpers", "") + ".log"); if (String.IsNullOrEmpty (logLevel)) logLevel = _logLevel ?? (System.Configuration.ConfigurationManager.AppSettings["logLevel"] ?? "Info"); // check if log was initialized with same options if (_logFileName == logFileName && _logLevel == logLevel) return; // try to parse loglevel LogLevel currentLogLevel; try { currentLogLevel = LogLevel.FromString (logLevel); } catch { currentLogLevel = LogLevel.Info; } // save current log configuration _logFileName = logFileName; _logLevel = currentLogLevel.ToString (); // check initialization options var localOptions = initOptions != null ? initOptions.Clone () : new InitializationOptions (); // adjust options based on arguments if (appOptions != null) { if (!localOptions.DisableLogFile.HasValue && appOptions.HasOption ("DisableLogFile")) localOptions.DisableLogFile = appOptions.Get ("DisableLogFile", false); if (localOptions.EnableLogTargets == null && !String.IsNullOrEmpty (appOptions.Get ("EnableLogTargets"))) localOptions.EnableLogTargets = appOptions.GetAsList ("EnableLogTargets").Where (i => !String.IsNullOrWhiteSpace (i)).Select (i => i.Trim ()).ToArray (); if (localOptions.DisableLogTargets == null && !String.IsNullOrEmpty (appOptions.Get ("DisableLogTargets"))) localOptions.DisableLogTargets = appOptions.GetAsList ("DisableLogTargets").Where (i => !String.IsNullOrWhiteSpace (i)).Select (i => i.Trim ()).ToArray (); } // prepare list of enabled targets HashSet<string> enabledTargets; // if enabled log targets was provided, use it! if (localOptions.EnableLogTargets != null && localOptions.EnableLogTargets.Count > 0) { enabledTargets = new HashSet<string> (localOptions.EnableLogTargets, StringComparer.OrdinalIgnoreCase); } // else we remove disabled target... else { enabledTargets = new HashSet<string> (StringComparer.OrdinalIgnoreCase) { "console", "file" }; // set enabled targets if (localOptions.Targets != null) { foreach (var i in localOptions.Targets) { foreach (var n in GetNLogTargetName (i)) enabledTargets.Add (n); } } // remove disabled targets if (localOptions.DisableLogTargets != null) foreach (var i in localOptions.DisableLogTargets) enabledTargets.Remove (i); if (localOptions.DisableLogFile ?? false) enabledTargets.Remove ("file"); } // prepare log configuration var config = new NLog.Config.LoggingConfiguration (); // console output if (!Console.IsOutputRedirected && enabledTargets.Contains ("console")) { var consoleTarget = new NLog.Targets.ColoredConsoleTarget (); consoleTarget.Layout = "${longdate}\t${callsite}\t${level}\t${message}\t${onexception: \\:[Exception] ${exception:format=tostring}}"; config.AddTarget ("console", consoleTarget); var rule1 = new NLog.Config.LoggingRule ("*", LogLevel.Trace, consoleTarget); config.LoggingRules.Add (rule1); } // file output if (enabledTargets.Contains ("file")) { var fileTarget = new NLog.Targets.FileTarget (); fileTarget.FileName = logFileName; fileTarget.Layout = "${longdate}\t${callsite}\t${level}\t\"${message}${onexception: \t [Exception] ${exception:format=tostring}}\""; fileTarget.ConcurrentWrites = true; fileTarget.ConcurrentWriteAttemptDelay = 10; fileTarget.ConcurrentWriteAttempts = 8; fileTarget.AutoFlush = true; fileTarget.KeepFileOpen = true; fileTarget.DeleteOldFileOnStartup = false; fileTarget.ArchiveAboveSize = (localOptions.MaxLogFileSize > 0) ? localOptions.MaxLogFileSize : 4 * 1024 * 1024; // 4 Mb fileTarget.MaxArchiveFiles = (localOptions.MaxArchiveLogFiles > 0) ? localOptions.MaxArchiveLogFiles : 10; fileTarget.ArchiveNumbering = NLog.Targets.ArchiveNumberingMode.DateAndSequence; fileTarget.ArchiveDateFormat = "yyyyMMdd"; fileTarget.ArchiveFileName = System.IO.Path.ChangeExtension (logFileName, ".{#}" + System.IO.Path.GetExtension (logFileName)); // set file output to be async (commented out since doesn't work well on mono) // var wrapper = new NLog.Targets.Wrappers.AsyncTargetWrapper (fileTarget); config.AddTarget ("file", fileTarget); // configure log from configuration file var rule2 = new NLog.Config.LoggingRule ("*", currentLogLevel, fileTarget); config.LoggingRules.Add (rule2); } // External Log Target if (localOptions.Targets != null) { foreach (var t in localOptions.Targets) { if (GetNLogTargetName (t).Any (i => enabledTargets.Contains (i))) { config.AddTarget (t); config.LoggingRules.Add (new NLog.Config.LoggingRule ("*", currentLogLevel, t)); } } } // set configuration options LogManager.Configuration = config; }
/// <summary> /// Main program entry point. /// </summary> static void Main(string[] args) { // set error exit code System.Environment.ExitCode = -50; try { // load configurations ProgramOptions = ConsoleUtils.Initialize (args, true); // start execution Execute (ProgramOptions); // check before ending for waitForKeyBeforeExit option if (ProgramOptions.Get ("waitForKeyBeforeExit", false)) ConsoleUtils.WaitForAnyKey (); } catch (Exception ex) { LogManager.GetCurrentClassLogger ().Fatal (ex); // check before ending for waitForKeyBeforeExit option if (ProgramOptions.Get ("waitForKeyBeforeExit", false)) ConsoleUtils.WaitForAnyKey (); ConsoleUtils.CloseApplication (-60, true); } // set success exit code ConsoleUtils.CloseApplication (0, false); }
/// <summary> /// Parses out the Arguments received from the "CLI" /// </summary> /// <param name="args">Array of arguments received from the "CLI"</param> private static void ParseArguments(FlexibleOptions options) { // parse arguments _sourceUri = options["source"]; _sourceServer = options["sourceServer"]; _sourceUsername = options["sourceUsername"]; _sourcePassword = options["sourcePassword"]; _sourceAuthDatabase = options.Get("sourceAuthDatabase", options["authDatabaseNameSource"]); _targetUri = options["target"]; _targetServer = options["targetServer"]; _targetUsername = options["targetUsername"]; _targetPassword = options["targetPassword"]; _targetAuthDatabase = options.Get("targetAuthDatabase", options["authDatabaseNameTarget"]); _insertBatchSize = options.Get("batch-size", options.Get("insertBatchSize", -1)); _threads = options.Get("threads", 1); _copyIndexes = options.Get("copy-indexes", false); _dropCollections = options.Get("drop-collections", false); _skipExisting = options.Get("skip-existing", false); // check parameter databases _sourceDatabases = ParseArgumentAsList(options, "databases").Concat(ParseArgumentAsList(options, "sourceDatabase")).Distinct().ToList(); _targetDatabases = ParseArgumentAsList(options, "targetDatabase"); // check collections parameter _collections = ParseArgumentAsList(options, "collections"); //******************* //** sanity checks ** //******************* if (String.IsNullOrEmpty(_sourceUri) && String.IsNullOrEmpty(_sourceServer)) { logger.Error("No source mongodb server connection information provided: use the argument 'source' to provide a mongodb uri with the connection information"); ConsoleUtils.CloseApplication(-101, true); } // TODO: target server should defaults to the source server if none is provided... if (String.IsNullOrEmpty(_targetUri) && String.IsNullOrEmpty(_targetServer)) { logger.Error("No source mongodb server connection information provided: use the argument 'source' to provide a mongodb uri with the connection information"); ConsoleUtils.CloseApplication(-102, true); } if (_sourceDatabases.Count == 0) { logger.Error("No database selected: use the argument 'databases' to provide a list of databases"); ConsoleUtils.CloseApplication(-103, true); } // if no target database is provided, lets use the sourcedatabases if (_targetDatabases.Count == 0) { _targetDatabases = null; // TODO: if same server... throw an error! // ... } else { // if we have target database names: // 1. sourceDatabase cannot contain wildcard if (_sourceDatabases.Any(i => SharedMethods.HasWildcard(i))) { logger.Error("Wildcard cannot be used in source database names if a list of target databases is provided!"); ConsoleUtils.CloseApplication(-104, true); } // 2. sourceDatabase cannot contain wildcard if (_sourceDatabases.Any(i => SharedMethods.HasWildcard(i))) { logger.Error("Wildcard cannot be used in target database names!"); ConsoleUtils.CloseApplication(-105, true); } // 2. check for database mapping discrepancy if (_sourceDatabases.Count != _targetDatabases.Count) { logger.Error("Different number of source and target databases detected: use the argument 'databases' and 'targetDatabases' to provide a list of databases"); ConsoleUtils.CloseApplication(-106, true); } if (_sourceDatabases.Any(i => i.IndexOf('=') > 0)) { logger.Error("Invalid use of target database parameter: if the argument 'databases' has a key=value format to indicate the target database name, the argument 'targetDatabases' cannot be used."); ConsoleUtils.CloseApplication(-106, true); } } }
private static void CheckConnections(FlexibleOptions options) { // source server if (!String.IsNullOrWhiteSpace(options.Get("source"))) { if (options.Get("source").IndexOf("://") < 1) { options.Set("source", "mongodb://" + options.Get("source")); } var mongoUri = new MongoUrlBuilder(options.Get("source")); if (mongoUri.ConnectTimeout.TotalSeconds < 30) { mongoUri.ConnectTimeout = TimeSpan.FromSeconds(30); } if (mongoUri.SocketTimeout.TotalMinutes < 4) { mongoUri.SocketTimeout = TimeSpan.FromMinutes(4); } if (mongoUri.MaxConnectionIdleTime.TotalSeconds < 30) { mongoUri.MaxConnectionIdleTime = TimeSpan.FromSeconds(30); } // check for missing uri parameters if (!String.IsNullOrWhiteSpace(_sourceUsername) && String.IsNullOrWhiteSpace(mongoUri.Username)) { mongoUri.Username = _sourceUsername; } if (!String.IsNullOrWhiteSpace(_sourcePassword) && String.IsNullOrWhiteSpace(mongoUri.Password)) { mongoUri.Password = _sourcePassword; } if (!String.IsNullOrWhiteSpace(_sourceAuthDatabase) && String.IsNullOrWhiteSpace(mongoUri.AuthenticationSource)) { mongoUri.AuthenticationSource = _sourceAuthDatabase; } options.Set("source", mongoUri.ToString()); } else { options.Set("source", MongoDbContext.BuildConnectionString(_sourceUsername, _sourcePassword, true, true, _sourceServer, 30000, 4 * 60000, _sourceAuthDatabase)); } // check connection try { MongoDbContext.GetServer(options.Get("source")).Ping(); } catch (Exception ex) { logger.Error("Failed to connect to source mongodb server. Uri: {0}. Details: {1}", options.Get("source"), ex.Message); ConsoleUtils.CloseApplication(-111, true); } // target server if (!String.IsNullOrWhiteSpace(options.Get("target"))) { if (options.Get("target").IndexOf("://") < 1) { options.Set("target", "mongodb://" + options.Get("target")); } var mongoUri = new MongoUrlBuilder(options.Get("target")); if (mongoUri.ConnectTimeout.TotalSeconds < 30) { mongoUri.ConnectTimeout = TimeSpan.FromSeconds(30); } if (mongoUri.SocketTimeout.TotalMinutes < 4) { mongoUri.SocketTimeout = TimeSpan.FromMinutes(4); } if (mongoUri.MaxConnectionIdleTime.TotalSeconds < 30) { mongoUri.MaxConnectionIdleTime = TimeSpan.FromSeconds(30); } // check for missing uri parameters if (!String.IsNullOrWhiteSpace(_sourceUsername) && String.IsNullOrWhiteSpace(mongoUri.Username)) { mongoUri.Username = _sourceUsername; } if (!String.IsNullOrWhiteSpace(_sourcePassword) && String.IsNullOrWhiteSpace(mongoUri.Password)) { mongoUri.Password = _sourcePassword; } if (!String.IsNullOrWhiteSpace(_sourceAuthDatabase) && String.IsNullOrWhiteSpace(mongoUri.AuthenticationSource)) { mongoUri.AuthenticationSource = _sourceAuthDatabase; } options.Set("target", mongoUri.ToString()); } else { options.Set("target", MongoDbContext.BuildConnectionString(_targetUsername, _targetPassword, true, true, _targetServer, 30000, 4 * 60000, _targetAuthDatabase)); } // check connection try { MongoDbContext.GetServer(options.Get("target")).Ping(); } catch (Exception ex) { logger.Error("Failed to connect to target mongodb server. Uri: {0}. Details: {1}", options.Get("target"), ex.Message); ConsoleUtils.CloseApplication(-112, true); } }