Beispiel #1
0
        public static void Main(string[] args)
        {
            Rnd = new Random();

            Stats = new List<Statistics>();

            // TEMP
            DumpGeneratedServicesToFile = false;

            // log4net init
            NReco.Logging.LogManager.Configure(new NReco.Log4Net.Logger());

            // winter service provider init
            IComponentsConfig config = ConfigurationManager.GetSection("components") as IComponentsConfig;
            SrvPrv = new NReco.Winter.ServiceProvider(config);

            // overlay params
            Params = (NService.Chord.OverlayParams)SrvPrv.GetService("chordOverlayParams");
            Params.FillFromConsoleArgs(args);

            DbMgr = (IDbManager)SrvPrv.GetService("mongoDbManager");

            var serviceGenerationInputs = args[0].Split(new char[] { ',' });

            var serviceDescriptions = GetServiceDescriptions(serviceGenerationInputs[0]);

            // init services
            var p2pServices = new List<Tuple<IOverlayNodeService, IOverlayStorageService>>();

            var registeredNodeIds = new List<long>();

            if (args.Length > 1 && !String.IsNullOrEmpty(args[1])) {
                var overlayNodesCount = Convert.ToInt32(args[1]);
                for (var i = 0; i < overlayNodesCount; ++i) {
                    var chordService = (IOverlayNodeService)SrvPrv.GetService("chordOverlayNodeService");

                    var nInit = new NodeDescriptor.NodeInitInfo(
                                        Params.IpAddress,
                                        OverlayHelper.GetRandomPortNumber(),
                                        1
                                    );

                    var nodeId = (chordService as IHashKeyProvider).GetHashKey(OverlayHelper.GetNodeHashArgument(nInit));

                    if (!registeredNodeIds.Any(x => x == nodeId)) {

                        chordService.Bind(new LocalNode(new NodeDescriptor(
                            nInit,
                            nodeId
                        )));

                        // respective storage service
                        var storageService = (IOverlayStorageService)SrvPrv.GetService("mongoDbOverlayStorageService");
                        storageService.Bind(new[] { chordService }.ToList());
                        storageService.Start();

                        p2pServices.Add(Tuple.Create(chordService, storageService));

                        registeredNodeIds.Add(nodeId);
                    }
                }
            } else {
                var alreadyProcessedOwnerNodeIds = new Dictionary<int, IOverlayNode>();
                foreach (var srv in serviceDescriptions) {
                    var existingOverlayNode = alreadyProcessedOwnerNodeIds.Where(n => Convert.ToInt64(n.Key) == srv.OwnerNode.Id);
                    if (existingOverlayNode == null || !existingOverlayNode.Any()) {
                        var chordService = (IOverlayNodeService)SrvPrv.GetService("chordOverlayNodeService");

                        var nInit = new NodeDescriptor.NodeInitInfo(
                                        Params.IpAddress,
                                        OverlayHelper.GetRandomPortNumber(),
                                        1
                                    );
                        chordService.Bind(new LocalNode(new NodeDescriptor(
                            nInit,
                            srv.OwnerNode.Id
                        )));

                        alreadyProcessedOwnerNodeIds.Add(Convert.ToInt32(srv.OwnerNode.Id), chordService.Node);

                        // set real owner node for the service description
                        srv.OwnerNode = chordService.Node;
                        foreach (var d in srv.Data) {
                            d.OwnerNode = srv.OwnerNode;
                        }

                        // respective storage service
                        var storageService = (IOverlayStorageService)SrvPrv.GetService("mongoDbOverlayStorageService");
                        storageService.Bind(new[] { chordService }.ToList());
                        storageService.Start();

                        p2pServices.Add(Tuple.Create(chordService, storageService));
                    } else {
                        // we already created all needed stuff for this node, just set real owner node for respective service description
                        srv.OwnerNode = existingOverlayNode.Single().Value;
                        foreach (var d in srv.Data) {
                            d.OwnerNode = srv.OwnerNode;
                        }
                    }
                }
            }

            // create the overlay
            var alreadyJoinedNodesPorts = new List<int>();
            foreach (var p2p in p2pServices) {
                var overlaySrv = p2p.Item1;

                if (!alreadyJoinedNodesPorts.Any()) {
                    overlaySrv.Join();
                    alreadyJoinedNodesPorts.Add(overlaySrv.Node.Port);
                } else {
                    var nInitLoc = new NodeDescriptor.NodeInitInfo(
                                    Params.IpAddress,
                                    alreadyJoinedNodesPorts[Rnd.Next(0, alreadyJoinedNodesPorts.Count)],
                                    1
                                );
                    var hashPrv = (IProvider<byte[], long>)SrvPrv.GetService("sha1KeyPrv");
                    overlaySrv.Join(new NodeDescriptor(
                        nInitLoc,
                        hashPrv.Provide(OverlayHelper.GetNodeHashArgument(nInitLoc))
                    ));
                    alreadyJoinedNodesPorts.Add(overlaySrv.Node.Port);
                }
                Thread.Sleep(1000);
            }

            System.Console.WriteLine("Waiting until the overlay stabilizes...");
            Thread.Sleep(10000);

            //populate storage
            PopulateStorage(p2pServices, serviceDescriptions, args.Length > 2 ? (int?)Convert.ToInt32(args[2]) : null);

            // auto-populate more for stats
            if (serviceGenerationInputs.Length > 1) {
                foreach (var srvGen in serviceGenerationInputs) {
                    PopulateStorage(p2pServices, GetServiceDescriptions(srvGen), args.Length > 2 ? (int?)Convert.ToInt32(args[2]) : null);
                }
            }

            System.Console.WriteLine("Ready");
            // listen for commands
            try {
                ProcessCommands(p2pServices);
            } finally {
                foreach (var srv in p2pServices) {
                    srv.Item2.Stop();
                }
                foreach (var srv in p2pServices) {
                    ((ChordOverlayNodeService)srv.Item1).MaintenanceSrv.Stop();
                }
                foreach (var srv in p2pServices) {
                    srv.Item1.SendMessage(new OverlayNodeServiceMessage { MsgType = OverlayNodeServiceMessage.MessageType.ExitRequested, WaitForProcessing = true });
                }
            }
        }
Beispiel #2
0
        public static void Main(string[] args)
        {
            Rnd = new Random();

            // log4net init
            NReco.Logging.LogManager.Configure(new NReco.Log4Net.Logger());

            // winter service provider init
            IComponentsConfig config = ConfigurationManager.GetSection("components") as IComponentsConfig;
            SrvPrv = new NReco.Winter.ServiceProvider(config);

            // overlay params
            Params = (NService.Chord.OverlayParams)SrvPrv.GetService("chordOverlayParams");
            Params.FillFromConsoleArgs(args);
            // hack for quick services disposal
            Params.TreatRejoinRequestedAsExit = true;

            var stateSrvConnector = (IRemoteServiceConnector<IOverlayStateService>)SrvPrv.GetService("overlayStateServiceWcfConnector");
            StateSrv = stateSrvConnector.Connect();

            DbMgr = (IDbManager)SrvPrv.GetService("mongoDbManager");
            DbMgr.Clear();

            var statResults = new StringBuilder();

            if (args.Length == 2) {
                NodesQuantity = new int[] { Convert.ToInt32(args[0]) };
                ServicesPerNodeQuantity = new int[] { Convert.ToInt32(args[1]) };
            }

            try {
                statResults.Append("{");

                foreach (var snq in ServicesPerNodeQuantity) {
                    statResults.AppendFormat("{0}: [ ", snq);

                    foreach (var nq in NodesQuantity) {
                        var alreadyJoinedNodesPorts = new List<int>();
                        var serviceDescriptions = new List<ServiceDescription>();
                        var testObjects = new List<TestObjects>();

                        for (var i = 0; i < nq; ++i) {
                            try {
                                TestObjects testObj = null;

                                testObj = InitNode(ref alreadyJoinedNodesPorts);
                                testObjects.Add(testObj);

                                for (var j = 0; j < snq; ++j) {
                                    var srvD = TestHelper.GetRandomServiceDescription(testObj.StorageSrv.Node, Rnd);
                                    serviceDescriptions.Add(srvD);
                                    testObj.StorageSrv.PutService(srvD);
                                }
                            } catch { }
                        }

                        Thread.Sleep(1000);

                        // obtain statistics
                        for (var i = 0; i < 10; ++i) {
                            var storSrv = testObjects[Rnd.Next(0, testObjects.Count)].StorageSrv;
                            var queryData = i < 5
                                                ? TestHelper.GetRandomServiceDescription(storSrv.Node, Rnd)
                                                : serviceDescriptions[Rnd.Next(0, serviceDescriptions.Count)];
                            var pos = Rnd.Next(0, queryData.Data.Count);
                            try {
                                storSrv.GetServices(
                                    Query.FromRelex(
                                        String.Format(
                                            "{0} = \"{1}\"",
                                            queryData.Data[pos].AttrValue.Key,
                                            queryData.Data[pos].AttrValue.Value
                                        )
                                    ),
                                    false
                                );
                            } catch { }
                        }

                        var stats = StateSrv.CollectStatistics("Query", true);

                        statResults.AppendFormat("[{0},{1}], ", nq, stats.Data.Average(st => st.TimeElapsed.TotalMilliseconds));

                        //System.Console.WriteLine("\r\n******************************************\r\n");
                        //System.Console.ReadLine();

                        System.Console.WriteLine("Previous session cleanup started...");
                        foreach (var t in testObjects) {
                            t.StorageSrv.Stop();
                        }
                        foreach (var t in testObjects) {
                            ((ChordOverlayNodeService)t.NodeSrv).MaintenanceSrv.Stop();
                        }
                        foreach (var t in testObjects) {
                            t.NodeSrv.SendMessage(new OverlayNodeServiceMessage { MsgType = OverlayNodeServiceMessage.MessageType.ExitRequested, WaitForProcessing = true });
                        }
                        DbMgr.Clear();
                        System.Console.WriteLine("Previous session cleanup finished!");
                    }
                    statResults.Remove(statResults.Length - 2, 2); // remove 'hanging comma & space'
                    statResults.Append("], ");
                }
                statResults.Remove(statResults.Length - 2, 2); // remove 'hanging comma & space'
                statResults.Append("}");
            } finally {
                stateSrvConnector.Close(StateSrv);

                var resDirName = "..\\..\\..\\..\\results\\" + Guid.NewGuid().ToString();
                Directory.CreateDirectory(resDirName);
                using (var sw = new StreamWriter(new FileStream(resDirName + "\\statresults_" + Guid.NewGuid().ToString() + ".json", FileMode.Create, FileAccess.Write, FileShare.None))) {
                    sw.WriteLine(statResults.ToString());
                }
            }
        }
Beispiel #3
0
        static int Main(string[] args)
        {
            Thread.CurrentThread.CurrentUICulture = CultureInfo.GetCultureInfo("en-US");

            LogManager.Configure(new TraceLogger(true, false)
            {
                TimestampFormat = "{0:hh:mm:ss}"
            });
            IDictionary <string, object> cmdParams;

            try {
                cmdParams = ExtractCmdParams(args, paramDescriptors);
            } catch (Exception ex) {
                log.Write(LogEvent.Fatal, ex.Message, ex);
                Console.Error.WriteLine(ex.Message);
                return(1);
            }
            if (!cmdParams.ContainsKey(BasePathParam))
            {
                cmdParams[BasePathParam] = Environment.CurrentDirectory;
                log.Write(LogEvent.Info, "Base path is not defined, using current directory: {0}", cmdParams[BasePathParam]);
            }
            if (!cmdParams.ContainsKey(IsIncrementalParam))
            {
                cmdParams[IsIncrementalParam] = false;
                log.Write(LogEvent.Info, "Incremental processing is not defined, by default: {0}.", cmdParams[IsIncrementalParam]);
            }
            if (!cmdParams.ContainsKey(IsWatchParam))
            {
                cmdParams[IsWatchParam] = false;
                log.Write(LogEvent.Info, "Watch mode is not defined, by default: {0}.", cmdParams[IsWatchParam]);
            }

            string                   rootFolder       = (string)cmdParams[BasePathParam];
            RuleStatsTracker         ruleStatsTracker = new RuleStatsTracker();
            LocalFolderRuleProcessor folderRuleProcessor;
            MergeConfig              mergeConfig = null;

            try {
                IComponentsConfig     config = ConfigurationSettings.GetConfig("components") as IComponentsConfig;
                INamedServiceProvider srvPrv = new NReco.Winter.ServiceProvider(config);

                folderRuleProcessor = srvPrv.GetService("folderRuleProcessor") as LocalFolderRuleProcessor;
                if (folderRuleProcessor == null)
                {
                    log.Write(LogEvent.Fatal, "Configuration error: missed or incorrect 'folderRuleProcessor' component");
                    return(2);
                }

                // read merge config
                if (cmdParams.ContainsKey(MergeParam))
                {
                    mergeConfig = new MergeConfig((string)cmdParams[MergeParam], rootFolder);
                }
                log.Write(LogEvent.Info, "Reading Folder: {0}", rootFolder);
                DateTime         dt           = DateTime.Now;
                LocalFileManager localFileMgr = new LocalFileManager(rootFolder);
                localFileMgr.Incremental = (bool)cmdParams[IsIncrementalParam];

                localFileMgr.Reading += new FileManagerEventHandler(ruleStatsTracker.OnFileReading);
                localFileMgr.Writing += new FileManagerEventHandler(ruleStatsTracker.OnFileWriting);
                folderRuleProcessor.RuleExecuting     += new FileRuleEventHandler(ruleStatsTracker.OnRuleExecuting);
                folderRuleProcessor.RuleExecuted      += new FileRuleEventHandler(ruleStatsTracker.OnRuleExecuted);
                folderRuleProcessor.RuleFileReadStart += new FileRuleEventHandler(ruleStatsTracker.OnRuleExecuting);
                folderRuleProcessor.RuleFileReadEnd   += new FileRuleEventHandler(ruleStatsTracker.OnRuleExecuted);

                folderRuleProcessor.FileManager = localFileMgr;
                localFileMgr.StartSession();
                folderRuleProcessor.Execute();
                localFileMgr.EndSession();

                log.Write(LogEvent.Info, "Apply time: {0}", DateTime.Now.Subtract(dt).TotalSeconds.ToString());
            } catch (Exception ex) {
                log.Write(LogEvent.Fatal, "Transformation failed: {0}", ex.ToString());
                return(3);
            }


            if (Convert.ToBoolean(cmdParams[IsWatchParam]))
            {
                // some delay for avoid watching for just changed files
                Thread.Sleep(100);

                Watcher w = new Watcher(rootFolder, ruleStatsTracker, folderRuleProcessor, mergeConfig);
                log.Write(LogEvent.Info, "Watching for filesystem changes... (press 'q' for exit)");
                w.Start();
                while (true)
                {
                    ConsoleKeyInfo keyInfo = System.Console.ReadKey();
                    if (keyInfo.KeyChar == 'q')
                    {
                        break;
                    }
                }
                w.Stop();
            }

            return(0);
        }
Beispiel #4
0
        static int Main(string[] args)
        {
            Thread.CurrentThread.CurrentUICulture = CultureInfo.GetCultureInfo("en-US");

            LogManager.Configure(new TraceLogger(true, false) { TimestampFormat = "{0:hh:mm:ss}" });
            IDictionary<string, object> cmdParams;
            try {
                cmdParams = ExtractCmdParams(args, paramDescriptors);
            } catch (Exception ex) {
                log.Write(LogEvent.Fatal, ex.Message, ex);
                Console.Error.WriteLine(ex.Message);
                return 1;
            }
            if (!cmdParams.ContainsKey(BasePathParam)) {
                cmdParams[BasePathParam] = Environment.CurrentDirectory;
                log.Write(LogEvent.Info, "Base path is not defined, using current directory: {0}", cmdParams[BasePathParam]);
            }
            if (!cmdParams.ContainsKey(IsIncrementalParam)) {
                cmdParams[IsIncrementalParam] = false;
                log.Write(LogEvent.Info, "Incremental processing is not defined, by default: {0}.", cmdParams[IsIncrementalParam]);
            }
            if (!cmdParams.ContainsKey(IsWatchParam)) {
                cmdParams[IsWatchParam] = false;
                log.Write(LogEvent.Info, "Watch mode is not defined, by default: {0}.", cmdParams[IsWatchParam]);
            }

            string rootFolder = (string)cmdParams[BasePathParam];
            RuleStatsTracker ruleStatsTracker = new RuleStatsTracker();
            LocalFolderRuleProcessor folderRuleProcessor;
            MergeConfig mergeConfig = null;
            try {
                IComponentsConfig config = ConfigurationSettings.GetConfig("components") as IComponentsConfig;
                INamedServiceProvider srvPrv = new NReco.Winter.ServiceProvider(config);

                folderRuleProcessor = srvPrv.GetService("folderRuleProcessor") as LocalFolderRuleProcessor;
                if (folderRuleProcessor == null) {
                    log.Write(LogEvent.Fatal, "Configuration error: missed or incorrect 'folderRuleProcessor' component");
                    return 2;
                }

                // read merge config
                if (cmdParams.ContainsKey(MergeParam)) {
                    mergeConfig = new MergeConfig( (string) cmdParams[MergeParam], rootFolder );
                }
                log.Write(LogEvent.Info, "Reading Folder: {0}", rootFolder);
                DateTime dt = DateTime.Now;
                LocalFileManager localFileMgr = new LocalFileManager(rootFolder);
                localFileMgr.Incremental = (bool)cmdParams[IsIncrementalParam];

                localFileMgr.Reading += new FileManagerEventHandler(ruleStatsTracker.OnFileReading);
                localFileMgr.Writing += new FileManagerEventHandler(ruleStatsTracker.OnFileWriting);
                folderRuleProcessor.RuleExecuting += new FileRuleEventHandler(ruleStatsTracker.OnRuleExecuting);
                folderRuleProcessor.RuleExecuted += new FileRuleEventHandler(ruleStatsTracker.OnRuleExecuted);
                folderRuleProcessor.RuleFileReadStart += new FileRuleEventHandler(ruleStatsTracker.OnRuleExecuting);
                folderRuleProcessor.RuleFileReadEnd += new FileRuleEventHandler(ruleStatsTracker.OnRuleExecuted);

                folderRuleProcessor.FileManager = localFileMgr;
                localFileMgr.StartSession();
                folderRuleProcessor.Execute();
                localFileMgr.EndSession();

                log.Write(LogEvent.Info, "Apply time: {0}", DateTime.Now.Subtract(dt).TotalSeconds.ToString());
            } catch (Exception ex) {
                log.Write(LogEvent.Fatal, "Transformation failed: {0}", ex.ToString());
                return 3;
            }

            if (Convert.ToBoolean(cmdParams[IsWatchParam])) {
                // some delay for avoid watching for just changed files
                Thread.Sleep(100);

                Watcher w = new Watcher(rootFolder, ruleStatsTracker, folderRuleProcessor, mergeConfig);
                log.Write(LogEvent.Info, "Watching for filesystem changes... (press 'q' for exit)");
                w.Start();
                while (true) {
                    ConsoleKeyInfo keyInfo = System.Console.ReadKey();
                    if (keyInfo.KeyChar == 'q')
                        break;
                }
                w.Stop();

            }

            return 0;
        }