示例#1
0
 static AppGlobal()
 {
     AppLogger = new AppLog();
     Logger = new AppErrorLog();
     FormatWriter = new LogFormatter(ref Logger);
     AppLogger.GenerateLogfiles(Application.StartupPath + "\\" + Application.ProductName + "\\ApplicationLog", "Trace_" + Application.ProductName);
     Logger.GenerateLogfiles(Application.StartupPath + "\\" + Application.ProductName + "\\ApplicationLog", Application.ProductName);
 }
示例#2
0
            public async Task <GrainStateRecord> Read(string partitionKey, string rowKey)
            {
                if (logger.IsVerbose3)
                {
                    logger.Verbose3((int)AzureProviderErrorCode.AzureTableProvider_Storage_Reading, "Reading: PartitionKey={0} RowKey={1} from Table={2}", partitionKey, rowKey, TableName);
                }
                try
                {
                    Tuple <DynamicTableEntity, string> data = await tableManager.ReadSingleTableEntryAsync(partitionKey, rowKey).ConfigureAwait(false);

                    if (data == null || data.Item1 == null)
                    {
                        if (logger.IsVerbose2)
                        {
                            logger.Verbose2((int)AzureProviderErrorCode.AzureTableProvider_DataNotFound, "DataNotFound reading: PartitionKey={0} RowKey={1} from Table={2}", partitionKey, rowKey, TableName);
                        }
                        return(null);
                    }
                    DynamicTableEntity stateEntity = data.Item1;
                    var record = new GrainStateRecord {
                        Entity = stateEntity, ETag = data.Item2
                    };
                    if (logger.IsVerbose3)
                    {
                        logger.Verbose3((int)AzureProviderErrorCode.AzureTableProvider_Storage_DataRead, "Read: PartitionKey={0} RowKey={1} from Table={2} with ETag={3}", stateEntity.PartitionKey, stateEntity.RowKey, TableName, record.ETag);
                    }
                    return(record);
                }
                catch (Exception exc)
                {
                    if (AzureStorageUtils.TableStorageDataNotFound(exc))
                    {
                        if (logger.IsVerbose2)
                        {
                            logger.Verbose2((int)AzureProviderErrorCode.AzureTableProvider_DataNotFound, "DataNotFound reading (exception): PartitionKey={0} RowKey={1} from Table={2} Exception={3}", partitionKey, rowKey, TableName, LogFormatter.PrintException(exc));
                        }
                        return(null);  // No data
                    }
                    throw;
                }
            }
示例#3
0
        /// <summary>
        /// Report an error during silo startup.
        /// </summary>
        /// <remarks>
        /// Information on the silo startup issue will be logged to any attached Loggers,
        /// then a timestamped StartupError text file will be written to
        /// the current working directory (if possible).
        /// </remarks>
        /// <param name="exc">Exception which caused the silo startup issue.</param>
        public void ReportStartupError(Exception exc)
        {
            if (string.IsNullOrWhiteSpace(Name))
            {
                Name = "Silo";
            }

            var errMsg = "ERROR starting Orleans silo name=" + Name + " Exception=" + LogFormatter.PrintException(exc);

            if (logger != null)
            {
                logger.Error(ErrorCode.Runtime_Error_100105, errMsg, exc);
            }

            // Dump Startup error to a log file
            var now = DateTime.UtcNow;

            var dateString = now.ToString(dateFormat, CultureInfo.InvariantCulture);
            var startupLog = Name + "-StartupError-" + dateString + ".txt";

            try
            {
                File.AppendAllText(startupLog, dateString + "Z" + Environment.NewLine + errMsg);
            }
            catch (Exception exc2)
            {
                if (logger != null)
                {
                    logger.Error(ErrorCode.Runtime_Error_100106, "Error writing log file " + startupLog, exc2);
                }
            }
        }
示例#4
0
        /// <summary>
        /// Generates and loads code for the specified inputs.
        /// </summary>
        /// <param name="inputs">The assemblies to generate code for.</param>
        public void GenerateAndLoadForAssemblies(params Assembly[] inputs)
        {
            if (inputs == null)
            {
                throw new ArgumentNullException(nameof(inputs));
            }

            var timer            = Stopwatch.StartNew();
            var emitDebugSymbols = false;

            foreach (var input in inputs)
            {
                if (!emitDebugSymbols)
                {
                    emitDebugSymbols |= RuntimeVersion.IsAssemblyDebugBuild(input);
                }

                RegisterGeneratedCodeTargets(input);
                TryLoadGeneratedAssemblyFromCache(input);
            }

            var grainAssemblies = inputs.Where(ShouldGenerateCodeForAssembly).ToList();

            if (grainAssemblies.Count == 0)
            {
                // Already up to date.
                return;
            }

            try
            {
                // Generate code for newly loaded assemblies.
                var generatedSyntax = GenerateForAssemblies(grainAssemblies, true);

                CachedAssembly generatedAssembly;
                if (generatedSyntax.Syntax != null)
                {
                    generatedAssembly = CompileAndLoad(generatedSyntax, emitDebugSymbols);
                }
                else
                {
                    generatedAssembly = new CachedAssembly {
                        Loaded = true
                    };
                }

                foreach (var assembly in generatedSyntax.SourceAssemblies)
                {
                    CompiledAssemblies.AddOrUpdate(
                        assembly.GetName().FullName,
                        generatedAssembly,
                        (_, __) => generatedAssembly);
                }

                if (Logger.IsVerbose2)
                {
                    Logger.Verbose2(
                        ErrorCode.CodeGenCompilationSucceeded,
                        "Generated code for {0} assemblies in {1}ms",
                        generatedSyntax.SourceAssemblies.Count,
                        timer.ElapsedMilliseconds);
                }
            }
            catch (Exception exception)
            {
                var assemblyNames = string.Join("\n", grainAssemblies.Select(_ => _.GetName().FullName));
                var message       =
                    $"Exception generating code for input assemblies:\n{assemblyNames}\nException: {LogFormatter.PrintException(exception)}";
                Logger.Warn(ErrorCode.CodeGenCompilationFailed, message, exception);
                throw;
            }
        }
示例#5
0
 private Lager(LogFormatter logFormatter)
 {
     Formatter = logFormatter;
 }
示例#6
0
 public override void Format(StringBuilder sb, LogFormatter formatter)
 {
     SerilogLogFormatter.FormatLogEvent(formatter, sb, _logEvent);
 }
示例#7
0
        /// <summary>
        /// Async method to validate specific cluster configuration
        /// </summary>
        /// <param name="config"></param>
        /// <returns>Task object of boolean type for this async method </returns>
        public async Task <bool> ValidateConfiguration(ClusterConfiguration config)
        {
            if (config.Globals.LivenessType == GlobalConfiguration.LivenessProviderType.AzureTable)
            {
                string deploymentId     = config.Globals.DeploymentId ?? serviceRuntimeWrapper.DeploymentId;
                string connectionString = config.Globals.DataConnectionString ??
                                          serviceRuntimeWrapper.GetConfigurationSettingValue(DataConnectionConfigurationSettingName);

                try
                {
                    var manager = siloInstanceManager ?? await OrleansSiloInstanceManager.GetManager(deploymentId, connectionString, loggerFactory);

                    var instances = await manager.DumpSiloInstanceTable();

                    logger.Verbose(instances);
                }
                catch (Exception exc)
                {
                    var error = String.Format("Connecting to the storage table has failed with {0}", LogFormatter.PrintException(exc));
                    Trace.TraceError(error);
                    logger.Error(ErrorCode.AzureTable_34, error, exc);
                    return(false);
                }
            }

            return(true);
        }
示例#8
0
 public void Log <T>(SemanticLogLevel logLevel, T context, Exception?exception, LogFormatter <T> action)
 {
     LogLevels[logLevel] = LogLevels.GetOrDefault(logLevel) + 1;
 }
示例#9
0
        public int RunMain(string[] args)
        {
            ConsoleText.WriteStatus("Orleans-CodeGen - command-line = {0}", Environment.CommandLine);

            if (args.Length < 1)
            {
                PrintUsage();

                return(1);
            }

            try
            {
                var options = new CodeGenOptions();

                // STEP 1 : Parse parameters
                if (args.Length == 1 && args[0].StartsWith("@"))
                {
                    // Read command line args from file
                    string arg      = args[0];
                    string argsFile = arg.Trim('"').Substring(1).Trim('"');
                    Console.WriteLine("Orleans-CodeGen - Reading code-gen params from file={0}", argsFile);
                    AssertWellFormed(argsFile, true);
                    args = File.ReadAllLines(argsFile);
                }
                int i = 1;
                foreach (string a in args)
                {
                    string arg = a.Trim('"').Trim().Trim('"');
                    if (GrainClientGeneratorFlags.Verbose)
                    {
                        Console.WriteLine("Orleans-CodeGen - arg #{0}={1}", i++, arg);
                    }
                    if (string.IsNullOrEmpty(arg) || string.IsNullOrWhiteSpace(arg))
                    {
                        continue;
                    }

                    if (arg.StartsWith("/"))
                    {
                        if (arg.StartsWith("/reference:") || arg.StartsWith("/r:"))
                        {
                            // list of references passed from from project file. separator =';'
                            string   refstr     = arg.Substring(arg.IndexOf(':') + 1);
                            string[] references = refstr.Split(';');
                            foreach (string rp in references)
                            {
                                AssertWellFormed(rp, true);
                                options.ReferencedAssemblies.Add(rp);
                            }
                        }
                        else if (arg.StartsWith("/in:"))
                        {
                            var infile = arg.Substring(arg.IndexOf(':') + 1);
                            AssertWellFormed(infile);
                            options.InputAssembly = new FileInfo(infile);
                        }
                        else if (arg.StartsWith("/out:"))
                        {
                            var outfile = arg.Substring(arg.IndexOf(':') + 1);
                            AssertWellFormed(outfile, false);
                            options.OutputFileName = outfile;
                        }
                    }
                    else
                    {
                        Console.WriteLine($"Invalid argument: {arg}.");

                        PrintUsage();

                        return(1);
                    }
                }

                // STEP 2 : Validate and calculate unspecified parameters
                if (options.InputAssembly == null)
                {
                    Console.WriteLine("ERROR: Orleans-CodeGen - no input file specified.");
                    return(2);
                }

                if (String.IsNullOrEmpty(options.OutputFileName))
                {
                    Console.WriteLine("ERROR: Orleans-Codegen - no output filename specified");
                    return(2);
                }

                // STEP 3 : Dump useful info for debugging
                Console.WriteLine($"Orleans-CodeGen - Options {Environment.NewLine}\tInputLib={options.InputAssembly.FullName}{Environment.NewLine}\tOutputFileName={options.OutputFileName}");

                if (options.ReferencedAssemblies != null)
                {
                    Console.WriteLine("Orleans-CodeGen - Using referenced libraries:");
                    foreach (string assembly in options.ReferencedAssemblies)
                    {
                        Console.WriteLine("\t{0} => {1}", Path.GetFileName(assembly), assembly);
                    }
                }

                // STEP 5 : Finally call code generation
                if (!CreateGrainClientAssembly(options))
                {
                    return(-1);
                }

                // DONE!
                return(0);
            }
            catch (Exception ex)
            {
                Console.WriteLine("-- Code-gen FAILED -- \n{0}", LogFormatter.PrintException(ex));
                return(3);
            }
        }
示例#10
0
        private static void InitializeImpl_FromConfig(ClientConfiguration config)
        {
            if (GrainClient.IsInitialized)
            {
                Trace.TraceInformation("Connection to Orleans gateway silo already initialized.");
                return;
            }

            //// Find endpoint info for the gateway to this Orleans silo cluster
            //Trace.WriteLine("Searching for Orleans gateway silo via Orleans instance table...");
            var clusterId        = config.ClusterId;
            var connectionString = config.DataConnectionString;

            if (String.IsNullOrEmpty(clusterId))
            {
                throw new ArgumentException("Cannot connect to Azure silos with null deploymentId", "config.ClusterId");
            }

            if (String.IsNullOrEmpty(connectionString))
            {
                throw new ArgumentException("Cannot connect to Azure silos with null connectionString", "config.DataConnectionString");
            }

            bool      initSucceeded = false;
            Exception lastException = null;

            for (int i = 0; i < MaxRetries; i++)
            {
                try
                {
                    GrainClient.ConfigureLoggingDelegate = ConfigureLoggingDelegate;
                    GrainClient.ConfigureClientDelegate  = ConfigureClientDelegate;
                    // Initialize will throw if cannot find Gateways
                    GrainClient.Initialize(config);
                    initSucceeded = true;
                    break;
                }
                catch (Exception exc)
                {
                    lastException = exc;
                    Trace.TraceError("Client.Initialize failed with exc -- {0}. Will try again", exc.Message);
                }
                // Pause to let Primary silo start up and register
                Trace.TraceInformation("Pausing {0} awaiting silo and gateways registration for Deployment={1}", StartupRetryPause, clusterId);
                Thread.Sleep(StartupRetryPause);
            }

            if (initSucceeded)
            {
                return;
            }

            OrleansException err;

            err = lastException != null ? new OrleansException(String.Format("Could not Initialize Client for ClusterId={0}. Last exception={1}",
                                                                             clusterId, lastException.Message), lastException) : new OrleansException(String.Format("Could not Initialize Client for ClusterId={0}.", clusterId));
            Trace.TraceError("Error starting Orleans Azure client application -- {0} -- bailing. {1}", err.Message, LogFormatter.PrintException(err));
            throw err;
        }
示例#11
0
        private static void InitializeImpl_FromFile(FileInfo configFile)
        {
            if (GrainClient.IsInitialized)
            {
                Trace.TraceInformation("Connection to Orleans gateway silo already initialized.");
                return;
            }

            ClientConfiguration config;

            try
            {
                if (configFile == null)
                {
                    Trace.TraceInformation("Looking for standard Orleans client config file");
                    config = ClientConfiguration.StandardLoad();
                }
                else
                {
                    var configFileLocation = configFile.FullName;
                    Trace.TraceInformation("Loading Orleans client config file {0}", configFileLocation);
                    config = ClientConfiguration.LoadFromFile(configFileLocation);
                }
            }
            catch (Exception ex)
            {
                var msg = String.Format("Error loading Orleans client configuration file {0} {1} -- unable to continue. {2}", configFile, ex.Message, LogFormatter.PrintException(ex));
                Trace.TraceError(msg);
                throw new AggregateException(msg, ex);
            }

            Trace.TraceInformation("Overriding Orleans client config from Azure runtime environment.");
            try
            {
                config.ClusterId            = GetDeploymentId();
                config.DataConnectionString = GetDataConnectionString();
                config.GatewayProvider      = ClientConfiguration.GatewayProviderType.AzureTable;
            }
            catch (Exception ex)
            {
                var msg = string.Format("ERROR: No AzureClient role setting value '{0}' specified for this role -- unable to continue", AzureConstants.DataConnectionConfigurationSettingName);
                Trace.TraceError(msg);
                throw new AggregateException(msg, ex);
            }

            InitializeImpl_FromConfig(config);
        }
示例#12
0
        public override string ToString()
        {
            var sb = new StringBuilder();

            sb.AppendLine("Platform version info:").Append(ConfigUtilities.RuntimeVersionInfo());
            sb.Append("   Host: ").AppendLine(Dns.GetHostName());
            sb.Append("   Processor Count: ").Append(System.Environment.ProcessorCount).AppendLine();

            sb.AppendLine("Client Configuration:");
            sb.Append("   Config File Name: ").AppendLine(string.IsNullOrEmpty(SourceFile) ? "" : Path.GetFullPath(SourceFile));
            sb.Append("   Start time: ").AppendLine(LogFormatter.PrintDate(DateTime.UtcNow));
            sb.Append("   Gateway Provider: ").Append(GatewayProvider);
            if (GatewayProvider == GatewayProviderType.None)
            {
                sb.Append(".   Gateway Provider that will be used instead: ").Append(GatewayProviderToUse);
            }
            sb.AppendLine();
            if (Gateways != null && Gateways.Count > 0)
            {
                sb.AppendFormat("   Gateways[{0}]:", Gateways.Count).AppendLine();
                foreach (var endpoint in Gateways)
                {
                    sb.Append("      ").AppendLine(endpoint.ToString());
                }
            }
            else
            {
                sb.Append("   Gateways: ").AppendLine("Unspecified");
            }
            sb.Append("   Preferred Gateway Index: ").AppendLine(PreferedGatewayIndex.ToString());
            if (Gateways != null && PreferedGatewayIndex >= 0 && PreferedGatewayIndex < Gateways.Count)
            {
                sb.Append("   Preferred Gateway Address: ").AppendLine(Gateways[PreferedGatewayIndex].ToString());
            }
            sb.Append("   GatewayListRefreshPeriod: ").Append(GatewayListRefreshPeriod).AppendLine();
            if (!String.IsNullOrEmpty(DeploymentId) || !String.IsNullOrEmpty(DataConnectionString))
            {
                sb.Append("   Azure:").AppendLine();
                sb.Append("      DeploymentId: ").Append(DeploymentId).AppendLine();
                string dataConnectionInfo = ConfigUtilities.RedactConnectionStringInfo(DataConnectionString); // Don't print Azure account keys in log files
                sb.Append("      DataConnectionString: ").Append(dataConnectionInfo).AppendLine();
            }
            if (!string.IsNullOrWhiteSpace(NetInterface))
            {
                sb.Append("   Network Interface: ").AppendLine(NetInterface);
            }
            if (Port != 0)
            {
                sb.Append("   Network Port: ").Append(Port).AppendLine();
            }
            sb.Append("   Preferred Address Family: ").AppendLine(PreferredFamily.ToString());
            sb.Append("   DNS Host Name: ").AppendLine(DNSHostName);
            sb.Append("   Client Name: ").AppendLine(ClientName);
            sb.Append(ConfigUtilities.TraceConfigurationToString(this));
            sb.Append(ConfigUtilities.IStatisticsConfigurationToString(this));
            sb.Append(LimitManager);
            sb.AppendFormat(base.ToString());
            sb.Append("   .NET: ").AppendLine();
            int workerThreads;
            int completionPortThreads;

            ThreadPool.GetMinThreads(out workerThreads, out completionPortThreads);
            sb.AppendFormat("       .NET thread pool sizes - Min: Worker Threads={0} Completion Port Threads={1}", workerThreads, completionPortThreads).AppendLine();
            ThreadPool.GetMaxThreads(out workerThreads, out completionPortThreads);
            sb.AppendFormat("       .NET thread pool sizes - Max: Worker Threads={0} Completion Port Threads={1}", workerThreads, completionPortThreads).AppendLine();
            sb.AppendFormat("   Providers:").AppendLine();
            sb.Append(ProviderConfigurationUtility.PrintProviderConfigurations(ProviderConfigurations));
            return(sb.ToString());
        }
示例#13
0
        public static int Main(string[] args)
        {
            CultureInfo.CurrentCulture                = CultureInfo.InvariantCulture;
            CultureInfo.CurrentUICulture              = CultureInfo.InvariantCulture;
            CultureInfo.DefaultThreadCurrentCulture   = CultureInfo.InvariantCulture;
            CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture;

            Console.WriteLine("Orleans-CodeGen - command-line = {0}", Environment.CommandLine);

            if (args.Length < 1)
            {
                PrintUsage();

                return(1);
            }

            try
            {
                var options = new CodeGenOptions();

                // STEP 1 : Parse parameters
                if (args.Length == 1 && args[0].StartsWith("@"))
                {
                    // Read command line args from file
                    string arg      = args[0];
                    string argsFile = arg.Trim('"').Substring(1).Trim('"');
                    Console.WriteLine("Orleans-CodeGen - Reading code-gen params from file={0}", argsFile);
                    AssertWellFormed(argsFile);
                    args = File.ReadAllLines(argsFile);
                }
                foreach (string a in args)
                {
                    string arg = a.Trim('"').Trim().Trim('"');
                    if (string.IsNullOrEmpty(arg) || string.IsNullOrWhiteSpace(arg))
                    {
                        continue;
                    }

                    if (arg.StartsWith("/"))
                    {
                        if (arg.StartsWith("/reference:") || arg.StartsWith("/r:"))
                        {
                            // list of references passed from from project file. separator =';'
                            string   refstr     = arg.Substring(arg.IndexOf(':') + 1);
                            string[] references = refstr.Split(';');
                            foreach (string rp in references)
                            {
                                AssertWellFormed(rp);
                                options.ReferencedAssemblies.Add(rp);
                            }
                        }
                        else if (arg.StartsWith("/in:"))
                        {
                            var infile = arg.Substring(arg.IndexOf(':') + 1);
                            AssertWellFormed(infile);
                            options.InputAssembly = new FileInfo(infile);
                        }
                        else if (arg.StartsWith("/out:"))
                        {
                            var outfile = arg.Substring(arg.IndexOf(':') + 1);
                            AssertWellFormed(outfile);
                            options.OutputFileName = outfile;
                        }
                        else if (arg.StartsWith("/loglevel:"))
                        {
                            var levelString = arg.Substring(arg.IndexOf(':') + 1);
                            if (!Enum.TryParse(ignoreCase: true, value: levelString, result: out LogLevel level))
                            {
                                var validValues = string.Join(", ", Enum.GetNames(typeof(LogLevel)).Select(v => v.ToString()));
                                Console.WriteLine($"ERROR: \"{levelString}\" is not a valid log level. Valid values are {validValues}");
                                return(1);
                            }

                            options.LogLevel = level;
                        }
                    }
                    else
                    {
                        Console.WriteLine($"Invalid argument: {arg}.");

                        PrintUsage();

                        return(1);
                    }
                }

                // STEP 2 : Validate and calculate unspecified parameters
                if (options.InputAssembly == null)
                {
                    Console.WriteLine("ERROR: Orleans-CodeGen - no input file specified.");
                    return(2);
                }

                if (String.IsNullOrEmpty(options.OutputFileName))
                {
                    Console.WriteLine("ERROR: Orleans-Codegen - no output filename specified");
                    return(2);
                }

                // STEP 3 : Dump useful info for debugging
                Console.WriteLine($"Orleans-CodeGen - Options {Environment.NewLine}\tInputLib={options.InputAssembly.FullName}{Environment.NewLine}\tOutputFileName={options.OutputFileName}");

                bool referencesOrleans = options.InputAssembly.Name.Equals(CodeGenerator.OrleansAssemblyFileName);
                foreach (string assembly in options.ReferencedAssemblies)
                {
                    var fileName = Path.GetFileName(assembly);
                    Console.WriteLine("\t{0} => {1}", fileName, assembly);
                    if (fileName != null && fileName.Equals(CodeGenerator.OrleansAssemblyFileName))
                    {
                        referencesOrleans = true;
                    }
                }

                // STEP 5 : Finally call code generation
                if (referencesOrleans)
                {
                    if (!CodeGenerator.GenerateCode(options))
                    {
                        Console.WriteLine("WARNING: Orleans-CodeGen - the input assembly contained no types which required code generation.");
                    }
                }
                else
                {
                    Console.WriteLine("ERROR: Orleans-CodeGen - the input assembly does not reference Orleans and therefore code can not be generated.");
                    return(-2);
                }

                // DONE!
                return(0);
            }
            catch (Exception ex)
            {
                Console.WriteLine("-- Code Generation FAILED -- \n{0}", LogFormatter.PrintException(ex));
                return(3);
            }
        }
示例#14
0
 public void PrintEntry(Severity severity, int code, string message)
 {
     writer.WriteLine(LogFormatter.Format(severity, code, indentLevel, message));
 }
        public async Task <bool> TryToSuspectOrKill(SiloAddress silo)
        {
            var table = await membershipTableProvider.ReadAll();

            if (log.IsEnabled(LogLevel.Debug))
            {
                log.Debug("-TryToSuspectOrKill: Read Membership table {0}", table.ToString());
            }

            if (this.IsStopping)
            {
                this.log.LogInformation(
                    (int)ErrorCode.MembershipFoundMyselfDead3,
                    "Ignoring call to TrySuspectOrKill for silo {Silo} since the local silo is dead",
                    silo);
                return(true);
            }

            var(localSiloEntry, _) = this.GetOrCreateLocalSiloEntry(table, this.CurrentStatus);
            if (localSiloEntry.Status == SiloStatus.Dead)
            {
                var msg = string.Format("I should be Dead according to membership table (in TryToSuspectOrKill): entry = {0}.", localSiloEntry.ToFullString(full: true));
                log.Warn(ErrorCode.MembershipFoundMyselfDead3, msg);
                KillMyselfLocally(msg);
                return(true);
            }

            if (!table.Contains(silo))
            {
                // this should not happen ...
                var str = string.Format("-Could not find silo entry for silo {0} in the table.", silo);
                log.Error(ErrorCode.MembershipFailedToReadSilo, str);
                throw new KeyNotFoundException(str);
            }

            var    tuple = table.Get(silo);
            var    entry = tuple.Item1.Copy();
            string eTag  = tuple.Item2;

            if (log.IsEnabled(LogLevel.Debug))
            {
                log.Debug("-TryToSuspectOrKill {siloAddress}: The current status of {siloAddress} in the table is {status}, its entry is {entry}",
                          entry.SiloAddress, // First
                          entry.SiloAddress, // Second
                          entry.Status,
                          entry.ToFullString());
            }
            // check if the table already knows that this silo is dead
            if (entry.Status == SiloStatus.Dead)
            {
                this.ProcessTableUpdate(table, "TrySuspectOrKill");
                return(true);
            }

            var allVotes = entry.SuspectTimes ?? new List <Tuple <SiloAddress, DateTime> >();

            // get all valid (non-expired) votes
            var freshVotes = entry.GetFreshVotes(DateTime.UtcNow, this.clusterMembershipOptions.DeathVoteExpirationTimeout);

            if (log.IsEnabled(LogLevel.Trace))
            {
                log.Trace("-Current number of fresh Voters for {0} is {1}", silo, freshVotes.Count.ToString());
            }

            if (freshVotes.Count >= this.clusterMembershipOptions.NumVotesForDeathDeclaration)
            {
                // this should not happen ...
                var str = string.Format("-Silo {0} is suspected by {1} which is more or equal than {2}, but is not marked as dead. This is a bug!!!",
                                        entry.SiloAddress, freshVotes.Count.ToString(), this.clusterMembershipOptions.NumVotesForDeathDeclaration.ToString());
                log.Error(ErrorCode.Runtime_Error_100053, str);
                KillMyselfLocally("Found a bug! Will stop.");
                return(false);
            }

            // handle the corner case when the number of active silos is very small (then my only vote is enough)
            int activeSilos = table.GetSiloStatuses(status => status == SiloStatus.Active, true, this.localSiloDetails.SiloAddress).Count;
            // find if I have already voted
            int myVoteIndex = freshVotes.FindIndex(voter => myAddress.Equals(voter.Item1));

            // Try to kill:
            //  if there is NumVotesForDeathDeclaration votes (including me) to kill - kill.
            //  otherwise, if there is a majority of nodes (including me) voting to kill – kill.
            bool declareDead      = false;
            int  myAdditionalVote = myVoteIndex == -1 ? 1 : 0;

            if (freshVotes.Count + myAdditionalVote >= this.clusterMembershipOptions.NumVotesForDeathDeclaration)
            {
                declareDead = true;
            }

            if (freshVotes.Count + myAdditionalVote >= (activeSilos + 1) / 2)
            {
                declareDead = true;
            }

            if (declareDead)
            {
                // kick this silo off
                log.Info(ErrorCode.MembershipMarkingAsDead,
                         "-Going to mark silo {0} as DEAD in the table #1. I am the last voter: #freshVotes={1}, myVoteIndex = {2}, NumVotesForDeathDeclaration={3} , #activeSilos={4}, suspect list={5}",
                         entry.SiloAddress,
                         freshVotes.Count,
                         myVoteIndex,
                         this.clusterMembershipOptions.NumVotesForDeathDeclaration,
                         activeSilos,
                         PrintSuspectList(allVotes));
                return(await DeclareDead(entry, eTag, table.Version));
            }

            // we still do not have enough votes - need to vote
            // find voting place:
            //      update my vote, if I voted previously
            //      OR if the list is not full - just add a new vote
            //      OR overwrite the oldest entry.
            int indexToWrite = allVotes.FindIndex(voter => myAddress.Equals(voter.Item1));

            if (indexToWrite == -1)
            {
                // My vote is not recorded. Find the most outdated vote if the list is full, and overwrite it
                if (allVotes.Count >= this.clusterMembershipOptions.NumVotesForDeathDeclaration) // if the list is full
                {
                    // The list is full.
                    DateTime minVoteTime = allVotes.Min(voter => voter.Item2); // pick the most outdated vote
                    indexToWrite = allVotes.FindIndex(voter => voter.Item2.Equals(minVoteTime));
                }
            }

            var prevList = allVotes.ToList(); // take a copy
            var now      = DateTime.UtcNow;

            if (indexToWrite == -1)
            {
                // if did not find specific place to write (the list is not full), just add a new element to the list
                entry.AddSuspector(myAddress, now);
            }
            else
            {
                var newEntry = new Tuple <SiloAddress, DateTime>(myAddress, now);
                entry.SuspectTimes[indexToWrite] = newEntry;
            }
            log.Info(ErrorCode.MembershipVotingForKill,
                     "-Putting my vote to mark silo {0} as DEAD #2. Previous suspect list is {1}, trying to update to {2}, eTag={3}, freshVotes is {4}",
                     entry.SiloAddress,
                     PrintSuspectList(prevList),
                     PrintSuspectList(entry.SuspectTimes),
                     eTag,
                     PrintSuspectList(freshVotes));

            // If we fail to update here we will retry later.
            return(await membershipTableProvider.UpdateRow(entry, eTag, table.Version.Next()));

            string PrintSuspectList(IEnumerable <Tuple <SiloAddress, DateTime> > list)
            {
                return(Utils.EnumerableToString(list, t => string.Format("<{0}, {1}>",
                                                                         t.Item1, LogFormatter.PrintDate(t.Item2))));
            }
        }
示例#16
0
        private static MembershipEntry Parse(SiloInstanceTableEntry tableEntry)
        {
            var parse = new MembershipEntry
            {
                HostName = tableEntry.HostName,
                Status   = (SiloStatus)Enum.Parse(typeof(SiloStatus), tableEntry.Status)
            };

            if (!string.IsNullOrEmpty(tableEntry.ProxyPort))
            {
                parse.ProxyPort = int.Parse(tableEntry.ProxyPort);
            }

            int port = 0;

            if (!string.IsNullOrEmpty(tableEntry.Port))
            {
                int.TryParse(tableEntry.Port, out port);
            }

            int gen = 0;

            if (!string.IsNullOrEmpty(tableEntry.Generation))
            {
                int.TryParse(tableEntry.Generation, out gen);
            }

            parse.SiloAddress = SiloAddress.New(new IPEndPoint(IPAddress.Parse(tableEntry.Address), port), gen);

            parse.RoleName = tableEntry.RoleName;
            if (!string.IsNullOrEmpty(tableEntry.SiloName))
            {
                parse.SiloName = tableEntry.SiloName;
            }
            else if (!string.IsNullOrEmpty(tableEntry.InstanceName))
            {
                // this is for backward compatability: in a mixed cluster of old and new version,
                // some entries will have the old InstanceName column.
                parse.SiloName = tableEntry.InstanceName;
            }
            if (!string.IsNullOrEmpty(tableEntry.UpdateZone))
            {
                parse.UpdateZone = int.Parse(tableEntry.UpdateZone);
            }

            if (!string.IsNullOrEmpty(tableEntry.FaultZone))
            {
                parse.FaultZone = int.Parse(tableEntry.FaultZone);
            }

            parse.StartTime = !string.IsNullOrEmpty(tableEntry.StartTime) ?
                              LogFormatter.ParseDate(tableEntry.StartTime) : default(DateTime);

            parse.IAmAliveTime = !string.IsNullOrEmpty(tableEntry.IAmAliveTime) ?
                                 LogFormatter.ParseDate(tableEntry.IAmAliveTime) : default(DateTime);

            var suspectingSilos = new List <SiloAddress>();
            var suspectingTimes = new List <DateTime>();

            if (!string.IsNullOrEmpty(tableEntry.SuspectingSilos))
            {
                string[] silos = tableEntry.SuspectingSilos.Split('|');
                foreach (string silo in silos)
                {
                    suspectingSilos.Add(SiloAddress.FromParsableString(silo));
                }
            }

            if (!string.IsNullOrEmpty(tableEntry.SuspectingTimes))
            {
                string[] times = tableEntry.SuspectingTimes.Split('|');
                foreach (string time in times)
                {
                    suspectingTimes.Add(LogFormatter.ParseDate(time));
                }
            }

            if (suspectingSilos.Count != suspectingTimes.Count)
            {
                throw new OrleansException(String.Format("SuspectingSilos.Length of {0} as read from Azure table is not eqaul to SuspectingTimes.Length of {1}", suspectingSilos.Count, suspectingTimes.Count));
            }

            for (int i = 0; i < suspectingSilos.Count; i++)
            {
                parse.AddSuspector(suspectingSilos[i], suspectingTimes[i]);
            }

            return(parse);
        }
 public override void Format(StringBuilder sb, LogFormatter formatter)
 {
     sb.Append(JsonConvert.SerializeObject(this));
 }
示例#18
0
 public void Log(SemanticLogLevel logLevel, Exception?exception, LogFormatter action)
 {
     LogLevels[logLevel] = LogLevels.GetOrDefault(logLevel) + 1;
 }
 public TestSink(ILogsApi api, LogFormatter formatter, BatchingSinkOptions sinkOptions)
     : base(api, formatter, sinkOptions)
 {
 }
示例#20
0
        /// <summary>
        /// Initialize this Orleans silo for execution with the specified Azure deploymentId
        /// </summary>
        /// <param name="config">If null, Config data will be read from silo config file as normal, otherwise use the specified config data.</param>
        /// <param name="deploymentId">Azure DeploymentId this silo is running under</param>
        /// <param name="connectionString">Azure DataConnectionString. If null, defaults to the DataConnectionString setting from the Azure configuration for this role.</param>
        /// <returns><c>true</c> if the silo startup was successful</returns>
        internal bool Start(ClusterConfiguration config, string deploymentId, string connectionString)
        {
            if (config != null && deploymentId != null)
            {
                throw new ArgumentException("Cannot use config and deploymentId on the same time");
            }

            // Program ident
            Trace.TraceInformation("Starting {0} v{1}", this.GetType().FullName, RuntimeVersion.Current);

            // Read endpoint info for this instance from Azure config
            string instanceName = serviceRuntimeWrapper.InstanceName;

            // Configure this Orleans silo instance
            if (config == null)
            {
                host = new SiloHost(instanceName);
                host.LoadOrleansConfig(); // Load config from file + Initializes logger configurations
            }
            else
            {
                host = new SiloHost(instanceName, config); // Use supplied config data + Initializes logger configurations
            }

            IPEndPoint myEndpoint    = serviceRuntimeWrapper.GetIPEndpoint(SiloEndpointConfigurationKeyName);
            IPEndPoint proxyEndpoint = serviceRuntimeWrapper.GetIPEndpoint(ProxyEndpointConfigurationKeyName);

            host.SetSiloType(Silo.SiloType.Secondary);

            int generation = SiloAddress.AllocateNewGeneration();

            // Bootstrap this Orleans silo instance

            // If deploymentId was not direclty provided, take the value in the config. If it is not
            // in the config too, just take the DeploymentId from Azure
            if (deploymentId == null)
            {
                deploymentId = string.IsNullOrWhiteSpace(host.Config.Globals.DeploymentId)
                    ? serviceRuntimeWrapper.DeploymentId
                    : host.Config.Globals.DeploymentId;
            }

            myEntry = new SiloInstanceTableEntry
            {
                DeploymentId = deploymentId,
                Address      = myEndpoint.Address.ToString(),
                Port         = myEndpoint.Port.ToString(CultureInfo.InvariantCulture),
                Generation   = generation.ToString(CultureInfo.InvariantCulture),

                HostName  = host.Config.GetOrCreateNodeConfigurationForSilo(host.Name).DNSHostName,
                ProxyPort = (proxyEndpoint != null ? proxyEndpoint.Port : 0).ToString(CultureInfo.InvariantCulture),

                RoleName   = serviceRuntimeWrapper.RoleName,
                SiloName   = instanceName,
                UpdateZone = serviceRuntimeWrapper.UpdateDomain.ToString(CultureInfo.InvariantCulture),
                FaultZone  = serviceRuntimeWrapper.FaultDomain.ToString(CultureInfo.InvariantCulture),
                StartTime  = LogFormatter.PrintDate(DateTime.UtcNow),

                PartitionKey = deploymentId,
                RowKey       = myEndpoint.Address + "-" + myEndpoint.Port + "-" + generation
            };

            if (connectionString == null)
            {
                connectionString = serviceRuntimeWrapper.GetConfigurationSettingValue(DataConnectionConfigurationSettingName);
            }

            try
            {
                siloInstanceManager = OrleansSiloInstanceManager.GetManager(
                    deploymentId, connectionString, this.loggerFactory).WithTimeout(AzureTableDefaultPolicies.TableCreationTimeout).Result;
            }
            catch (Exception exc)
            {
                var error = String.Format("Failed to create OrleansSiloInstanceManager. This means CreateTableIfNotExist for silo instance table has failed with {0}",
                                          LogFormatter.PrintException(exc));
                Trace.TraceError(error);
                logger.Error(ErrorCode.AzureTable_34, error, exc);
                throw new OrleansException(error, exc);
            }

            // Always use Azure table for membership when running silo in Azure
            host.SetSiloLivenessType(GlobalConfiguration.LivenessProviderType.AzureTable);
            if (host.Config.Globals.ReminderServiceType == GlobalConfiguration.ReminderServiceProviderType.NotSpecified ||
                host.Config.Globals.ReminderServiceType == GlobalConfiguration.ReminderServiceProviderType.ReminderTableGrain)
            {
                host.SetReminderServiceType(GlobalConfiguration.ReminderServiceProviderType.AzureTable);
            }
            host.SetExpectedClusterSize(serviceRuntimeWrapper.RoleInstanceCount);
            siloInstanceManager.RegisterSiloInstance(myEntry);

            // Initialize this Orleans silo instance
            host.SetDeploymentId(deploymentId, connectionString);
            host.SetSiloEndpoint(myEndpoint, generation);
            host.SetProxyEndpoint(proxyEndpoint);

            host.InitializeOrleansSilo();
            return(StartSilo());
        }
 public ILoggerDuckTypingTests()
 {
     _sink                = new NullDatadogSink();
     _formatter           = LogSettingsHelper.GetFormatter();
     _iloggerProviderType = LoggerFactoryIntegrationCommon <ILoggerProvider> .ProviderInterfaces;
 }
示例#22
0
 public LogPipeSpecification(TextWriter writer, LogFormatter <T> formatter)
 {
     _writer    = writer;
     _formatter = formatter;
 }
示例#23
0
        public void Log(
            LogLevel logLevel,
            int eventId,
            object state,
            Exception exception,
            Func <object, Exception, string> formatter)
        {
            if (!IsEnabled(logLevel))
            {
                return;
            }
            var message = string.Empty;
            var values  = state as ILogValues;

            if (formatter != null)
            {
                message = formatter(state, exception);
            }
            else if (values != null)
            {
                var builder = new StringBuilder();
                FormatLogValues(
                    builder,
                    values,
                    level: 1,
                    bullet: false);

                message = builder.ToString();
                if (exception != null)
                {
                    message += Environment.NewLine + exception;
                }
            }
            else
            {
                message = LogFormatter.Formatter(state, exception);
            }
            if (string.IsNullOrEmpty(message))
            {
                return;
            }

            contextAccessor = services.GetService <IHttpContextAccessor>();

            string ipAddress = GetIpAddress();
            string culture   = CultureInfo.CurrentCulture.Name;
            string url       = GetRequestUrl();
            string shortUrl  = GetShortUrl(url);
            string thread    = System.Threading.Thread.CurrentThread.Name + " " + eventId.ToString();
            string logLev    = logLevel.ToString();

            // an exception is expected here if the db has not yet been populated
            // or if the db is not accessible
            // we cannot allow logging to raise exceptions
            // so we must swallow any exception here
            // would be good if we could only swallow specific exceptions
            try
            {
                logRepo.AddLogItem(
                    DateTime.UtcNow,
                    ipAddress,
                    culture,
                    url,
                    shortUrl,
                    thread,
                    logLev,
                    logger,
                    message);
            }
            catch
            { }
        }
示例#24
0
        /// <summary>
        /// Ensures that code generation has been run for the provided assembly.
        /// </summary>
        /// <param name="input">
        /// The assembly to generate code for.
        /// </param>
        public void GenerateAndLoadForAssembly(Assembly input)
        {
            try
            {
                RegisterGeneratedCodeTargets(input);
                if (!ShouldGenerateCodeForAssembly(input))
                {
                    TryLoadGeneratedAssemblyFromCache(input);

                    return;
                }

                var timer     = Stopwatch.StartNew();
                var generated = GenerateForAssemblies(new List <Assembly> {
                    input
                }, true);

                CachedAssembly generatedAssembly;
                if (generated.Syntax != null)
                {
                    var emitDebugSymbols = RuntimeVersion.IsAssemblyDebugBuild(input);
                    generatedAssembly = CompileAndLoad(generated, emitDebugSymbols);
                }
                else
                {
                    generatedAssembly = new CachedAssembly {
                        Loaded = true
                    };
                }

                foreach (var assembly in generated.SourceAssemblies)
                {
                    CompiledAssemblies.AddOrUpdate(
                        assembly.GetName().FullName,
                        generatedAssembly,
                        (_, __) => generatedAssembly);
                }

                if (Logger.IsVerbose2)
                {
                    Logger.Verbose2(
                        ErrorCode.CodeGenCompilationSucceeded,
                        "Generated code for 1 assembly in {0}ms",
                        timer.ElapsedMilliseconds);
                }
            }
            catch (Exception exception)
            {
                var message =
                    $"Exception generating code for input assembly {input.GetName().FullName}\nException: {LogFormatter.PrintException(exception)}";
                Logger.Warn(ErrorCode.CodeGenCompilationFailed, message, exception);
                throw;
            }
        }
示例#25
0
        private string MakeErrorMsg(string what, Exception exc)
        {
            HttpStatusCode httpStatusCode;
            string         errorCode = string.Empty;

            var decoder = store as IRestExceptionDecoder;

            decoder?.DecodeException(exc, out httpStatusCode, out errorCode, true);

            return(string.Format("Error from storage provider {0} during {1} for grain Type={2} Pk={3} Id={4} Error={5}" + Environment.NewLine + " {6}",
                                 $"{this.store.GetType().Name}.{this.name}", what, name, grainRef.GrainId.ToDetailedString(), grainRef, errorCode, LogFormatter.PrintException(exc)));
        }
示例#26
0
        private static async Task <T> ExecuteWithRetriesHelper <T>(
            Func <int, Task <T> > function,
            int callCounter,
            int maxNumSuccessTries,
            int maxNumErrorTries,
            TimeSpan maxExecutionTime,
            DateTime startExecutionTime,
            Func <T, int, bool> retryValueFilter             = null,
            Func <Exception, int, bool> retryExceptionFilter = null,
            IBackoffProvider onSuccessBackOff = null,
            IBackoffProvider onErrorBackOff   = null)
        {
            T result = default(T);
            ExceptionDispatchInfo lastExceptionInfo = null;
            bool retry;

            do
            {
                retry = false;

                if (maxExecutionTime != Constants.INFINITE_TIMESPAN && maxExecutionTime != default(TimeSpan))
                {
                    DateTime now = DateTime.UtcNow;
                    if (now - startExecutionTime > maxExecutionTime)
                    {
                        if (lastExceptionInfo == null)
                        {
                            throw new TimeoutException(
                                      $"ExecuteWithRetries has exceeded its max execution time of {maxExecutionTime}. Now is {LogFormatter.PrintDate(now)}, started at {LogFormatter.PrintDate(startExecutionTime)}, passed {now - startExecutionTime}");
                        }

                        lastExceptionInfo.Throw();
                    }
                }

                int counter = callCounter;

                try
                {
                    callCounter++;
                    result = await function(counter);

                    lastExceptionInfo = null;

                    if (callCounter < maxNumSuccessTries || maxNumSuccessTries == INFINITE_RETRIES) // -1 for infinite retries
                    {
                        if (retryValueFilter != null)
                        {
                            retry = retryValueFilter(result, counter);
                        }
                    }

                    if (retry)
                    {
                        TimeSpan?delay = onSuccessBackOff?.Next(counter);

                        if (delay.HasValue)
                        {
                            await Task.Delay(delay.Value);
                        }
                    }
                }
                catch (Exception exc)
                {
                    retry = false;

                    if (callCounter < maxNumErrorTries || maxNumErrorTries == INFINITE_RETRIES)
                    {
                        if (retryExceptionFilter != null)
                        {
                            retry = retryExceptionFilter(exc, counter);
                        }
                    }

                    if (!retry)
                    {
                        throw;
                    }

                    lastExceptionInfo = ExceptionDispatchInfo.Capture(exc);

                    TimeSpan?delay = onErrorBackOff?.Next(counter);

                    if (delay.HasValue)
                    {
                        await Task.Delay(delay.Value);
                    }
                }
            } while (retry);

            return(result);
        }
示例#27
0
        /// <summary>
        /// Generates, compiles, and loads the
        /// </summary>
        /// <param name="assemblies">
        /// The assemblies to generate code for.
        /// </param>
        public Assembly GenerateAndLoadForAssemblies(IEnumerable <Assembly> assemblies)
        {
            var assemblyList = assemblies.Where(ShouldGenerateCodeForAssembly).ToList();

            try
            {
                var timer     = Stopwatch.StartNew();
                var generated = this.GenerateCode(targetAssembly: null, assemblies: assemblyList);

                Assembly generatedAssembly;
                if (generated.Syntax != null)
                {
                    var emitDebugSymbols = assemblyList.Any(RuntimeVersion.IsAssemblyDebugBuild);
                    generatedAssembly = this.CompileAssembly(generated, "OrleansCodeGen", emitDebugSymbols);
                }
                else
                {
                    generatedAssembly = null;
                }

                if (this.logger.IsEnabled(LogLevel.Debug))
                {
                    this.logger.Debug(
                        ErrorCode.CodeGenCompilationSucceeded,
                        "Generated code for 1 assembly in {0}ms",
                        timer.ElapsedMilliseconds);
                }

                return(generatedAssembly);
            }
            catch (Exception exception)
            {
                var message =
                    $"Exception generating code for input assemblies {string.Join(",", assemblyList.Select(asm => asm.GetName().FullName))}\nException: {LogFormatter.PrintException(exception)}";
                this.logger.Warn(ErrorCode.CodeGenCompilationFailed, message, exception);
                throw;
            }
        }
 /// <summary>
 /// Handles cache purge signals
 /// </summary>
 /// <param name="lastItemPurged"></param>
 /// <param name="newestItem"></param>
 private void OnPurge(CachedMessage?lastItemPurged, CachedMessage?newestItem)
 {
     if (logger.IsEnabled(LogLevel.Debug) && lastItemPurged.HasValue && newestItem.HasValue)
     {
         logger.Debug($"CachePeriod: EnqueueTimeUtc: {LogFormatter.PrintDate(lastItemPurged.Value.EnqueueTimeUtc)} to {LogFormatter.PrintDate(newestItem.Value.EnqueueTimeUtc)}, DequeueTimeUtc: {LogFormatter.PrintDate(lastItemPurged.Value.DequeueTimeUtc)} to {LogFormatter.PrintDate(newestItem.Value.DequeueTimeUtc)}");
     }
     if (lastItemPurged.HasValue)
     {
         checkpointer.Update(this.dataAdapter.GetOffset(lastItemPurged.Value), DateTime.UtcNow);
     }
 }
示例#29
0
 public static string FormatLogEvent(LogFormatter logFormatter, in LogEntry logEntryWrapper)
 private static ILogFormatter SetupLogFormatter()
 {
     var logFormatter = new LogFormatter();
     return logFormatter;
 }
示例#31
0
        public async Task StatelessWorkerActivationsPerSiloDoNotExceedMaxLocalWorkersCount()
        {
            var gatewaysCount = HostedCluster.ClientConfiguration.Gateways.Count;
            // do extra calls to trigger activation of ExpectedMaxLocalActivations local activations
            int numberOfCalls = ExpectedMaxLocalActivations * 3 * gatewaysCount;

            IStatelessWorkerGrain grain    = this.GrainFactory.GetGrain <IStatelessWorkerGrain>(GetRandomGrainId());
            List <Task>           promises = new List <Task>();

            // warmup
            for (int i = 0; i < gatewaysCount; i++)
            {
                promises.Add(grain.LongCall());
            }
            await Task.WhenAll(promises);

            await Task.Delay(2000);

            promises.Clear();
            var stopwatch = Stopwatch.StartNew();

            for (int i = 0; i < numberOfCalls; i++)
            {
                promises.Add(grain.LongCall());
            }
            await Task.WhenAll(promises);

            stopwatch.Stop();

            promises.Clear();

            var statsTasks = new List <Task <Tuple <Guid, string, List <Tuple <DateTime, DateTime> > > > >();

            for (int i = 0; i < numberOfCalls; i++)
            {
                statsTasks.Add(grain.GetCallStats());  // gather stats
            }
            await Task.WhenAll(promises);

            var responsesPerSilo = statsTasks.Select(t => t.Result).GroupBy(s => s.Item2);

            foreach (var siloGroup in responsesPerSilo)
            {
                var silo = siloGroup.Key;

                HashSet <Guid> activations = new HashSet <Guid>();

                foreach (var response in siloGroup)
                {
                    if (activations.Contains(response.Item1))
                    {
                        continue; // duplicate response from the same activation
                    }
                    activations.Add(response.Item1);

                    output.WriteLine($"Silo {silo} with {activations.Count} activations: Activation {response.Item1}");
                    int count = 1;
                    foreach (Tuple <DateTime, DateTime> call in response.Item3)
                    {
                        output.WriteLine($"\t{count++}: {LogFormatter.PrintDate(call.Item1)} - {LogFormatter.PrintDate(call.Item2)}");
                    }
                }

                Assert.True(activations.Count <= ExpectedMaxLocalActivations, $"activations.Count = {activations.Count} in silo {silo} but expected no more than {ExpectedMaxLocalActivations}");
            }
        }
 public void SetUp()
 {
     xmlLogFormatter = new XmlLogFormatter();
 }
示例#33
0
        public async Task <bool> TryToSuspectOrKill(SiloAddress silo)
        {
            var table = await membershipTableProvider.ReadAll();

            var now = GetDateTimeUtcNow();

            if (log.IsEnabled(LogLevel.Debug))
            {
                log.Debug("-TryToSuspectOrKill: Read Membership table {0}", table.ToString());
            }

            if (this.IsStopping)
            {
                this.log.LogInformation(
                    (int)ErrorCode.MembershipFoundMyselfDead3,
                    "Ignoring call to TrySuspectOrKill for silo {Silo} since the local silo is dead",
                    silo);
                return(true);
            }

            var(localSiloEntry, _) = this.GetOrCreateLocalSiloEntry(table, this.CurrentStatus);
            if (localSiloEntry.Status == SiloStatus.Dead)
            {
                var msg = string.Format("I should be Dead according to membership table (in TryToSuspectOrKill): entry = {0}.", localSiloEntry.ToFullString(full: true));
                log.Warn(ErrorCode.MembershipFoundMyselfDead3, msg);
                KillMyselfLocally(msg);
                return(true);
            }

            if (!table.Contains(silo))
            {
                // this should not happen ...
                var str = string.Format("-Could not find silo entry for silo {0} in the table.", silo);
                log.Error(ErrorCode.MembershipFailedToReadSilo, str);
                throw new KeyNotFoundException(str);
            }

            var    tuple = table.Get(silo);
            var    entry = tuple.Item1.Copy();
            string eTag  = tuple.Item2;

            if (log.IsEnabled(LogLevel.Debug))
            {
                log.Debug("-TryToSuspectOrKill {siloAddress}: The current status of {siloAddress} in the table is {status}, its entry is {entry}",
                          entry.SiloAddress, // First
                          entry.SiloAddress, // Second
                          entry.Status,
                          entry.ToFullString());
            }

            // Check if the table already knows that this silo is dead
            if (entry.Status == SiloStatus.Dead)
            {
                this.ProcessTableUpdate(table, "TrySuspectOrKill");
                return(true);
            }

            // Get all valid (non-expired) votes
            var freshVotes = entry.GetFreshVotes(now, this.clusterMembershipOptions.DeathVoteExpirationTimeout);

            if (log.IsEnabled(LogLevel.Trace))
            {
                log.Trace("-Current number of fresh Voters for {0} is {1}", silo, freshVotes.Count.ToString());
            }

            if (freshVotes.Count >= this.clusterMembershipOptions.NumVotesForDeathDeclaration)
            {
                // this should not happen ...
                var str = string.Format("-Silo {0} is suspected by {1} which is more or equal than {2}, but is not marked as dead. This is a bug!!!",
                                        entry.SiloAddress, freshVotes.Count.ToString(), this.clusterMembershipOptions.NumVotesForDeathDeclaration.ToString());
                log.Error(ErrorCode.Runtime_Error_100053, str);
                KillMyselfLocally("Found a bug! Will stop.");
                return(false);
            }

            // Try to add our vote to the list and tally the fresh votes again.
            var prevList = entry.SuspectTimes?.ToList() ?? new List <Tuple <SiloAddress, DateTime> >();

            entry.AddOrUpdateSuspector(myAddress, now, clusterMembershipOptions.NumVotesForDeathDeclaration);
            freshVotes = entry.GetFreshVotes(now, this.clusterMembershipOptions.DeathVoteExpirationTimeout);

            // Determine if there are enough votes to evict the silo.
            // Handle the corner case when the number of active silos is very small (then my only vote is enough)
            int activeSilos = table.GetSiloStatuses(status => status == SiloStatus.Active, true, myAddress).Count;

            if (freshVotes.Count >= clusterMembershipOptions.NumVotesForDeathDeclaration || freshVotes.Count >= (activeSilos + 1) / 2)
            {
                // Find the local silo's vote index
                int myVoteIndex = freshVotes.FindIndex(voter => myAddress.Equals(voter.Item1));

                // Kick this silo off
                log.Info(ErrorCode.MembershipMarkingAsDead,
                         "-Going to mark silo {0} as DEAD in the table #1. This silo is the last voter: #FreshVotes={1}, MyVoteIndex = {2}, NumVotesForDeathDeclaration={3} , #activeSilos={4}, suspect list={5}",
                         entry.SiloAddress,
                         freshVotes.Count,
                         myVoteIndex,
                         this.clusterMembershipOptions.NumVotesForDeathDeclaration,
                         activeSilos,
                         PrintSuspectList(entry.SuspectTimes));
                return(await DeclareDead(entry, eTag, table.Version, now));
            }

            log.Info(ErrorCode.MembershipVotingForKill,
                     "-Putting my vote to mark silo {0} as DEAD #2. Previous suspect list is {1}, trying to update to {2}, eTag={3}, freshVotes is {4}",
                     entry.SiloAddress,
                     PrintSuspectList(prevList),
                     PrintSuspectList(entry.SuspectTimes),
                     eTag,
                     PrintSuspectList(freshVotes));

            // If we fail to update here we will retry later.
            var ok = await membershipTableProvider.UpdateRow(entry, eTag, table.Version.Next());

            if (ok)
            {
                table = await membershipTableProvider.ReadAll();

                this.ProcessTableUpdate(table, "TrySuspectOrKill");

                // Gossip using the local silo status, since this is just informational to propagate the suspicion vote.
                GossipToOthers(localSiloEntry.SiloAddress, localSiloEntry.Status).Ignore();
            }

            return(ok);

            string PrintSuspectList(IEnumerable <Tuple <SiloAddress, DateTime> > list)
            {
                return(Utils.EnumerableToString(list, t => string.Format("<{0}, {1}>",
                                                                         t.Item1, LogFormatter.PrintDate(t.Item2))));
            }
        }