public long[] GetAnalyticsStats(IServiceAddress server) { Message message = new Message("reportStats"); Message m = Command(server, ServiceType.Admin, message.AsStream()); if (m.HasError) { throw new NetworkAdminException(m.ErrorMessage); } return (long[]) m.Arguments[0].Value; }
private void InformRootServerOfManagers(IServiceAddress rootServer) { // Make the managers list List<IServiceAddress> managers = new List<IServiceAddress>(64); lock (managerServersList) { foreach (ManagerServiceInfo m in managerServersList) { managers.Add(m.Address); } } //TODO: verfy this ... // add the current manager address to the list managers.Add(address); IServiceAddress[] managersSet = managers.ToArray(); Message message = new Message("informOfManagers", new object[] {managersSet}); // Open a connection to the root server, IMessageProcessor processor = connector.Connect(rootServer, ServiceType.Root); IEnumerable<Message> response = processor.Process(message.AsStream()); foreach (Message m in response) { if (m.HasError) { // If we failed, log a severe error but don't stop trying to register Logger.Error("Couldn't inform root server of managers"); Logger.Error(m.ErrorStackTrace); if (ReplicatedValueStore.IsConnectionFault(m)) { serviceTracker.ReportServiceDownClientReport(rootServer, ServiceType.Root); } } } }
private void RegisterBlockServer(IServiceAddress blockServerAddress) { // Get the block server uid, Message message = new Message("serverGUID"); // Connect to the block server, IMessageProcessor processor = connector.Connect(blockServerAddress, ServiceType.Block); IEnumerable<Message> response = processor.Process(message.AsStream()); Message rm = null; foreach (Message m in response) { if (m.HasError) throw new ApplicationException(m.ErrorMessage); rm = m; } long serverGuid = (long) rm.Arguments[0].Value; // Add lookup for this server_guid <-> service address to the db, managerDb.SetValue("block.sguid." + serverGuid, blockServerAddress.ToString()); managerDb.SetValue("block.addr." + blockServerAddress, serverGuid.ToString()); // TODO: Block discovery on the introduced machine, // Set the status and guid BlockServiceInfo blockServer = new BlockServiceInfo(serverGuid, blockServerAddress); // Add it to the map lock (blockServersMap) { blockServersMap[serverGuid] = blockServer; blockServersList.Add(blockServer); PersistBlockServers(blockServersList); } }
private void SendProposalComplete(List<ServiceMessageQueue> pendingQueue, long[] uid, BlockId blockId, long[] blockServerUids) { List<IServiceAddress> machines = new List<IServiceAddress>(17); lock (cluster) { machines.AddRange(cluster); } // Create the message, Message message = new Message("internalBSComplete", uid, blockId, blockServerUids); // Send the complete proposal message out to the machines on the network, SendCommand(pendingQueue, machines, message.AsStream()); // Enqueue all pending messages, foreach (ServiceMessageQueue queue in pendingQueue) { queue.Enqueue(); } }
private void FetchNextBlock() { Message message = new Message("internalFetchLogBundle", firstUid, initial ? 1 : 0); // Clear the log entries, logEntries.Clear(); index = 0; // Send the open stream command. // If the service is up, if (valueStore.tracker.IsServiceUp(machine, ServiceType.Manager)) { // Send to the service, IMessageProcessor processor = valueStore.connector.Connect(machine, ServiceType.Manager); IEnumerable<Message> response = processor.Process(message.AsStream()); // If it's a connection error, return null, foreach (Message m in response) { if (m.HasError) { // Report the service down if connection failure if (IsConnectionFault(m)) { valueStore.tracker.ReportServiceDownClientReport(machine, ServiceType.Manager); } throw new ApplicationException(m.ErrorMessage); } else { long[] uid = (long[]) m.Arguments[0].Value; byte[] buf = (byte[]) m.Arguments[1].Value; logEntries.Add(new LogEntry(uid, buf)); } } } else { throw new ApplicationException("Service down"); } // Update the first uid of the next block, if (logEntries.Count > 0) { LogEntry lastEntry = logEntries[logEntries.Count - 1]; firstUid = lastEntry.Uid; } }
private void SendRootServer(IServiceAddress root, String functionName, params object[] args) { Message message = new Message(functionName, args); // Send the command to all the root servers, Message lastError = null; IMessageProcessor proc = Connector.Connect(root, ServiceType.Root); IEnumerable<Message> response = proc.Process(message.AsStream()); int successCount = 0; foreach (Message m in response) { if (m.HasError) { if (!IsConnectionFailure(m)) { throw new NetworkAdminException(m.ErrorMessage); } lastError = m; } else { ++successCount; } } // Any one root failed, if (successCount != 1) { throw new NetworkAdminException(lastError.ErrorMessage); } }
private void ReportBlockServerFailure(IServiceAddress address) { // Report the failure, log.Warning(String.Format("Reporting failure for {0} to manager server", address)); // Failure throttling, lock (failureFloodControl) { DateTime currentTime = DateTime.Now; DateTime lastAddressFailTime; if (failureFloodControl.TryGetValue(address, out lastAddressFailTime) && lastAddressFailTime.AddMilliseconds((30*1000)) > currentTime) { // We don't respond to failure notifications on the same address if a // failure notice arrived within a minute of the last one accepted. return; } failureFloodControl[address] = currentTime; } Message message = new Message("notifyBlockServerFailure", address); // Process the failure report message on the manager server, NotifyAllManagers(message.AsStream()); }
public String GetPathStats(PathInfo pathInfo) { InspectNetwork(); IServiceAddress rootLeader = pathInfo.RootLeader; // Check machine is in the schema, MachineProfile machineP = CheckMachineInNetwork(rootLeader); // Check it's root, if (!machineP.IsRoot) throw new NetworkAdminException("Machine '" + rootLeader + "' is not a root"); // Perform the command, Message message = new Message("getPathStats", pathInfo.PathName, pathInfo.VersionNumber); Message m = Command(rootLeader, ServiceType.Root, message.AsStream()); if (m.HasError) throw new NetworkAdminException(m.ErrorMessage); // Return the stats string for this path return (String) m.Arguments[0].Value; }
public void ProcessSendBlock(long blockId, IServiceAddress sourceBlockServer, IServiceAddress destBlockServer, long destServerSguid) { InspectNetwork(); // Get the current manager server, MachineProfile[] mans = GetManagerServers(); if (mans.Length == 0) { throw new NetworkAdminException("No manager server found"); } // Use the manager servers, IServiceAddress[] managerServers = new IServiceAddress[mans.Length]; for (int i = 0; i < mans.Length; ++i) { managerServers[i] = mans[i].ServiceAddress; } Message message = new Message("sendBlockTo", blockId, destBlockServer, destServerSguid, managerServers); Message m = Command(sourceBlockServer, ServiceType.Block, message.AsStream()); if (m.HasError) { throw new NetworkAdminException(m.ErrorMessage); } }
public IDictionary<IServiceAddress, ServiceStatus> GetBlocksStatus() { InspectNetwork(); // Get the current manager server, MachineProfile[] mans = GetManagerServers(); if (mans.Length == 0) throw new NetworkAdminException("No manager server found"); IServiceAddress managerServer = mans[0].ServiceAddress; Message message = new Message("getRegisteredServerList"); Message m = Command(managerServer, ServiceType.Manager, message.AsStream()); if (m.HasError) throw new NetworkAdminException(m.ErrorMessage); // The list of block servers registered with the manager, IServiceAddress[] regservers = (IServiceAddress[]) m.Arguments[0].Value; string[] regserversStatus = (string[]) m.Arguments[1].Value; Dictionary<IServiceAddress, ServiceStatus> map = new Dictionary<IServiceAddress, ServiceStatus>(); for (int i = 0; i < regservers.Length; ++i) { map.Add(regservers[i], (ServiceStatus) Enum.Parse(typeof (ServiceStatus), regserversStatus[i], true)); } // Return the map, return map; }
public DataAddress[] GetHistoricalPathRoots(IServiceAddress root, String pathName, DateTime timestamp, int maxCount) { InspectNetwork(); // Check machine is in the schema, MachineProfile machineP = CheckMachineInNetwork(root); // Check it's root, if (!machineP.IsRoot) throw new NetworkAdminException("Machine '" + root + "' is not a root"); // Perform the command, Message message = new Message("getPathHistorical", pathName, DateTimeUtil.GetMillis(timestamp), DateTimeUtil.GetMillis(timestamp)); Message m = Command(root, ServiceType.Root, message.AsStream()); if (m.HasError) { throw new NetworkAdminException(m.ErrorMessage); } // Return the data address array, return (DataAddress[]) m.Arguments[0].Value; }
public long[] GetBlockMappingRange(long p1, long p2) { InspectNetwork(); // Get the current manager server, MachineProfile[] mans = GetManagerServers(); if (mans.Length == 0) throw new NetworkAdminException("No manager server found"); IServiceAddress managerServer = mans[0].ServiceAddress; Message message = new Message("getBlockMappingRange", p1, p2); Message m = Command(managerServer, ServiceType.Manager, message.AsStream()); if (m.HasError) throw new NetworkAdminException(m.ErrorMessage); // Return the service address for the root server, return (long[]) m.Arguments[0].Value; }
public long[] GetBlockList(IServiceAddress block) { InspectNetwork(); // Check machine is in the schema, MachineProfile machineP = CheckMachineInNetwork(block); // Check it's a block server, if (!machineP.IsBlock) { throw new NetworkAdminException("Machine '" + block + "' is not a block role"); } Message message = new Message("blockSetReport"); Message m = Command(block, ServiceType.Block, message.AsStream()); if (m.HasError) { throw new NetworkAdminException(m.ErrorMessage); } // Return the block list, return (long[]) m.Arguments[1].Value; }
public long GetBlockGuid(IServiceAddress block) { InspectNetwork(); // Check machine is in the schema, MachineProfile machineP = CheckMachineInNetwork(block); // Check it's a block server, if (!machineP.IsBlock) throw new NetworkAdminException("Machine '" + block + "' is not a block role"); Message message = new Message("serverGUID", block); Message m = Command(block, ServiceType.Block, message.AsStream()); if (m.HasError) throw new NetworkAdminException(m.ErrorMessage); // Return the GUID return (long) m.Arguments[0].Value; }
public void AddBlockAssociation(long blockId, long serverGuid) { InspectNetwork(); // Get the current manager server, MachineProfile[] mans = GetManagerServers(); if (mans.Length == 0) throw new NetworkAdminException("No manager server found"); IServiceAddress[] managerServers = new IServiceAddress[mans.Length]; for (int i = 0; i < mans.Length; ++i) { managerServers[i] = mans[i].ServiceAddress; } // NOTE: This command will be propogated through all the other managers on // the network by the manager. Message message = new Message("internalAddBlockServerMapping", blockId, new long[] {serverGuid}); // Send the command to all the managers, if all fail throw an exception. bool success = false; Message lastError = null; for (int i = 0; i < managerServers.Length; ++i) { Message m = Command(managerServers[i], ServiceType.Manager, message.AsStream()); if (m.HasError) lastError = m; else success = true; } if (!success) { throw new NetworkAdminException(lastError.ErrorMessage); } }
public void RegisterManager(IServiceAddress manager) { InspectNetwork(); // Check machine is in the schema, MachineProfile machineP = CheckMachineInNetwork(manager); MachineProfile[] currentManagers = GetManagerServers(); if (currentManagers.Length == 0) throw new NetworkAdminException("No manager server found"); // Check it is a manager server, if (!machineP.IsManager) throw new NetworkAdminException("Machine '" + manager + "' is not assigned as a manager"); // The list of manager servers, IServiceAddress[] managerServers = new IServiceAddress[currentManagers.Length]; for (int i = 0; i < currentManagers.Length; ++i) { managerServers[i] = currentManagers[i].ServiceAddress; } Message message = new Message("registerManagerServers", new object[] {managerServers}); // Register the root server with all the managers currently on the network, for (int i = 0; i < currentManagers.Length; ++i) { Message m = Command(currentManagers[i].ServiceAddress, ServiceType.Manager, message.AsStream()); if (m.HasError) { throw new NetworkAdminException(m.ErrorMessage); } } }
private object SendManagerFunction(string functionName, params object[] args) { // Send the add path command to the first available manager server. MachineProfile[] managerServers = GetManagerServers(); Message message = new Message(functionName, args); // The first manager that takes the command, Object result = null; Message lastError = null; for (int i = 0; i < managerServers.Length && result == null; ++i) { IServiceAddress managerServer = managerServers[i].ServiceAddress; Message m = Command(managerServer, ServiceType.Manager, message.AsStream()); if (m.HasError) { if (!IsConnectionFailure(m)) { throw new NetworkAdminException(m.ErrorMessage); } lastError = m; } else { return m.Arguments[0].Value; } } // All managers failed, throw new NetworkAdminException(lastError.ErrorMessage); }
public void RegisterService(IServiceAddress address, ServiceType serviceType) { InspectNetwork(); // Check machine is in the schema, MachineProfile machineP = CheckMachineInNetwork(address); MachineProfile[] currentManagers = GetManagerServers(); if (currentManagers.Length == 0) throw new NetworkAdminException("No manager server found"); // Check it is a root server, if (!machineP.IsInRole(serviceType)) throw new NetworkAdminException("Machine '" + address + "' is assigned as a " + serviceType); string command = null; if (serviceType == ServiceType.Manager) { RegisterManager(address); } else if (serviceType == ServiceType.Root) { command = "registerRootServer"; } else if (serviceType == ServiceType.Block) { command = "registerBlockServer"; } else { throw new ArgumentException(); } Message message = new Message(command, address); // Register the root server with all the managers currently on the network, for (int i = 0; i < currentManagers.Length; ++i) { Message m = Command(currentManagers[i].ServiceAddress, ServiceType.Manager, message.AsStream()); if (m.HasError) throw new NetworkAdminException(m.ErrorMessage); } }
private void ReportBlockIdCorruption(IServiceAddress blockServer, BlockId blockId, String failType) { // Report the failure, log.Warning(String.Format("Reporting a data failure (type = {0}) for block {1} at block server {2}", failType, blockId, blockServer)); // Failure throttling, lock (failureFloodControlBidc) { DateTime currentTime = DateTime.Now; DateTime lastAddressFailTime; if (failureFloodControlBidc.TryGetValue(blockServer, out lastAddressFailTime) && lastAddressFailTime.AddMilliseconds((10*1000)) > currentTime) { // We don't respond to failure notifications on the same address if a // failure notice arrived within a minute of the last one accepted. return; } failureFloodControlBidc[blockServer] = currentTime; } Message message = new Message("notifyBlockIdCorruption", blockServer, blockId, failType); // Process the failure report message on the manager server, // (Ignore any error message generated) ProcessManager(message.AsStream()); }
public void SetPathRoot(IServiceAddress root, String pathName, DataAddress address) { InspectNetwork(); // Check machine is in the schema, MachineProfile machineP = CheckMachineInNetwork(root); // Check it's root, if (!machineP.IsRoot) throw new NetworkAdminException("Machine '" + root + "' is not a root"); // Perform the command, Message message = new Message("publishPath", pathName, address); Message m = Command(root, ServiceType.Root, message.AsStream()); if (m.HasError) { throw new NetworkAdminException(m.ErrorMessage); } }
private DataAddress[] InternalGetPathHistorical(PathInfo pathInfo, IServiceAddress server, long timeStart, long timeEnd) { Message message = new Message("getPathHistorical", pathInfo.PathName, pathInfo.VersionNumber, timeStart, timeEnd); Message m = ProcessSingleRoot(message.AsStream(), server); if (m.HasError) throw new ApplicationException(m.ErrorMessage); return (DataAddress[]) m.Arguments[0].Value; }
private void ChangeRole(MachineProfile machine, string status, ServiceType roleType) { Message message = new Message(status); message.Arguments.Add((byte)roleType); Message m = Command(machine.ServiceAddress, ServiceType.Admin, message.AsStream()); if (m.HasError) { throw new NetworkAdminException(m.ErrorMessage); } // Success, // Update the network profile, if (roleType == ServiceType.Manager && status.Equals("start")) { machine.Roles |= MachineRoles.Manager; } else if (roleType == ServiceType.Root && status.Equals("start")) { machine.Roles |= MachineRoles.Root; } else if (roleType == ServiceType.Block && status.Equals("start")) { machine.Roles |= MachineRoles.Block; } }
private int SendProposalToNetwork(List<ServiceMessageQueue> pendingQueue, long[] uid, BlockId blockId, long[] blockServerUids) { List<IServiceAddress> machines = new List<IServiceAddress>(17); lock (cluster) { machines.AddRange(cluster); } // Create the message, Message message = new Message("internalBSProposal", uid, blockId, blockServerUids); // Send the proposal command out to the machines on the network, int sendCount = SendCommand(pendingQueue, machines, message.AsStream()); // If we sent to a majority, return 1 if (sendCount > machines.Count/2) return 1; // Otherwise return 2, (majority of machines in the cluster not available). return 2; }
private List<MachineProfile> InspectNetwork() { // If cached, if (machineProfiles != null) { return machineProfiles; } // The sorted list of all servers in the schema, IEnumerable<IServiceAddress> slist = SortedServers; // The list of machine profiles, List<MachineProfile> machines = new List<MachineProfile>(); // For each machine in the network, foreach (IServiceAddress server in slist) { MachineProfile machineProfile = new MachineProfile(server); // Request a report from the administration role on the machine, IMessageProcessor mp = Connector.Connect(server, ServiceType.Admin); Message message = new Message("report"); IEnumerable<Message> response = mp.Process(message.AsStream()); Message lastM = null; foreach (Message m in response) { lastM = m; } if (lastM.HasError) { machineProfile.ErrorMessage = lastM.ErrorMessage; } else { // Get the message replies, MachineRoles roles = (MachineRoles)(byte) lastM.Arguments[0].Value; long usedMem = (long) lastM.Arguments[1].Value; long totalMem = (long) lastM.Arguments[2].Value; long usedDisk = (long) lastM.Arguments[3].Value; long totalDisk = (long) lastM.Arguments[4].Value; // Populate the lists, machineProfile.Roles = roles; machineProfile.MemoryUsed = usedMem; machineProfile.MemoryTotal = totalMem; machineProfile.DiskUsed = usedDisk; machineProfile.DiskTotal = totalDisk; } // Add the machine profile to the list machines.Add(machineProfile); } machineProfiles = machines; return machineProfiles; }
private void ClearRootServerOfManagers(IServiceAddress rootServer) { Message message = new Message("clearOfManagers"); // Open a connection to the root server, IMessageProcessor processor = connector.Connect(rootServer, ServiceType.Root); IEnumerable<Message> response = processor.Process(message.AsStream()); foreach (Message m in response) { if (m.HasError) { // If we failed, log a severe error but don't stop trying to register Logger.Error("Couldn't inform root server of managers"); Logger.Error(m.ErrorStackTrace); if (ReplicatedValueStore.IsConnectionFault(m)) { serviceTracker.ReportServiceDownClientReport(rootServer, ServiceType.Root); } } } }
private void SendAllRootServers(IServiceAddress[] roots, string functionName, params object[] args) { Message message = new Message(functionName, args); // Send the command to all the root servers, Message lastError = null; IEnumerable<Message>[] responses = new IEnumerable<Message>[roots.Length]; for (int i = 0; i < roots.Length; ++i) { IServiceAddress rootServer = roots[i]; IMessageProcessor proc = Connector.Connect(rootServer, ServiceType.Root); responses[i] = proc.Process(message.AsStream()); } int successCount = 0; foreach (MessageStream response in responses) { foreach (Message m in response) { if (m.HasError) { if (!IsConnectionFailure(m)) throw new NetworkAdminException(m.ErrorMessage); lastError = m; } else { ++successCount; } } } // Any one root failed, if (successCount != roots.Length) { throw new NetworkAdminException(lastError.ErrorMessage); } }
private void NotifyBlockServerOfMaxBlockId(IServiceAddress blockServer, BlockId blockId) { if (serviceTracker.IsServiceUp(blockServer, ServiceType.Block)) { Message message = new Message("notifyCurrentBlockId", blockId); // Connect to the block server, IMessageProcessor processor = connector.Connect(blockServer, ServiceType.Block); IEnumerable<Message> response = processor.Process(message.AsStream()); // If the block server is down, report it to the tracker, foreach (Message m in response) { if (m.HasError) { if (ReplicatedValueStore.IsConnectionFault(m)) { serviceTracker.ReportServiceDownClientReport(blockServer, ServiceType.Block); } } } } }
private void SendManagerCommand(string functionName, params object[] args) { // Send the add path command to the first available manager server. MachineProfile[] managerServers = GetManagerServers(); Message message = new Message(functionName, args); // The first manager that takes the command, bool success = false; Message lastError = null; for (int i = 0; i < managerServers.Length && success == false; ++i) { IServiceAddress managerServer = managerServers[i].ServiceAddress; Message m = Command(managerServer, ServiceType.Manager, message.AsStream()); if (m.HasError) { if (!IsConnectionFailure(m)) { throw new NetworkAdminException(m.ErrorMessage); } lastError = m; } else { success = true; } } // All managers failed, if (!success) { throw new NetworkAdminException(lastError.ErrorMessage); } }
private void RegisterManagerServers(IServiceAddress[] managerServerAddresses) { // Sanity check on number of manager servers (10 should be enough for // everyone !) if (managerServerAddresses.Length > 100) throw new ApplicationException("Number of manager servers > 100"); // Query all the manager servers on the network and generate a unique id // for this manager, if we need to create a new unique id, if (managerUniqueId == -1) { int sz = managerServerAddresses.Length; List<int> blacklistId = new List<int>(sz); for (int i = 0; i < sz; ++i) { IServiceAddress man = managerServerAddresses[i]; if (!man.Equals(address)) { // Open a connection with the manager server, IMessageProcessor processor = connector.Connect(man, ServiceType.Manager); // Query the unique id of the manager server, Message message = new Message("getUniqueId"); IEnumerable<Message> response = processor.Process(message.AsStream()); foreach (Message m in response) { if (m.HasError) throw new ApplicationException(m.ErrorMessage); long uniqueId = (long) m.Arguments[0].Value; if (uniqueId == -1) throw new ApplicationException("getUniqueId = -1"); // Add this to blacklist, blacklistId.Add((int) uniqueId); } } } // Find a random id not found in the blacklist, int genId; while (true) { genId = rng.Next(200); if (!blacklistId.Contains(genId)) { break; } } // Set the unique id, managerUniqueId = genId; } lock (managerServersList) { managerServersList.Clear(); managerDb.ClearAllMachines(); foreach (IServiceAddress m in managerServerAddresses) { if (!m.Equals(address)) { ManagerServiceInfo managerServer = new ManagerServiceInfo(m); managerServersList.Add(managerServer); // Add to the manager database managerDb.AddMachine(managerServer.Address); } } PersistManagerServers(managerServersList); PersistManagerUniqueId(managerUniqueId); } // Perform initialization on the manager managerDb.Initialize(); // Wait for initialization to complete, managerDb.WaitInitComplete(); // Add a manager server entry, foreach (IServiceAddress managerAddr in managerServerAddresses) { managerDb.SetValue("ms." + managerAddr, ""); } // Tell all the root servers of the new manager set, List<IServiceAddress> rootServersSet = new List<IServiceAddress>(64); lock (rootServersList) { foreach (RootServiceInfo rs in rootServersList) { rootServersSet.Add(rs.Address); } } foreach (IServiceAddress r in rootServersSet) { InformRootServerOfManagers(r); } }
public void DeregisterManager(IServiceAddress root) { InspectNetwork(); // Check machine is in the schema, MachineProfile machineP = CheckMachineInNetwork(root); MachineProfile[] currentManagers = GetManagerServers(); if (currentManagers.Length == 0) throw new NetworkAdminException("No manager server found"); // Check it is a manager server, if (!machineP.IsManager) throw new NetworkAdminException("Machine '" + root + "' is not assigned as a manager"); Message message = new Message("deregisterManagerServer", root); // Register the root server with all the managers currently on the network, for (int i = 0; i < currentManagers.Length; ++i) { Message m = Command(currentManagers[i].ServiceAddress, ServiceType.Manager, message.AsStream()); if (m.HasError) throw new NetworkAdminException(m.ErrorMessage); } }