Exemplo n.º 1
0
        public DataAddress Commit(string pathName, DataAddress proposal)
        {
            CheckConnected();

            IServiceAddress rootAddress = storageSystem.GetRootServer(pathName);
            return storageSystem.Commit(rootAddress, pathName, proposal);
        }
Exemplo n.º 2
0
        public void PublishPath(IServiceAddress root, string pathName, DataAddress address)
        {
            InspectNetwork();

            // Check machine is in the schema,
            MachineProfile machine = CheckMachineInNetwork(root);

            // Check it's root,
            if (!machine.IsRoot)
            {
                throw new NetworkAdminException("Machine '" + root + "' is not a root");
            }

            // Perform the command,
            RequestMessage request = new RequestMessage("publishPath");

            request.Arguments.Add(pathName);
            request.Arguments.Add(address);

            Message m = Command(root, ServiceType.Root, request);

            if (m.HasError)
            {
                throw new NetworkAdminException(m.ErrorMessage);
            }
        }
Exemplo n.º 3
0
        public DataAddress Max(DataAddress address)
        {
            if (Value.CompareTo(address.Value) >= 0)
            {
                return(this);
            }

            return(address);
        }
Exemplo n.º 4
0
        public override bool Equals(object obj)
        {
            DataAddress other = obj as DataAddress;

            if (other == null)
            {
                return(false);
            }

            return(Value.Equals(other.Value));
        }
Exemplo n.º 5
0
        private void AddPath(string pathName, string pathTypeName, DataAddress rootNode)
        {
            // Check the name given is valid,
            CheckPathNameValid(pathName);

            lock (pathCache) {
                if (pathCache.ContainsKey(pathName))
                    // If it's input the local map, generate an error,
                    throw new ApplicationException("Path '" + pathName + "' already exists.");

                CreatePath(pathName, pathTypeName);
            }

            if (rootNode != null) {
                // Finally publish the base_root to the path,
                PublishPath(pathName, rootNode);
            }
        }
Exemplo n.º 6
0
        public void SetPathRoot(IServiceAddress root, String pathName, DataAddress address)
        {
            InspectNetwork();

            // Check machine is in the schema,
            MachineProfile machineP = CheckMachineInNetwork(root);

            // Check it's root,
            if (!machineP.IsRoot)
            {
                throw new NetworkAdminException("Machine '" + root + "' is not a root");
            }

            // Perform the command,
            Message message = new Message("publishPath", pathName, address);

            Message m = Command(root, ServiceType.Root, message.AsStream());

            if (m.HasError)
            {
                throw new NetworkAdminException(m.ErrorMessage);
            }
        }
Exemplo n.º 7
0
            public object[] FetchPathDataBundle(long fromUid, DataAddress fromAddr, int bundleSize)
            {
                List<PathRecordEntry> entriesList = new List<PathRecordEntry>(bundleSize);

                // Synchronize over the object
                lock (accessLock) {
                    // Check near the end (most likely to be found there),

                    long setSize = internalFile.Length/RootItemSize;

                    long searchS;
                    long found = -1;
                    long pos;
                    if (fromUid > 0) {
                        // If from_uid is real,
                        pos = BinarySearch(pagedFile, 0, setSize - 1, fromUid);
                        pos = Math.Max(0, pos - 256);

                        searchS = pos;

                        // Search to the end,
                        while (true) {
                            // End condition,
                            if (pos >= setSize) {
                                break;
                            }

                            PathRecordEntry entry = GetEntryAt(pos*RootItemSize);
                            if (entry.Uid == fromUid && entry.Address.Equals(fromAddr)) {
                                // Found!
                                found = pos;
                                break;
                            }
                            ++pos;
                        }

                    } else {
                        // If from_uid is less than 0, it indicates to fetch the bundle of
                        // path entries from the start.
                        pos = -1;
                        found = 0;
                        searchS = 0;
                    }

                    // If not found,
                    if (found < 0) {
                        // Try from search_s to 0
                        pos = searchS - 1;
                        while (true) {
                            // End condition,
                            if (pos < 0) {
                                break;
                            }

                            PathRecordEntry entry = GetEntryAt(pos*RootItemSize);
                            if (entry.Uid == fromUid && entry.Address.Equals(fromAddr)) {
                                // Found!
                                found = pos;
                                break;
                            }
                            --pos;
                        }
                    }

                    // Go to the next entry,
                    ++pos;

                    // Still not found, or at the end
                    if (found < 0 || pos >= setSize) {
                        return new Object[] {new long[0], new DataAddress[0]};
                    }

                    // Fetch a bundle of records from the position
                    while (true) {
                        // End condition,
                        if (pos >= setSize || entriesList.Count >= bundleSize) {
                            break;
                        }

                        PathRecordEntry entry = GetEntryAt(pos*RootItemSize);
                        entriesList.Add(entry);

                        ++pos;
                    }
                }

                // Format it as long[] and DataAddress[] arrays
                int sz = entriesList.Count;
                long[] uids = new long[sz];
                DataAddress[] addrs = new DataAddress[sz];
                for (int i = 0; i < sz; ++i) {
                    uids[i] = entriesList[i].Uid;
                    addrs[i] = entriesList[i].Address;
                }
                return new Object[] {uids, addrs};
            }
Exemplo n.º 8
0
 public PathRecordEntry(long uid, DataAddress address)
 {
     Uid = uid;
     Address = address;
 }
Exemplo n.º 9
0
        private DataAddress PerformCommit(PathInfo pathInfo, DataAddress proposal)
        {
            IServiceAddress[] manSrvs = managerServers;

            // Fetch the path access object for the given name.
            PathAccess pathFile = GetPathAccess(pathInfo.PathName);

            IPath pathFunction;
            try {
                pathFunction = pathFile.Path;
            } catch (TypeLoadException e) {
                throw new CommitFaultException(String.Format("Type not found: {0}", e.Message));
            } catch (TypeInitializationException e) {
                throw new CommitFaultException(String.Format("Type instantiation exception: {0}", e.Message));
            } catch (AccessViolationException e) {
                throw new CommitFaultException(String.Format("Illegal Access exception: {0}", e.Message));
            }

            // Create the connection object (should be fairly lightweight)
            INetworkCache localNetCache = MachineState.GetCacheForManager(manSrvs);
            IPathConnection connection = new PathConnection(this, pathInfo, connector, manSrvs, localNetCache, serviceTracker);

            // Perform the commit,
            return pathFunction.Commit(connection, proposal);
        }
Exemplo n.º 10
0
        private void NotifyAllRootServersOfPost(PathInfo pathInfo, long uid, DataAddress rootNode)
        {
            // The root servers for the path,
            IServiceAddress[] roots = pathInfo.RootServers;

            // Create the message,
            MessageStream outputStream = new MessageStream();
            outputStream.AddMessage(new Message("notifyNewProposal", pathInfo.PathName, uid, rootNode));

            for (int i = 0; i < roots.Length; ++i) {
                IServiceAddress machine = roots[i];
                // Don't notify this service,
                if (!machine.Equals(address)) {
                    // If the service is up in the tracker,
                    if (serviceTracker.IsServiceUp(machine, ServiceType.Root)) {
                        // Send the message to the service,
                        IMessageProcessor processor = connector.Connect(machine, ServiceType.Root);
                        IEnumerable<Message> inputStream = processor.Process(outputStream);
                        // If return is a connection fault,
                        foreach (Message m in inputStream) {
                            if (m.HasError &&
                                ReplicatedValueStore.IsConnectionFault(m)) {
                                serviceTracker.ReportServiceDownClientReport(machine, ServiceType.Root);
                            }
                        }
                    }
                }
            }
        }
Exemplo n.º 11
0
 public DataAddress Commit(string pathName, DataAddress proposal)
 {
     return(treeSystem.PerformCommit(pathName, proposal));
 }
Exemplo n.º 12
0
        public DataAddress Commit(IServiceAddress root_server, String path_name, DataAddress proposal)
        {
            IMessageProcessor processor = connector.Connect(root_server, ServiceType.Root);
            RequestMessage request = new RequestMessage("commit");
            request.Arguments.Add(path_name);
            request.Arguments.Add(proposal);
            Message response = processor.Process(request);

            if (response.HasError) {
                MessageError et = response.Error;
                // If it's a commit fault exception, wrap it locally.
                if (et.Source.Equals("Deveel.Data.Net.CommitFaultException"))
                    throw new CommitFaultException(et.Message);

                throw new Exception(et.Message);
            }

            // Return the DataAddress of the result transaction,
            return (DataAddress) response.Arguments[0].Value;
        }
Exemplo n.º 13
0
 public ITransaction CreateTransaction(DataAddress rootNode)
 {
     return new Transaction(this, 0, rootNode);
 }
Exemplo n.º 14
0
 public void SetNode(DataAddress address, ITreeNode node)
 {
     lock (heapCache) {
         heapCache.Set(address, node);
     }
 }
Exemplo n.º 15
0
        public NodeSet GetNodeSet(int dataId)
        {
            if (dataId < 0 || dataId >= 16384)
            {
                throw new ArgumentException("dataId out of range");
            }

            try {
                int dataPointer  = dataId;
                int pos          = dataPointer * 6;
                int dataIdPos    = pagedAccess.ReadInt32(pos);
                int dataIdLength = ((int)pagedAccess.ReadInt16(pos + 4)) & 0x0FFFF;

                if (dataIdPos < 0)
                {
                    dataPointer  = -(dataIdPos + 1);
                    pos          = dataPointer * 6;
                    dataIdPos    = pagedAccess.ReadInt32(pos);
                    dataIdLength = ((int)pagedAccess.ReadInt16(pos + 4)) & 0x0FFFF;
                }

                // Fetch the node set,
                List <int> nodeIds = new List <int>(24);
                nodeIds.Add(dataPointer);
                while (true)
                {
                    ++dataPointer;
                    pos += 6;
                    int check_v = pagedAccess.ReadInt32(pos);
                    if (check_v < 0)
                    {
                        nodeIds.Add(dataPointer);
                    }
                    else
                    {
                        break;
                    }
                }

                // Turn it into a node array,
                int      sz       = nodeIds.Count;
                NodeId[] lnodeIds = new NodeId[sz];
                for (int i = 0; i < sz; ++i)
                {
                    DataAddress daddr = new DataAddress(blockId, nodeIds[i]);
                    lnodeIds[i] = daddr.Value;
                }

                // Read the encoded form into a byte[] array,
                byte[] buf = new byte[dataIdLength];
                content.Seek(dataIdPos, SeekOrigin.Begin);
                content.Read(buf, 0, dataIdLength);

                // Return it,
                return(new CompressedNodeSet(lnodeIds, buf));
            } catch (IOException e) {
                // We wrap this IOException around a BlockReadException. This can only
                // indicate a corrupt compressed block file or access to a dataId that
                // is out of range of the nodes stored in this file.
                throw new BlockReadException("IOError reading data from block file", e);
            }
        }
Exemplo n.º 16
0
 public ITransaction CreateTransaction(DataAddress rootNode)
 {
     // Create the transaction object and return it,
     return treeSystem.CreateTransaction(rootNode);
 }
Exemplo n.º 17
0
 public InvalidDataState(string message, DataAddress address)
     : base(message)
 {
     this.address = address;
 }
Exemplo n.º 18
0
 public ITreeNode GetNode(DataAddress address)
 {
     lock (heapCache) {
         return(heapCache.Get(address) as ITreeNode);
     }
 }
Exemplo n.º 19
0
 public bool IsNodeAvailable(long node_ref)
 {
     // Special node ref,
     if ((node_ref & 0x01000000000000000L) != 0) {
         return true;
     }
     // Check if it's in the local network cache
     DataAddress address = new DataAddress(node_ref);
     return (networkCache.GetNode(address) != null);
 }
Exemplo n.º 20
0
 public void RemoveNode(DataAddress address)
 {
     lock (heapCache) {
         heapCache.Remove(address);
     }
 }
Exemplo n.º 21
0
        private void CheckAndFixAllocationServers()
        {
            // If the failure report is on a block service that is servicing allocation
            // requests, we push the allocation requests to the next block.
            long currentBlockId;
            lock (allocationLock) {
                currentBlockId = currentAddressSpaceEnd.BlockId;
            }

            long[] bservers = GetOnlineServersWithBlock(currentBlockId);

            int okServerCount = 0;

            // Change the status of the block service to STATUS_DOWN_CLIENT_REPORT
            lock (blockServersMap) {
                // For each service that stores the block,
                for (int i = 0; i < bservers.Length; ++i) {
                    long serverGuid = bservers[i];
                    // Is the status of this service UP?
                    foreach (BlockServerInfo block_server in blockServers) {
                        // If this matches the guid, and is up, we add to 'ok_server_count'
                        if (block_server.Guid == serverGuid &&
                            block_server.Status == ServiceStatus.Up) {
                            ++okServerCount;
                        }
                    }
                }
            }

            // If the count of ok servers for the allocation set size is not
            // the same then there are one or more servers that are inoperable
            // in the allocation set. So, we increment the block id ref of
            // 'current address space end' by 1 to force a reevaluation of the
            // servers to allocate the current block.
            if (okServerCount != bservers.Length) {
                lock (allocationLock) {
                    long blockId = currentAddressSpaceEnd.BlockId;
                    int dataId = currentAddressSpaceEnd.DataId;
                    if (currentBlockId == blockId) {
                        ++blockId;
                        dataId = 0;
                        currentAddressSpaceEnd = new DataAddress(blockId, dataId);
                    }
                }
            }
        }
Exemplo n.º 22
0
            public IEnumerable <Message> Process(IEnumerable <Message> stream)
            {
                // The map of containers touched,
                Dictionary <BlockId, BlockContainer> containersTouched = new Dictionary <BlockId, BlockContainer>();
                // The reply message,
                MessageStream replyMessage = new MessageStream();
                // The nodes fetched in this message,
                List <NodeId> readNodes = null;

                foreach (Message m in stream)
                {
                    try {
                        // Check for stop state,
                        service.CheckErrorState();

                        // writeToBlock(DataAddress, byte[], int, int)
                        if (m.Name.Equals("writeToBlock"))
                        {
                            WriteToBlock(containersTouched,
                                         (DataAddress)m.Arguments[0].Value,
                                         (byte[])m.Arguments[1].Value, (int)m.Arguments[2].Value,
                                         (int)m.Arguments[3].Value);
                            replyMessage.AddMessage(new Message(1L));
                        }
                        // readFromBlock(DataAddress)
                        else if (m.Name.Equals("readFromBlock"))
                        {
                            if (readNodes == null)
                            {
                                readNodes = new List <NodeId>();
                            }
                            DataAddress addr = (DataAddress)m.Arguments[0].Value;
                            if (!readNodes.Contains(addr.Value))
                            {
                                NodeSet nodeSet = ReadFromBlock(containersTouched, addr);
                                replyMessage.AddMessage(new Message(nodeSet));
                                readNodes.AddRange(nodeSet.NodeIds);
                            }
                        }
                        // rollbackNodes(DataAddress[] )
                        else if (m.Name.Equals("rollbackNodes"))
                        {
                            RemoveNodes(containersTouched, (DataAddress[])m.Arguments[0].Value);
                            replyMessage.AddMessage(new Message(1L));
                        }
                        // deleteBlock(BlockId)
                        else if (m.Name.Equals("deleteBlock"))
                        {
                            DeleteBlock((BlockId)m.Arguments[0].Value);
                            replyMessage.AddMessage(new Message(1L));
                        }
                        // serverGUID()
                        else if (m.Name.Equals("serverGUID"))
                        {
                            replyMessage.AddMessage(new Message(service.Id));
                        }
                        // blockSetReport()
                        else if (m.Name.Equals("blockSetReport"))
                        {
                            BlockId[] arr = BlockSetReport();
                            replyMessage.AddMessage(new Message(service.Id, arr));
                        }
                        // poll(String)
                        else if (m.Name.Equals("poll"))
                        {
                            replyMessage.AddMessage(new Message(1L));
                        }

                        // notifyCurrentBlockId(BlockId)
                        else if (m.Name.Equals("notifyCurrentBlockId"))
                        {
                            service.NotifyCurrentBlockId((BlockId)m.Arguments[0].Value);
                            replyMessage.AddMessage(new Message(1L));
                        }

                        // blockChecksum(BlockId)
                        else if (m.Name.Equals("blockChecksum"))
                        {
                            long checksum = BlockChecksum(containersTouched, (BlockId)m.Arguments[0].Value);
                            replyMessage.AddMessage(new Message(checksum));
                        }
                        // sendBlockTo(BlockId, IServiceAddress, long, IServerAddress[])
                        else if (m.Name.Equals("sendBlockTo"))
                        {
                            // Returns immediately. There's currently no way to determine
                            // when this process will happen or if it will happen.
                            BlockId           blockId        = (BlockId)m.Arguments[0].Value;
                            IServiceAddress   destAddress    = (IServiceAddress)m.Arguments[1].Value;
                            long              destServerId   = (long)m.Arguments[2].Value;
                            IServiceAddress[] managerServers = (IServiceAddress[])m.Arguments[3].Value;
                            long              processId      = SendBlockTo(blockId, destAddress, destServerId, managerServers);
                            replyMessage.AddMessage(new Message(processId));
                        }
                        // sendBlockPart(BlockId, long, int, byte[], int)
                        else if (m.Name.Equals("sendBlockPart"))
                        {
                            BlockId blockId  = (BlockId)m.Arguments[0].Value;
                            long    pos      = (long)m.Arguments[1].Value;
                            int     fileType = (int)m.Arguments[2].Value;
                            byte[]  buf      = (byte[])m.Arguments[3].Value;
                            int     bufSize  = (int)m.Arguments[4].Value;
                            service.WriteBlockPart(blockId, pos, fileType, buf, bufSize);
                            replyMessage.AddMessage(new Message(1L));
                        }
                        // sendBlockComplete(BlockId, int)
                        else if (m.Name.Equals("sendBlockComplete"))
                        {
                            BlockId blockId  = (BlockId)m.Arguments[0].Value;
                            int     fileType = (int)m.Arguments[1].Value;
                            service.WriteBlockComplete(blockId, fileType);
                            replyMessage.AddMessage(new Message(1L));
                        }

                        // createAvailabilityMapForBlocks(BlockId[])
                        else if (m.Name.Equals("createAvailabilityMapForBlocks"))
                        {
                            BlockId[] blockIds = (BlockId[])m.Arguments[0].Value;
                            byte[]    map      = service.CreateAvailabilityMapForBlocks(blockIds);
                            replyMessage.AddMessage(new Message(map));
                        }

                        // bindWithManager()
                        else if (m.Name.Equals("bindWithManager"))
                        {
                            BindWithManager();
                            replyMessage.AddMessage(new Message(1L));
                        }
                        // unbindWithManager()
                        else if (m.Name.Equals("unbindWithManager"))
                        {
                            UnbindWithManager();
                            replyMessage.AddMessage(new Message(1L));
                        }
                        else
                        {
                            throw new ApplicationException("Unknown command: " + m.Name);
                        }
                    } catch (OutOfMemoryException e) {
                        service.Logger.Error("Memory Error", e);
                        service.SetErrorState(e);
                        throw;
                    } catch (Exception e) {
                        service.Logger.Error("Exception during process", e);
                        replyMessage.AddMessage(new Message(new MessageError(e)));
                    }
                }

                // Release any containers touched,
                try {
                    CloseContainers(containersTouched);
                } catch (IOException e) {
                    service.Logger.Error("IOError when closing containers", e);
                }

                return(replyMessage);
            }
Exemplo n.º 23
0
        private object[] InternalFetchPathDataBundle(string pathName, long uid, DataAddress addr)
        {
            // Fetch the path access object for the given name.
            PathAccess pathFile = GetPathAccess(pathName);
            // Init the local data if we need to,
            pathFile.OpenLocalData();

            // Fetch 256 entries,
            return pathFile.FetchPathDataBundle(uid, addr, 256);
        }
Exemplo n.º 24
0
            private NodeSet ReadFromBlock(Dictionary <BlockId, BlockContainer> containersTouched, DataAddress address)
            {
                // The block being written to,
                BlockId blockId = address.BlockId;
                // The data identifier,
                int dataId = address.DataId;

                // Fetch the block container,
                BlockContainer container = GetBlock(containersTouched, blockId);

                // Read the data,
                return(container.Read(dataId));
            }
Exemplo n.º 25
0
        private void NotifyNewProposal(String pathName, long uid, DataAddress node)
        {
            // Fetch the path access object for the given name.
            PathAccess pathFile = GetPathAccess(pathName);

            // Go tell the PathAccess
            pathFile.NotifyNewProposal(uid, node);
        }
Exemplo n.º 26
0
            private void WriteToBlock(Dictionary <BlockId, BlockContainer> containersTouched, DataAddress address, byte[] buffer,
                                      int offset, int length)
            {
                // The block being written to,
                BlockId blockId = address.BlockId;
                // The data identifier,
                int dataId = address.DataId;

                // Fetch the block container,
                BlockContainer container = GetBlock(containersTouched, blockId);

                // Write the data,
                container.Write(dataId, buffer, offset, length);

                // Schedule the block to be file synch'd 5 seconds after a write
                service.ScheduleFileFlush(container, 5000);
            }
Exemplo n.º 27
0
        private void PostToPath(PathInfo pathInfo, DataAddress rootNode)
        {
            // We can't post if this service is not the root leader,
            if (!pathInfo.RootLeader.Equals(address)) {
                Logger.Error(String.Format("Failed, {0} is not root leader for {1}", address, pathInfo.PathName));
                throw new ApplicationException("Can't post update, this root service (" + address +
                                               ") is not the root leader for the path: " + pathInfo.PathName);
            }

            // Fetch the path access object for the given name.
            PathAccess pathFile = GetPathAccess(pathInfo.PathName);

            // Only allow post if complete and synchronized
            pathFile.CheckIsSynchronized();

            // Create a unique time based uid.
            long uid = CreateUID();

            // Post the data address to the path,
            pathFile.PostProposalToPath(uid, rootNode);

            // Notify all the root servers of this post,
            NotifyAllRootServersOfPost(pathInfo, uid, rootNode);
        }
Exemplo n.º 28
0
        public void AddPath(IServiceAddress root, string pathName, string pathType)
        {
            InspectNetwork();

            // Check machine is in the schema,
            MachineProfile machineProfile = CheckMachineInNetwork(root);

            if (!machineProfile.IsRoot)
            {
                throw new NetworkAdminException("Machine '" + root + "' is not a root");
            }

            // Get the current manager server,
            MachineProfile man = ManagerServer;

            if (man == null)
            {
                throw new NetworkAdminException("No manager server found");
            }

            // Check with the root server that the class instantiates,
            Message outputStream = new RequestMessage("checkPathType");

            outputStream.Arguments.Add(pathType);

            Message m = Command(root, ServiceType.Root, outputStream);

            if (m.HasError)
            {
                throw new NetworkAdminException("Type '" + pathType + "' doesn't instantiate on the root");
            }

            IServiceAddress managerServer = man.Address;

            // Create a new empty database,
            NetworkClient client = new NetworkClient(managerServer, connector);

            client.Connect();
            DataAddress dataAddress = client.CreateEmptyDatabase();

            client.Disconnect();

            // Perform the command,
            outputStream = new MessageStream(MessageType.Request);
            RequestMessage request = new RequestMessage("addPath");

            request.Arguments.Add(pathName);
            request.Arguments.Add(pathType);
            request.Arguments.Add(dataAddress);
            ((MessageStream)outputStream).AddMessage(request);

            request = new RequestMessage("initPath");
            request.Arguments.Add(pathName);
            ((MessageStream)outputStream).AddMessage(request);

            Message message = Command(root, ServiceType.Root, outputStream);

            if (message.HasError)
            {
                throw new NetworkAdminException(message.ErrorMessage);
            }

            // Tell the manager server about this path,
            outputStream = new RequestMessage("addPathRootMapping");
            outputStream.Arguments.Add(pathName);
            outputStream.Arguments.Add(root);

            message = Command(managerServer, ServiceType.Manager, outputStream);
            if (message.HasError)
            {
                throw new NetworkAdminException(message.ErrorMessage);
            }
        }
Exemplo n.º 29
0
 public void AddPathDataEntries(long[] uids, DataAddress[] addrs)
 {
     lock (accessLock) {
         int sz = uids.Length;
         for (int i = 0; i < sz; ++i) {
             long uid = uids[i];
             DataAddress addr = addrs[i];
             // Post the proposal if it's not present
             PostProposalIfNotPresent(uid, addr);
         }
     }
 }
Exemplo n.º 30
0
 public ITransaction CreateTransaction(DataAddress rootNode)
 {
     return treeSystem.CreateTransaction(rootNode);
 }
Exemplo n.º 31
0
 internal DbRootAddress(DbSession session, DataAddress address)
 {
     this.session = session;
     this.address = address;
 }
Exemplo n.º 32
0
 public DataAddress[] GetSnapshots(DataAddress rootNode)
 {
     try {
         return service.GetPathRootsSince(pathInfo, rootNode);
     } catch (IOException e) {
         throw HandleIOError(e);
     }
 }
Exemplo n.º 33
0
 public DataAddress Commit(string pathName, DataAddress proposal)
 {
     return treeSystem.PerformCommit(pathName, proposal);
 }
Exemplo n.º 34
0
 public void Publish(DataAddress rootNode)
 {
     try {
         service.PostToPath(pathInfo, rootNode);
     } catch (IOException e) {
         throw HandleIOError(e);
     }
 }
Exemplo n.º 35
0
 internal Transaction(ITreeSystem storeSystem, long versionId, DataAddress rootNode)
     : base(storeSystem, versionId, rootNode.Value, false)
 {
 }
Exemplo n.º 36
0
            private DataAddress Commit(string pathName, int pathInfoVersion, DataAddress proposal)
            {
                // Find the PathInfo object from the path_info_version. If the path
                // version is out of date then an exception is generated.
                PathInfo pathInfo = service.GetPathInfo(pathName, pathInfoVersion);

                return service.PerformCommit(pathInfo, proposal);
            }
Exemplo n.º 37
0
        public IList<ITreeNode> FetchNodes(long[] nids)
        {
            // The number of nodes,
            int node_count = nids.Length;
            // The array of read nodes,
            ITreeNode[] result_nodes = new ITreeNode[node_count];

            // Resolve special nodes first,
            {
                int i = 0;
                foreach (long nodeId in nids) {
                    if ((nodeId & 0x01000000000000000L) != 0)
                        result_nodes[i] = SparseLeafNode.Create(nodeId);

                    ++i;
                }
            }

            // Group all the nodes to the same block,
            List<long> uniqueBlocks = new List<long>();
            List<List<long>> uniqueBlockList = new List<List<long>>();
            {
                int i = 0;
                foreach (long node_ref in nids) {
                    // If it's not a special node,
                    if ((node_ref & 0x01000000000000000L) == 0) {
                        // Get the block id and add it to the list of unique blocks,
                        DataAddress address = new DataAddress(node_ref);
                        // Check if the node is in the local cache,
                        ITreeNode node = networkCache.GetNode(address);
                        if (node != null) {
                            result_nodes[i] = node;
                        } else {
                            // Not in the local cache so we need to bundle this up in a node
                            // request on the block servers,
                            // Group this node request by the block identifier
                            long blockId = address.BlockId;
                            int ind = uniqueBlocks.IndexOf(blockId);
                            if (ind == -1) {
                                ind = uniqueBlocks.Count;
                                uniqueBlocks.Add(blockId);
                                uniqueBlockList.Add(new List<long>());
                            }
                            List<long> blist = uniqueBlockList[ind];
                            blist.Add(node_ref);
                        }
                    }
                    ++i;
                }
            }

            // Exit early if no blocks,
            if (uniqueBlocks.Count == 0)
                return result_nodes;

            // Resolve server records for the given block identifiers,
            IDictionary<long, IList<BlockServerElement>> servers_map = GetServersForBlock(uniqueBlocks);

            // The result nodes list,
            List<ITreeNode> nodes = new List<ITreeNode>();

            // For each unique block list,
            foreach (List<long> blist in uniqueBlockList) {
                // Make a block server request for each node in the block,
                MessageStream block_server_msg = new MessageStream(MessageType.Request);
                long block_id = -1;
                foreach (long node_ref in blist) {
                    DataAddress address = new DataAddress(node_ref);
                    RequestMessage request = new RequestMessage("readFromBlock");
                    request.Arguments.Add(address);
                    block_server_msg.AddMessage(request);
                    block_id = address.BlockId;
                }

                if (block_id == -1)
                    throw new ApplicationException("block_id == -1");

                // Get the shuffled list of servers the block is stored on,
                IList<BlockServerElement> servers = servers_map[block_id];

                // Go through the servers one at a time to fetch the block,
                bool success = false;
                for (int z = 0; z < servers.Count && !success; ++z) {
                    BlockServerElement server = servers[z];

                    // If the server is up,
                    if (server.IsStatusUp) {
                        // Open a connection with the block server,
                        IMessageProcessor block_server_proc = connector.Connect(server.Address, ServiceType.Block);
                        MessageStream message_in = (MessageStream) block_server_proc.Process(block_server_msg);
                        // DEBUG: ++networkCommCount;
                        // DEBUG: ++networkFetchCommCount;

                        bool is_error = false;
                        bool severe_error = false;
                        // Turn each none-error message into a node
                        foreach (ResponseMessage m in message_in) {
                            if (m.HasError) {
                                // See if this error is a block read error. If it is, we don't
                                // tell the manager server to lock this server out completely.
                                bool is_block_read_error = m.Error.Source.Equals("Deveel.Data.Net.BlockReadException");
                                if (!is_block_read_error) {
                                    // If it's something other than a block read error, we mark
                                    // this error as severe,
                                    severe_error = true;
                                }
                                is_error = true;
                            } else if (!is_error) {
                                // The reply contains the block of data read.
                                NodeSet node_set = (NodeSet)m.Arguments[0].Value;

                                // Decode the node items into node objects,
                                IEnumerator<Node> item_iterator = node_set.GetEnumerator();

                                while (item_iterator.MoveNext()) {
                                    // Get the node item,
                                    Node node_item = item_iterator.Current;

                                    long node_ref = node_item.Id;

                                    DataAddress address = new DataAddress(node_ref);
                                    // Wrap around a buffered DataInputStream for reading values
                                    // from the store.
                                    BinaryReader input = new BinaryReader(node_item.Input, Encoding.Unicode);
                                    short node_type = input.ReadInt16();

                                    ITreeNode read_node;

                                    // Is the node type a leaf node?
                                    if (node_type == LeafType) {
                                        // Read the key
                                        int leaf_size = input.ReadInt32();

                                        byte[] buf = ReadNodeAsBuffer(node_item);
                                        if (buf == null) {
                                            buf = new byte[leaf_size + 6];
                                            input.Read(buf, 6, leaf_size);
                                            // Technically, we could comment these next two lines out.
                                            ByteBuffer.WriteInt2(node_type, buf, 0);
                                            ByteBuffer.WriteInt4(leaf_size, buf, 2);
                                        }

                                        // Create a leaf that's mapped to this data
                                        read_node = new ByteArrayTreeLeaf(node_ref, buf); ;

                                    }
                                        // Is the node type a branch node?
                                    else if (node_type == BranchType) {
                                        // Note that the entire branch is loaded into memory,
                                        int child_data_size = input.ReadInt32();
                                        long[] data_arr = new long[child_data_size];
                                        for (int n = 0; n < child_data_size; ++n) {
                                            data_arr[n] = input.ReadInt64();
                                        }
                                        // Create the branch node,
                                        read_node = new TreeBranch(node_ref, data_arr, child_data_size);
                                    } else {
                                        throw new InvalidDataState("Unknown node type: " + node_type, address);
                                    }

                                    // Is the node already in the list? If so we don't add it.
                                    if (!IsInNodeList(node_ref, nodes)) {
                                        // Put the read node in the cache and add it to the 'nodes'
                                        // list.
                                        networkCache.SetNode(address, read_node);
                                        nodes.Add(read_node);
                                    }
                                }
                            }
                        }

                        // If there was no error while reading the result, we assume the node
                        // requests were successfully read.
                        if (is_error == false) {
                            success = true;
                        } else {
                            if (severe_error) {
                                // If this is an error, we need to report the failure to the
                                // manager server,
                                ReportBlockServerFailure(server.Address);
                                // Remove the block id from the server list cache,
                                networkCache.RemoveServers(block_id);
                            } else {
                                // Otherwise, not a severe error (probably a corrupt block on a
                                // server), so shuffle the server list for this block_id so next
                                // time there's less chance of hitting this bad block.
                                IList<BlockServerElement> srvs = networkCache.GetServers(block_id);
                                List<BlockServerElement> server_list = new List<BlockServerElement>();
                                server_list.AddRange(srvs);
                                CollectionsUtil.Shuffle(server_list);
                                networkCache.SetServers(block_id, server_list, 15 * 60 * 1000);
                            }
                        }

                    }
                }

                // If the nodes were not successfully read, we generate an exception,
                if (!success) {
                    // Remove from the cache,
                    networkCache.RemoveServers(block_id);
                    throw new ApplicationException("Unable to fetch node from block server");
                }
            }

            int sz = nodes.Count;
            if (sz == 0)
                throw new ApplicationException("Empty nodes list");

            for (int i = 0; i < sz; ++i) {
                ITreeNode node = nodes[i];
                long node_ref = node.Id;
                for (int n = 0; n < nids.Length; ++n) {
                    if (nids[n] == node_ref)
                        result_nodes[n] = node;
                }
            }

            // Check the result_nodes list is completely populated,
            for (int n = 0; n < result_nodes.Length; ++n) {
                if (result_nodes[n] == null)
                    throw new ApplicationException("Assertion failed: result_nodes not completely populated.");
            }

            return result_nodes;
        }
Exemplo n.º 38
0
            private string GetSnapshotStats(string pathName, int pathInfoVersion, DataAddress address)
            {
                // Find the PathInfo object from the path_info_version. If the path
                // version is out of date then an exception is generated.
                PathInfo pathInfo = service.GetPathInfo(pathName, pathInfoVersion);

                return service.iGetSnapshotStats(pathInfo, address);
            }
Exemplo n.º 39
0
        private List<long> DoPersist(TreeWrite sequence, int tryCount)
        {
            // NOTE: nodes are written in order of branches and then leaf nodes. All
            //   branch nodes and leafs are grouped together.

            // The list of nodes to be allocated,
            IList<ITreeNode> allBranches = sequence.BranchNodes;
            IList<ITreeNode> allLeafs = sequence.LeafNodes;
            List<ITreeNode> nodes = new List<ITreeNode>(allBranches.Count + allLeafs.Count);
            nodes.AddRange(allBranches);
            nodes.AddRange(allLeafs);
            int sz = nodes.Count;
            // The list of allocated referenced for the nodes,
            DataAddress[] refs = new DataAddress[sz];
            long[] outRefs = new long[sz];

            MessageStream allocateMessage = new MessageStream(MessageType.Request);

            // Make a connection with the manager server,
            IMessageProcessor manager = connector.Connect(managerAddress, ServiceType.Manager);

            // Allocate the space first,
            for (int i = 0; i < sz; ++i) {
                ITreeNode node = nodes[i];
                RequestMessage request = new RequestMessage("allocateNode");
                // Is it a branch node?
                if (node is TreeBranch) {
                    // Branch nodes are 1K in size,
                    request.Arguments.Add(1024);
                } else {
                    // Leaf nodes are 4k in size,
                    request.Arguments.Add(4096);
                }

                allocateMessage.AddMessage(request);
            }

            // The result of the set of allocations,
            MessageStream resultStream = (MessageStream) manager.Process(allocateMessage);
            //DEBUG: ++network_comm_count;

            // The unique list of blocks,
            List<long> uniqueBlocks = new List<long>();

            // Parse the result stream one message at a time, the order will be the
            // order of the allocation messages,
            int n = 0;
            foreach (ResponseMessage m in resultStream) {
                if (m.HasError)
                    throw m.Error.AsException();

                DataAddress addr = (DataAddress) m.Arguments[0].Value;
                refs[n] = addr;
                // Make a list of unique block identifiers,
                if (!uniqueBlocks.Contains(addr.BlockId)) {
                    uniqueBlocks.Add(addr.BlockId);
                }
                ++n;
            }

            // Get the block to server map for each of the blocks,

            IDictionary<long, IList<BlockServerElement>> blockToServerMap = GetServersForBlock(uniqueBlocks);

            // Make message streams for each unique block
            int ubid_count = uniqueBlocks.Count;
            MessageStream[] ubidStream = new MessageStream[ubid_count];
            for (int i = 0; i < ubidStream.Length; ++i) {
                ubidStream[i] = new MessageStream(MessageType.Request);
            }

            // Scan all the blocks and create the message streams,
            for (int i = 0; i < sz; ++i) {
                byte[] nodeBuf;

                ITreeNode node = nodes[i];
                // Is it a branch node?
                if (node is TreeBranch) {
                    TreeBranch branch = (TreeBranch)node;
                    // Make a copy of the branch (NOTE; we Clone() the array here).
                    long[] curNodeData = (long[])branch.ChildPointers.Clone();
                    int curNdsz = branch.DataSize;
                    branch = new TreeBranch(refs[i].Value, curNodeData, curNdsz);

                    // The number of children
                    int chsz = branch.ChildCount;
                    // For each child, if it's a heap node, look up the child id and
                    // reference map in the sequence and set the reference accordingly,
                    for (int o = 0; o < chsz; ++o) {
                        long childRef = branch.GetChild(o);
                        if (childRef < 0) {
                            // The ref is currently on the heap, so adjust accordingly
                            int ref_id = sequence.LookupRef(i, o);
                            branch.SetChildOverride(o, refs[ref_id].Value);
                        }
                    }

                    // Turn the branch into a 'node_buf' byte[] array object for
                    // serialization.
                    long[] nodeData = branch.ChildPointers;
                    int ndsz = branch.DataSize;
                    MemoryStream bout = new MemoryStream(1024);
                    BinaryWriter dout = new BinaryWriter(bout, Encoding.Unicode);
                    dout.Write(BranchType);
                    dout.Write(ndsz);
                    for (int o = 0; o < ndsz; ++o) {
                        dout.Write(nodeData[o]);
                    }
                    dout.Flush();

                    // Turn it into a byte array,
                    nodeBuf = bout.ToArray();

                    // Put this branch into the local cache,
                    networkCache.SetNode(refs[i], branch);

                } else {
                    // If it's a leaf node,

                    TreeLeaf leaf = (TreeLeaf)node;
                    int lfsz = leaf.Length;

                    nodeBuf = new byte[lfsz + 6];
                    // Technically, we could comment these next two lines out.
                    ByteBuffer.WriteInt2(LeafType, nodeBuf, 0);
                    ByteBuffer.WriteInt4(lfsz, nodeBuf, 2);
                    leaf.Read(0, nodeBuf, 6, lfsz);

                    // Put this leaf into the local cache,
                    leaf = new ByteArrayTreeLeaf(refs[i].Value, nodeBuf);
                    networkCache.SetNode(refs[i], leaf);

                }

                // The DataAddress this node is being written to,
                DataAddress address = refs[i];
                // Get the block id,
                long blockId = address.BlockId;
                int bid = uniqueBlocks.IndexOf(blockId);
                RequestMessage request = new RequestMessage("writeToBlock");
                request.Arguments.Add(address);
                request.Arguments.Add(nodeBuf);
                request.Arguments.Add(0);
                request.Arguments.Add(nodeBuf.Length);
                ubidStream[bid].AddMessage(request);

                // Update 'outRefs' array,
                outRefs[i] = refs[i].Value;
            }

            // A log of successfully processed operations,
            List<object> successProcess = new List<object>(64);

            // Now process the streams on the servers,
            for (int i = 0; i < ubidStream.Length; ++i) {
                // The output message,
                MessageStream requestMessageStream = ubidStream[i];

                // Get the servers this message needs to be sent to,
                long block_id = uniqueBlocks[i];
                IList<BlockServerElement> blockServers = blockToServerMap[block_id];

                // Format a message for writing this node out,
                int bssz = blockServers.Count;
                IMessageProcessor[] blockServerProcs = new IMessageProcessor[bssz];

                // Make the block server connections,
                for (int o = 0; o < bssz; ++o) {
                    IServiceAddress address = blockServers[o].Address;
                    blockServerProcs[o] = connector.Connect(address, ServiceType.Block);
                    MessageStream responseMessageStream = (MessageStream) blockServerProcs[o].Process(requestMessageStream);
                    //DEBUG: ++network_comm_count;

                    if (responseMessageStream.HasError) {
                        // If this is an error, we need to report the failure to the
                        // manager server,
                        ReportBlockServerFailure(address);
                        // Remove the block id from the server list cache,
                        networkCache.RemoveServers(block_id);

                        // Rollback any server writes already successfully made,
                        for (int p = 0; p < successProcess.Count; p += 2) {
                            IServiceAddress blockAddress = (IServiceAddress) successProcess[p];
                            MessageStream toRollback = (MessageStream) successProcess[p + 1];

                            List<DataAddress> rollbackNodes = new List<DataAddress>(128);
                            foreach(Message rm in toRollback) {
                                DataAddress raddr = (DataAddress) rm.Arguments[0].Value;
                                rollbackNodes.Add(raddr);
                            }

                            // Create the rollback message,
                            RequestMessage rollbackRequest = new RequestMessage("rollbackNodes");
                            rollbackRequest.Arguments.Add(rollbackNodes.ToArray());

                            // Send it to the block server,
                            Message responseMessage = connector.Connect(blockAddress, ServiceType.Block).Process(rollbackRequest);
                            //DEBUG: ++network_comm_count;

                            // If rollback generated an error we throw the error now
                            // because this likely is a serious network error.
                            if (responseMessage.HasError)
                                throw new NetworkException("Rollback wrote failed: " + responseMessage.ErrorMessage);
                        }

                        // Retry,
                        if (tryCount > 0)
                            return DoPersist(sequence, tryCount - 1);

                        // Otherwise we fail the write
                        throw new NetworkException(responseMessageStream.ErrorMessage);
                    }

                    // If we succeeded without an error, add to the log
                    successProcess.Add(address);
                    successProcess.Add(requestMessageStream);

                }
            }

            // Return the references,
            return new List<long>(outRefs);
        }
Exemplo n.º 40
0
            private void PublishPath(string pathName, int pathInfoVersion, DataAddress rootNode)
            {
                // Find the PathInfo object from the path_info_version. If the path
                // version is out of date then an exception is generated.
                PathInfo pathInfo = service.GetPathInfo(pathName, pathInfoVersion);

                service.PostToPath(pathInfo, rootNode);
            }
Exemplo n.º 41
0
        private DataAddress AllocateNode(int nodeSize)
        {
            if (nodeSize >= 65536)
                throw new ArgumentException("node_size too large");
            if (nodeSize < 0)
                throw new ArgumentException("node_size too small");

            long blockId;
            int dataId;

            lock (allocationLock) {
                // Fetch the current block of the end of the address space,
                blockId = currentAddressSpaceEnd.BlockId;
                // Get the data identifier,
                dataId = currentAddressSpaceEnd.DataId;

                // The next position,
                int nextDataId = dataId;
                long nextBlockId = blockId;
                ++nextDataId;
                if (nextDataId >= 16384) {
                    nextDataId = 0;
                    ++nextBlockId;
                }

                // Create the new end address
                currentAddressSpaceEnd = new DataAddress(nextBlockId, nextDataId);
            }

            // Return the data address,
            return new DataAddress(blockId, dataId);
        }
Exemplo n.º 42
0
        private DataAddress[] GetPathRootsSince(PathInfo pathInfo, DataAddress root)
        {
            // Fetch the path access object for the given name.
            PathAccess pathFile = GetPathAccess(pathInfo.PathName);

            return pathFile.GetPathRootsSince(root);
        }
Exemplo n.º 43
0
        private void UpdateAddressSpaceEnd()
        {
            lock (blockDbWriteLock) {
                ITransaction transaction = blockDatabase.CreateTransaction();
                try {
                    // Get the map,
                    BlockServerTable blockServerTable = new BlockServerTable(transaction.GetFile(BlockServerKey, FileAccess.Read));

                    // Set the 'current address space end' object with a value that is
                    // past the end of the address space.

                    // Fetch the last block added,
                    DataAddress addressSpaceEnd = new DataAddress(0, 0);
                    // If the map is empty,
                    if (blockServerTable.Count != 0) {
                        long lastBlockId = blockServerTable.LastBlockId;
                        addressSpaceEnd = new DataAddress(lastBlockId + 1024, 0);
                    }
                    lock (allocationLock) {
                        if (currentAddressSpaceEnd == null) {
                            currentAddressSpaceEnd = addressSpaceEnd;
                        } else {
                            currentAddressSpaceEnd = currentAddressSpaceEnd.Max(addressSpaceEnd);
                        }
                    }
                } finally {
                    blockDatabase.Dispose(transaction);
                }
            }
        }
Exemplo n.º 44
0
 public ITransaction CreateTransaction(DataAddress rootNode)
 {
     // Create the transaction object and return it,
     return(treeSystem.CreateTransaction(rootNode));
 }