public DataAddress(BlockId blockId, int dataId)
{
// TODO: Check for overflow?
long[] blockAddr = blockId.ReferenceAddress;
blockAddr[1] |= dataId & 0x0FFFF;
value = new NodeId(blockAddr);
}
public IList<BlockServerElement> GetServersWithBlock(BlockId blockId)
{
lock (s2BlockCache) {
BlockCacheElement ele;
if (!s2BlockCache.TryGetValue(blockId, out ele) ||
DateTime.Now > ele.TimeToEnd) {
return null;
}
return ele.BlockServers;
}
}
protected override int DiscoverBlockType(BlockId blockId)
{
// Get the block unit,
String blockFileName = FormatFileName(blockId);
int fileType = 1;
string f = Path.Combine(path, blockFileName);
if (!File.Exists(f)) {
fileType = 2;
f = Path.Combine(path, blockFileName + ".mcd");
}
if (!File.Exists(f))
return -1;
return fileType;
}
protected override BlockId[] FetchBlockList()
{
string[] dir = Directory.GetFiles(path);
List <BlockId> blocks = new List <BlockId>(dir.Length);
foreach (string f in dir)
{
String fnameStr = Path.GetFileName(f);
if (!fnameStr.Equals("block_server_guid") &&
!fnameStr.EndsWith(".tempc") &&
!fnameStr.EndsWith(".tmpc1") &&
!fnameStr.EndsWith(".tmpc2"))
{
if (fnameStr.EndsWith(".mcd"))
{
fnameStr = fnameStr.Substring(0, fnameStr.Length - 4);
}
BlockId blockId = ParseFileName(fnameStr);
blocks.Add(blockId);
}
}
return(blocks.ToArray());
}
public IEnumerable <Message> Process(IEnumerable <Message> stream)
{
// The map of containers touched,
Dictionary <BlockId, BlockContainer> containersTouched = new Dictionary <BlockId, BlockContainer>();
// The reply message,
MessageStream replyMessage = new MessageStream();
// The nodes fetched in this message,
List <NodeId> readNodes = null;
foreach (Message m in stream)
{
try {
// Check for stop state,
service.CheckErrorState();
// writeToBlock(DataAddress, byte[], int, int)
if (m.Name.Equals("writeToBlock"))
{
WriteToBlock(containersTouched,
(DataAddress)m.Arguments[0].Value,
(byte[])m.Arguments[1].Value, (int)m.Arguments[2].Value,
(int)m.Arguments[3].Value);
replyMessage.AddMessage(new Message(1L));
}
// readFromBlock(DataAddress)
else if (m.Name.Equals("readFromBlock"))
{
if (readNodes == null)
{
readNodes = new List <NodeId>();
}
DataAddress addr = (DataAddress)m.Arguments[0].Value;
if (!readNodes.Contains(addr.Value))
{
NodeSet nodeSet = ReadFromBlock(containersTouched, addr);
replyMessage.AddMessage(new Message(nodeSet));
readNodes.AddRange(nodeSet.NodeIds);
}
}
// rollbackNodes(DataAddress[] )
else if (m.Name.Equals("rollbackNodes"))
{
RemoveNodes(containersTouched, (DataAddress[])m.Arguments[0].Value);
replyMessage.AddMessage(new Message(1L));
}
// deleteBlock(BlockId)
else if (m.Name.Equals("deleteBlock"))
{
DeleteBlock((BlockId)m.Arguments[0].Value);
replyMessage.AddMessage(new Message(1L));
}
// serverGUID()
else if (m.Name.Equals("serverGUID"))
{
replyMessage.AddMessage(new Message(service.Id));
}
// blockSetReport()
else if (m.Name.Equals("blockSetReport"))
{
BlockId[] arr = BlockSetReport();
replyMessage.AddMessage(new Message(service.Id, arr));
}
// poll(String)
else if (m.Name.Equals("poll"))
{
replyMessage.AddMessage(new Message(1L));
}
// notifyCurrentBlockId(BlockId)
else if (m.Name.Equals("notifyCurrentBlockId"))
{
service.NotifyCurrentBlockId((BlockId)m.Arguments[0].Value);
replyMessage.AddMessage(new Message(1L));
}
// blockChecksum(BlockId)
else if (m.Name.Equals("blockChecksum"))
{
long checksum = BlockChecksum(containersTouched, (BlockId)m.Arguments[0].Value);
replyMessage.AddMessage(new Message(checksum));
}
// sendBlockTo(BlockId, IServiceAddress, long, IServerAddress[])
else if (m.Name.Equals("sendBlockTo"))
{
// Returns immediately. There's currently no way to determine
// when this process will happen or if it will happen.
BlockId blockId = (BlockId)m.Arguments[0].Value;
IServiceAddress destAddress = (IServiceAddress)m.Arguments[1].Value;
long destServerId = (long)m.Arguments[2].Value;
IServiceAddress[] managerServers = (IServiceAddress[])m.Arguments[3].Value;
long processId = SendBlockTo(blockId, destAddress, destServerId, managerServers);
replyMessage.AddMessage(new Message(processId));
}
// sendBlockPart(BlockId, long, int, byte[], int)
else if (m.Name.Equals("sendBlockPart"))
{
BlockId blockId = (BlockId)m.Arguments[0].Value;
long pos = (long)m.Arguments[1].Value;
int fileType = (int)m.Arguments[2].Value;
byte[] buf = (byte[])m.Arguments[3].Value;
int bufSize = (int)m.Arguments[4].Value;
service.WriteBlockPart(blockId, pos, fileType, buf, bufSize);
replyMessage.AddMessage(new Message(1L));
}
// sendBlockComplete(BlockId, int)
else if (m.Name.Equals("sendBlockComplete"))
{
BlockId blockId = (BlockId)m.Arguments[0].Value;
int fileType = (int)m.Arguments[1].Value;
service.WriteBlockComplete(blockId, fileType);
replyMessage.AddMessage(new Message(1L));
}
// createAvailabilityMapForBlocks(BlockId[])
else if (m.Name.Equals("createAvailabilityMapForBlocks"))
{
BlockId[] blockIds = (BlockId[])m.Arguments[0].Value;
byte[] map = service.CreateAvailabilityMapForBlocks(blockIds);
replyMessage.AddMessage(new Message(map));
}
// bindWithManager()
else if (m.Name.Equals("bindWithManager"))
{
BindWithManager();
replyMessage.AddMessage(new Message(1L));
}
// unbindWithManager()
else if (m.Name.Equals("unbindWithManager"))
{
UnbindWithManager();
replyMessage.AddMessage(new Message(1L));
}
else
{
throw new ApplicationException("Unknown command: " + m.Name);
}
} catch (OutOfMemoryException e) {
service.Logger.Error("Memory Error", e);
service.SetErrorState(e);
throw;
} catch (Exception e) {
service.Logger.Error("Exception during process", e);
replyMessage.AddMessage(new Message(new MessageError(e)));
}
}
// Release any containers touched,
try {
CloseContainers(containersTouched);
} catch (IOException e) {
service.Logger.Error("IOError when closing containers", e);
}
return(replyMessage);
}
protected BlockData(BlockId blockId, int blockType)
{
this.blockId = blockId;
this.blockType = blockType;
}
protected abstract IBlockStore GetBlockStore(BlockId blockId);
public CompressedBlockStore(BlockId blockId, string fileName)
{
this.blockId = blockId;
this.fileName = fileName;
}
public CompressedBlockStore(BlockId blockId, string fileName)
{
this.blockId = blockId;
this.fileName = fileName;
}
public FileBlockData(FileSystemBlockService blockService, BlockId blockId, int fileType)
: base(blockId, fileType)
{
this.blockService = blockService;
fileName = FormatFileName();
}
public MemoryBlockData(MemoryBlockService blockService, BlockId blockId, int blockType)
: base(blockId, blockType)
{
this.blockService = blockService;
}
private long[] AllocateOnlineServerNodesForBlock(BlockId blockId)
{
// Fetch the list of all online servers,
List<BlockServiceInfo> servSet;
lock (blockServersMap) {
servSet = new List<BlockServiceInfo>(blockServersList.Count);
foreach (BlockServiceInfo server in blockServersList) {
// Add the servers with status 'up'
if (serviceTracker.IsServiceUp(server.Address, ServiceType.Block)) {
servSet.Add(server);
}
}
}
// TODO: This is a simple random server picking method for a block.
// We should prioritize servers picked based on machine specs, etc.
int sz = servSet.Count;
// If serv_set is 3 or less, we return the servers available,
if (sz <= 3) {
long[] returnVal = new long[sz];
for (int i = 0; i < sz; ++i) {
BlockServiceInfo blockServer = servSet[i];
returnVal[i] = blockServer.ServerGuid;
}
return returnVal;
} else {
// Randomly pick three servers from the list,
long[] returnVal = new long[3];
for (int i = 0; i < 3; ++i) {
// java.util.Random is specced to be thread-safe,
int randomI = rng.Next(servSet.Count);
BlockServiceInfo blockServer = servSet[i];
servSet.RemoveAt(randomI);
returnVal[i] = blockServer.ServerGuid;
}
// Return the array,
return returnVal;
}
}
private long[] AllocateNewBlock(BlockId blockId)
{
lock (allocationLock) {
// Schedule a task that informs all the current block servers what the
// new block being allocated against is. This notifies them that they
// can perform maintenance on all blocks preceding, such as compression.
long[] blockServersNotify = currentBlockIdServers;
if (blockServersNotify != null && blockServersNotify.Length > 0) {
NewBlockAllocInfo info = new NewBlockAllocInfo(blockId, blockServersNotify);
allocTimer = new Timer(NewBlockAllocTask, info, 500, Timeout.Infinite);
}
// Assert the block isn't already allocated,
long[] currentServers = managerDb.GetBlockIdServerMap(blockId);
if (currentServers.Length > 0) {
throw new NetworkWriteException("Block already allocated: " + blockId);
}
// Allocate a group of servers from the poll of block servers for the
// given block_id
long[] servers = AllocateOnlineServerNodesForBlock(blockId);
// Update the database if we have servers to allocate the block id,
if (servers.Length > 0)
managerDb.SetBlockIdServerMap(blockId, servers);
// Return the list,
return servers;
}
}
public NewBlockAllocInfo(BlockId blockId, long[] blockServersToNotify)
{
BlockId = blockId;
BlockServersToNotify = blockServersToNotify;
}
private BlockServiceInfo[] GetServerList(BlockId blockId)
{
// Query the local database for the server list of the block. If the
// block doesn't exist in the database then it provisions it over the
// network.
long[] serverIds = service.GetOnlineServersWithBlock(blockId);
// Resolve the server ids into server names and parse it as a reply
int sz = serverIds.Length;
// No online servers contain the block
if (sz == 0)
throw new ApplicationException("No online servers for block: " + blockId);
BlockServiceInfo[] reply = service.GetServersInfo(serverIds);
service.Logger.Info(String.Format("getServersInfo replied {0} for {1}", reply.Length, blockId));
return reply;
}
private long PlaybackMessages(LogEntryIterator messages)
{
long count = 0;
List<LogEntry> bundle = new List<LogEntry>(32);
while (true) {
bool done = false;
bundle.Clear();
for (int i = 0; i < 32 && !done; ++i) {
LogEntry entry = messages.NextLogEntry();
if (entry == null) {
done = true;
} else {
bundle.Add(entry);
}
}
// Finished,
if (bundle.Count == 0 && done)
break;
lock (blockDbWriteLock) {
ITransaction transaction = blockDatabase.CreateTransaction();
try {
foreach (LogEntry entry in bundle) {
// Deserialize the entry,
long[] uid = entry.Uid;
byte[] buf = entry.Buffer;
// If this hasn't applied the uid then we apply it,
if (!HasAppliedUid(transaction, uid)) {
MemoryStream bin = new MemoryStream(buf);
BinaryReader din = new BinaryReader(bin, Encoding.Unicode);
byte m = din.ReadByte();
if (m == 18) {
// Block_id to server map
long blockIdH = din.ReadInt64();
long blockIdL = din.ReadInt64();
BlockId blockId = new BlockId(blockIdH, blockIdL);
int sz = din.ReadInt32();
long[] servers = new long[sz];
for (int i = 0; i < sz; ++i) {
servers[i] = din.ReadInt64();
}
// Replay this command,
InsertBlockIdServerEntry(transaction, uid, blockId, servers);
} else if (m == 19) {
// Key/Value pair
String key = din.ReadString();
String value = null;
byte vb = din.ReadByte();
if (vb == 1) {
value = din.ReadString();
}
// Replay this command,
InsertKeyValueEntry(transaction, uid, key, value);
} else {
throw new ApplicationException("Unknown entry type: " + m);
}
// Increment the count,
++count;
}
} // For each entry in the bundle
// Commit and check point the update,
blockDatabase.Publish(transaction);
blockDatabase.CheckPoint();
} catch (IOException e) {
throw new ApplicationException(e.Message, e);
} finally {
blockDatabase.Dispose(transaction);
}
} // lock
} // while true
return count;
}
private void InternalBsProposal(long[] uid, BlockId blockId, long[] serverUids)
{
// Check this store is connected. Throws an exception if not.
CheckConnected();
}
private void InternalBsComplete(long[] uid, BlockId blockId, long[] serverUids)
{
// Check this store is connected. Throws an exception if not.
CheckConnected();
// Perform this under a lock. This lock is also active for block queries
// and administration updates.
lock (blockDbWriteLock) {
// Create a transaction
ITransaction transaction = blockDatabase.CreateTransaction();
try {
// We must handle the case when multiple identical proposals come in,
if (!HasAppliedUid(transaction, uid)) {
// Insert the block id server mapping,
InsertBlockIdServerEntry(transaction, uid, blockId, serverUids);
// Commit and check point the update,
blockDatabase.Publish(transaction);
blockDatabase.CheckPoint();
}
} finally {
blockDatabase.Dispose(transaction);
}
}
}
private long BlockChecksum(Dictionary <BlockId, BlockContainer> containersTouched, BlockId blockId)
{
// Fetch the block container,
BlockContainer container = GetBlock(containersTouched, blockId);
// Calculate the checksum value,
return(container.CreateChecksum());
}
public MemoryBlockData(MemoryBlockService blockService, BlockId blockId, int blockType)
: base(blockId, blockType)
{
this.blockService = blockService;
}
private BlockId GetCurrentBlockIdAlloc()
{
lock (allocationLock) {
if (currentAddressSpaceEnd == null) {
// Ask the manager cluster for the last block id
BlockId blockId = managerDb.GetLastBlockId();
if (blockId == null) {
// Initial state when the server map is empty,
long nl = (256L & -256 /*0x0FFFFFFFFFFFFFF00L*/);
nl += UniqueManagerId;
blockId = new BlockId(0, nl);
} else {
blockId = blockId.Add(1024);
// Clear the lower 8 bits and put the manager unique id there
long nl = (blockId.Low & -256 /*0x0FFFFFFFFFFFFFF00L*/);
nl += UniqueManagerId;
blockId = new BlockId(blockId.High, nl);
}
return blockId;
} else {
return currentAddressSpaceEnd.BlockId;
}
}
}
protected override BlockData GetBlockData(BlockId blockId, int blockType)
{
return(new FileBlockData(this, blockId, blockType));
}
private long[] GetOnlineServersWithBlock(BlockId blockId)
{
// Fetch the server map for the block from the db cluster,
return managerDb.GetBlockIdServerMap(blockId);
}
private void InsertBlockIdServerEntry(ITransaction t, long[] uid, BlockId blockId, long[] servers)
{
byte[] buf;
// Put this proposal in a local log,
try {
MemoryStream bout = new MemoryStream(64);
BinaryWriter dout = new BinaryWriter(bout);
dout.Write((byte)18);
dout.Write(blockId.High);
dout.Write(blockId.Low);
dout.Write(servers.Length);
for (int i = 0; i < servers.Length; ++i) {
dout.Write(servers[i]);
}
buf = bout.ToArray();
} catch (IOException e) {
throw new ApplicationException(e.Message, e);
}
// Inserts the value
InsertValue(t, uid, buf);
// Inserts a reference
InsertBlockIdRef(t, blockId, uid);
}
private void InternalAddBlockServerMapping(BlockId blockId, long[] serverGuids)
{
long[] currentServerGuids = managerDb.GetBlockIdServerMap(blockId);
List<long> serverList = new List<long>(64);
foreach (long s in currentServerGuids) {
serverList.Add(s);
}
// Add the servers to the list,
foreach (long s in serverGuids) {
if (!serverList.Contains(s)) {
serverList.Add(s);
}
}
if (serverList.Count > 0) {
// Set the new list
long[] newServerGuids = new long[serverList.Count];
for (int i = 0; i < serverList.Count; ++i) {
newServerGuids[i] = serverList[i];
}
managerDb.SetBlockIdServerMap(blockId, newServerGuids);
}
}
public void RemoveServersWithBlock(BlockId blockId)
{
lock (s2BlockCache) {
s2BlockCache.Remove(blockId);
}
}
private void InternalRemoveBlockServerMapping(BlockId blockId, long[] serverGuids)
{
long[] currentServerGuids = managerDb.GetBlockIdServerMap(blockId);
List<long> serverList = new List<long>(64);
foreach (long s in currentServerGuids) {
serverList.Add(s);
}
// Remove the servers from the list
foreach (long s in serverGuids) {
int index = serverList.IndexOf(s);
if (index >= 0) {
serverList.RemoveAt(index);
}
}
// Set the new list
long[] newServerGuids = new long[serverList.Count];
for (int i = 0; i < serverList.Count; ++i) {
newServerGuids[i] = serverList[i];
}
managerDb.SetBlockIdServerMap(blockId, newServerGuids);
}
protected abstract BlockData GetBlockData(BlockId blockId, int blockType);
private void NotifyBlockIdCorruption(IServiceAddress serverAddress, BlockId blockId, string failureType)
{
// TODO:
}
protected virtual int DiscoverBlockType(BlockId blockId)
{
return(1);
}
private void NotifyBlockServerOfMaxBlockId(IServiceAddress blockServer, BlockId blockId)
{
if (serviceTracker.IsServiceUp(blockServer, ServiceType.Block)) {
Message message = new Message("notifyCurrentBlockId", blockId);
// Connect to the block server,
IMessageProcessor processor = connector.Connect(blockServer, ServiceType.Block);
IEnumerable<Message> response = processor.Process(message.AsStream());
// If the block server is down, report it to the tracker,
foreach (Message m in response) {
if (m.HasError) {
if (ReplicatedValueStore.IsConnectionFault(m)) {
serviceTracker.ReportServiceDownClientReport(blockServer, ServiceType.Block);
}
}
}
}
}
private BlockContainer GetBlock(Dictionary <BlockId, BlockContainer> touched, BlockId blockId)
{
BlockContainer b;
if (!touched.TryGetValue(blockId, out b))
{
b = service.FetchBlockContainer(blockId);
bool created = b.Open();
if (created)
{
++service.blockCount;
}
touched[blockId] = b;
}
return(b);
}
private void NotifyBlockServersOfCurrentBlockId(long[] blockServersNotify, BlockId blockId)
{
// Copy the block servers list for concurrency safety,
List<BlockServiceInfo> blockServersListCopy = new List<BlockServiceInfo>(64);
lock (blockServersMap) {
blockServersListCopy.AddRange(blockServersList);
}
// For each block server
foreach (BlockServiceInfo blockServer in blockServersListCopy) {
// Is it in the block_servers_notify list?
bool found = false;
foreach (long bsn in blockServersNotify) {
if (blockServer.ServerGuid == bsn) {
found = true;
break;
}
}
// If found and the service is up,
if (found) {
NotifyBlockServerOfMaxBlockId(blockServer.Address, blockId);
}
}
}
private void DeleteBlock(BlockId blockId)
{
}
public void Execute(object state)
{
BlockServiceInfo blockToCheck;
// Cycle through the block servers list,
lock (service.blockServersMap) {
if (service.blockServersList.Count == 0) {
return;
}
if (init == false)
blockIdIndex = service.rng.Next(service.blockServersList.Count);
blockToCheck = service.blockServersList[blockIdIndex];
++blockIdIndex;
if (blockIdIndex >= service.blockServersList.Count) {
blockIdIndex = 0;
}
init = true;
}
// Notify the block server of the current block,
BlockId currentBlockId;
lock (service.allocationLock) {
if (service.currentAddressSpaceEnd == null) {
if (currentEndBlock == null) {
currentEndBlock = service.GetCurrentBlockIdAlloc();
}
currentBlockId = currentEndBlock;
} else {
currentBlockId = service.currentAddressSpaceEnd.BlockId;
}
}
// Notify the block server we are cycling through of the maximum block id.
service.NotifyBlockServerOfMaxBlockId(blockToCheck.Address, currentBlockId);
}
protected override IBlockStore GetBlockStore(BlockId blockId)
{
return new MemoryBlockStore(blockId);
}
internal MemoryBlockStore(BlockId blockId)
{
this.blockId = blockId;
}
protected override IBlockStore GetBlockStore(BlockId blockId)
{
return(new MemoryBlockStore(blockId));
}
public FileBlockStore(BlockId blockId, string fileName)
{
this.blockId = blockId;
this.fileName = fileName;
}
protected override BlockData GetBlockData(BlockId blockId, int blockType)
{
return new FileBlockData(this, blockId, blockType);
}
public SingleNodeSet(BlockId blockId, int dataId, byte[] buffer)
: base(new NodeId[] { new DataAddress(blockId, dataId).Value }, buffer)
{
}
private String FormatFileName(BlockId blockId)
{
long blockIdH = blockId.High;
long blockIdL = blockId.Low;
StringBuilder b = new StringBuilder();
b.AppendFormat("0x{0:x}", blockIdH);
string l = String.Format("0x{0:x}", blockIdL);
int pad = 16 - l.Length;
b.Append("X");
for (int i = 0; i < pad; ++i) {
b.Append("0");
}
b.Append(l);
return b.ToString();
}
public void SetServersForBlock(BlockId blockId, IList<BlockServerElement> servers, int ttlHint)
{
BlockCacheElement ele = new BlockCacheElement();
ele.BlockServers = servers;
ele.TimeToEnd = DateTime.Now.AddMilliseconds(ttlHint);
lock (s2BlockCache) {
s2BlockCache[blockId] = ele;
}
}
public void RemoveServersWithBlock(BlockId blockId)
{
lock (s2BlockCache) {
s2BlockCache.Remove(blockId);
}
}
private static void InsertBlockIdRef(ITransaction t, BlockId blockId, long[] uid)
{
IDataFile bidUidListDf = t.GetFile(BlockidUidMapKey, FileAccess.ReadWrite);
BlockIdUidList blockidUidList = new BlockIdUidList(bidUidListDf);
blockidUidList.AddBlockIdRef(blockId, uid);
}
protected override IBlockStore GetBlockStore(BlockId blockId)
{
// If it's not found in the map,
// Turn the block id into a filename,
string blockFname = FormatFileName(blockId);
string blockFileName = Path.Combine(path, blockFname + ".mcd");
IBlockStore blockStore;
if (!File.Exists(blockFileName)) {
blockFileName = Path.Combine(path, blockFname);
blockStore = new FileBlockStore(blockId, blockFileName);
} else {
blockStore = new CompressedBlockStore(blockId, blockFileName);
}
return blockStore;
}
public FileBlockStore(BlockId blockId, string fileName)
{
this.blockId = blockId;
this.fileName = fileName;
}
