void ReceiverOnPredictionConfirm(List <ChunkMetaData> chunkMetaDataAndId, uint chunksCount)
{
Monitor.Enter(m_libMutex);
uint idx;
uint savedInThisCall = 0;
idx = 0;
foreach (ChunkMetaData chMetaData in chunkMetaDataAndId)
{
int ChunkLength;
object[] o;
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " writing acked " + Convert.ToString(chMetaData.chunk), ModuleLogLevel);
ChunkLength = PackChunking.chunkToLen(chMetaData.chunk);
byte[] buff = ChunkAndChainManager.ChunkAndChainManager.GetChunkData(chMetaData.chunk);
m_onDataReceived(buff, 0, buff.Length);
m_CurrentOffset += ChunkLength;
m_TotalSaved += (uint)ChunkLength;
savedInThisCall += (uint)ChunkLength;
idx++;
if (idx == chunksCount)
{
break;
}
}
Monitor.Exit(m_libMutex);
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " Saved in this call " + Convert.ToString(savedInThisCall), LogLevels.LEVEL_LOG_HIGH3);
}
void EncodePredictionAckMessage(byte[] buffer, int offset, List <ChunkMetaData> chunkMetaDataList, uint chunksCount, uint firstChunk)
{
uint buffer_idx = (uint)offset;
uint chunkCounter = 0;
uint thisTimeSaved = 0;
buffer_idx +=
ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.Uint2ByteArray(buffer, buffer_idx, chunksCount);
foreach (ChunkMetaData chunkMetaData in chunkMetaDataList)
{
if (chunkCounter >= firstChunk)
{
buffer[buffer_idx++] = chunkMetaData.hint;
buffer_idx +=
ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.Long2ByteArray(buffer, buffer_idx, chunkMetaData.chunk);
thisTimeSaved += (uint)PackChunking.chunkToLen(chunkMetaData.chunk);
}
chunkCounter++;
if ((chunkCounter - firstChunk) == chunksCount)
{
break;
}
}
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " sending PRED ACK saved now " + Convert.ToString(thisTimeSaved) + " chunks " + Convert.ToString((chunkCounter - firstChunk)), ModuleLogLevel);
m_TotalSavedData += (uint)thisTimeSaved;
}
static uint GetChainLength(List <long> chunkList)
{
uint length = 0;
foreach (long chunk in chunkList)
{
length += (uint)PackChunking.chunkToLen(chunk);
}
return(length);
}
uint GetMatchLen(uint firstChunkIdx, uint matchLen)
{
uint len = 0;
for (int idx = (int)firstChunkIdx; idx < (firstChunkIdx + matchLen); idx++)
{
len += (uint)PackChunking.chunkToLen(m_SenderChunkList[idx]);
}
return(len);
}
uint FindFirstMatchingChunk(List <ChunkMetaData> chunksList, long chunk, byte hint)
{
uint idx;
for (idx = 0; idx < chunksList.Count; idx++)
{
if ((PackChunking.chunkCode(0, PackChunking.chunkToLen(chunksList[(int)idx].chunk)) == chunk) &&
(chunksList[(int)idx].hint == hint))
{
//LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + "match " + Convert.ToString(idx), ModuleLogLevel);
break;
}
}
return(idx);
}
uint IsSha1Match(uint senderFirstIdx, List <ChunkMetaData> receiverChunksList, uint receiverFirstIdx, uint matchLength)
{
long sha1;
uint idx;
for (idx = 0; idx < matchLength; idx++)
{
sha1 = PackChunking.chunkToSha1(m_SenderChunkListWithSha1[(int)senderFirstIdx]);
if (sha1 != PackChunking.chunkToSha1(receiverChunksList[(int)receiverFirstIdx].chunk))
{
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " sha mismatch " + Convert.ToString(idx) + " " + Convert.ToString(senderFirstIdx), ModuleLogLevel);
return(idx);
}
//LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " sha match " + Convert.ToString(idx) + " " + Convert.ToString(senderFirstIdx), ModuleLogLevel);
senderFirstIdx++;
receiverFirstIdx++;
}
return(idx);
}
uint ReceiverOnDataMsg(byte[] packet, int packet_offset, byte flag, out List <ChunkListAndChainId> chainChunkList, out uint chainOffset)
{
m_onDataReceived(packet, packet_offset, packet.Length - packet_offset);
List <long> chunkList = new List <long>();
Monitor.Enter(m_libMutex);
/* process the stream (+reminder) to get chunks */
int processed_bytes = m_packChunking.getChunks(chunkList, packet, packet_offset, packet.Length, /*is_last*/ true, true);
uint offset = (uint)packet_offset;
List <ChunkMetaData[]> chunkMetaDataList = new List <ChunkMetaData[]>(100);
int idx = 0;
int lastNonMatchingChunk = chunkList.Count;
int firstNonMatchingChunk = chunkList.Count;
int firstNonMatchingChunkOffset = 0;
chainChunkList = new List <ChunkListAndChainId>(100);
List <ChunkListAndChainId> dummyChainChunkList = new List <ChunkListAndChainId>();
int rc;
bool foundMatch = false;
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " processing " + Convert.ToString(chunkList.Count) + " chunks, CurrentOffset " + Convert.ToString(m_CurrentOffset), ModuleLogLevel);
#if true
if (chunkList.Count > 0)
{
rc = m_chunkAndChainManager.ChainMatch(chunkList, chunkList.Count - 1, chainChunkList, m_SentChainList);
if (rc > 0)
{
m_SentChainList.Add(chainChunkList[0].chainId);
foundMatch = true;
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " found long chain (fast path) ", ModuleLogLevel);
}
else if (rc == 0)
{
foundMatch = true;
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " found equal chain (fast path) ", ModuleLogLevel);
}
}
if (foundMatch)
{
chainOffset = (uint)(m_CurrentOffset + processed_bytes);
m_CurrentOffset += (uint)(packet.Length - packet_offset);
Monitor.Exit(m_libMutex);
return(GetChainsListSize(chainChunkList));
}
#endif
foreach (long chunk in chunkList)
{
rc = m_chunkAndChainManager.ChainMatch(chunkList, idx, chainChunkList, m_SentChainList);
if (rc < 0)
{
lastNonMatchingChunk = idx;
if (firstNonMatchingChunk == chunkList.Count)
{
// LogUtility.LogUtility.LogFile("starting non-mactching range " + Convert.ToString(idx), ModuleLogLevel);
firstNonMatchingChunk = idx;
firstNonMatchingChunkOffset = (int)offset;
}
else
{
// LogUtility.LogUtility.LogFile("updating non-matching range " + Convert.ToString(idx), ModuleLogLevel);
}
}
else if (lastNonMatchingChunk != chunkList.Count)
{
// LogUtility.LogUtility.LogFile("end of non-matching range " + Convert.ToString(lastNonMatchingChunk), ModuleLogLevel);
Chains2Save chain2Save = new Chains2Save(chunkList, firstNonMatchingChunk, lastNonMatchingChunk, packet, firstNonMatchingChunkOffset, m_chunkAndChainManager);
AddChain2Save(chain2Save, m_Id);
firstNonMatchingChunk = chunkList.Count;
lastNonMatchingChunk = chunkList.Count;
}
m_ChunksProcessed++;
offset += (uint)PackChunking.chunkToLen(chunkList[idx]);
idx++;
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " AddUpdateChunk: offset=" + Convert.ToString(m_CurrentOffset + offset) + " chainChunkList " + Convert.ToString(chainChunkList.Count), ModuleLogLevel);
if (rc > 0)
{
m_SentChainList.Add(chainChunkList[0].chainId);
break;
}
else if (rc == 0)
{
break;
}
}
if (lastNonMatchingChunk != chunkList.Count)
{
// LogUtility.LogUtility.LogFile("end of non-matching range (last) " + Convert.ToString(lastNonMatchingChunk), ModuleLogLevel);
Chains2Save chain2Save = new Chains2Save(chunkList, firstNonMatchingChunk, lastNonMatchingChunk, packet, firstNonMatchingChunkOffset, m_chunkAndChainManager);
AddChain2Save(chain2Save, m_Id);
}
//Vadim 10/01/13 onDataReceived(packet, packet_offset, packet.Length - packet_offset);
chainOffset = (uint)(m_CurrentOffset + processed_bytes);
m_CurrentOffset += (uint)(packet.Length - packet_offset);
if (chainChunkList.Count == 0)
{
Monitor.Exit(m_libMutex);
return(0);
}
Monitor.Exit(m_libMutex);
return(GetChainsListSize(chainChunkList));
}
static public bool SaveChain(List <long> chunkList, int chunkListIdx, int LastChunkId, byte[] data, int offset, ChunkAndChainManager chunkAndChainManager)
{
ChunkCB chunkCB;
byte[] buf;
bool presentChunks = false;
int savedChunkListIdx = chunkListIdx;
int idx;
Chain chain4Lookup;
if (chunkAndChainManager.chainId4Lookup == 0)
{
GetChainAsByteArray(new long[0], out buf);
chunkAndChainManager.chainId4Lookup = fileManager.AddUpdateChainFile(buf, chunkAndChainManager.chainId4Lookup);
// LogUtility.LogUtility.LogFile("****created new chain " + Convert.ToString(chainId4Lookup), ModuleLogLevel);
ChainsCreated++;
}
else
{
// LogUtility.LogUtility.LogFile("****updating chain " + Convert.ToString(chainId4Lookup), ModuleLogLevel);
ChainsUpdated++;
}
for (; chunkListIdx <= LastChunkId; chunkListIdx++)
{
#if false
if (chunkMap.TryGetValue(chunkId, out chunkCB))
#else
if (!GetChunkCB(chunkList[chunkListIdx], out chunkCB))
#endif
{
chunkCB = new ChunkCB();
chunkCB.sha1 = PackChunking.chunkToSha1(chunkList[chunkListIdx]);
chunkCB.size = (uint)PackChunking.chunkToLen(chunkList[chunkListIdx]);
fileManager.AddNewChunk(data, (uint)offset, chunkCB.size, chunkList[chunkListIdx], out chunkCB.fo);
#if false
{
byte[] buff2log = new byte[chunkCB.size];
for (int i = 0; i < chunkCB.size; i++)
{
buff2log[i] = data[i + offset];
}
LogUtility.LogUtility.LogBinary("_stored", buff2log);
}
#endif
chunkCB.hint = PackChunking.GetChunkHint(data, (uint)offset, chunkCB.size);
chunkCB.chains = new long[1];
chunkCB.chains[0] = chunkAndChainManager.chainId4Lookup;
chunkCB.ChainsListSize = 1;
// LogUtility.LogUtility.LogFile("****creating new chunk CB " + Convert.ToString(chunkCB.size) + " " + Convert.ToString(chunkList[chunkListIdx]) + " len " + Convert.ToString(PackChunking.chunkToLen(chunkList[chunkListIdx])) + " hint " + Convert.ToString(chunkCB.hint) + " sha1 " + Convert.ToString(chunkCB.sha1), ModuleLogLevel);
ChunksCreated++;
}
else if (!ChainIsPresentInChunkCB(chunkCB, chunkAndChainManager.chainId4Lookup))
{
presentChunks = true;
long[] newChainList = new long[chunkCB.chains.Length + 1];
chunkCB.chains.CopyTo(newChainList, 0);
newChainList[chunkCB.chains.Length] = chunkAndChainManager.chainId4Lookup;
chunkCB.chains = newChainList;
chunkCB.ChainsListSize++;
// LogUtility.LogUtility.LogFile("****updating existing chunk CB " + Convert.ToString(chunkCB.size) + " " + Convert.ToString(chunkList[chunkListIdx]) + " hint " + Convert.ToString(chunkCB.hint) + " sha1 " + Convert.ToString(chunkCB.sha1), ModuleLogLevel);
ChainAdded2ExistingChunk++;
}
else
{
// LogUtility.LogUtility.LogFile("****chain exists in chunk CB " + Convert.ToString(chunkList[chunkListIdx]) + " " + Convert.ToString(PackChunking.chunkToLen(chunkList[chunkListIdx])) + " chain " + Convert.ToString(chainId4Lookup) + " hint " + Convert.ToString(chunkCB.hint) + " sha1 " + Convert.ToString(chunkCB.sha1), ModuleLogLevel);
offset += (int)chunkCB.size;
ChainExistInChunkChainsList++;
continue;
}
offset += (int)chunkCB.size;
GetChunkCBAsByteArray(chunkCB, out buf);
fileManager.AddUpdateChunkCtrlFile(chunkList[chunkListIdx], buf, 0, (uint)buf.Length);
}
if (!GetChain(chunkAndChainManager.chainId4Lookup, out chain4Lookup))
{
LogUtility.LogUtility.LogFile("****cannot get chain!!! " + Convert.ToString(chunkAndChainManager.chainId4Lookup), ModuleLogLevel);
CannotGetChain++;
return(presentChunks);
}
// LogUtility.LogUtility.LogFile("prepare chain, now " + Convert.ToString(chain4Lookup.NumberOfEntries) + " ( " + Convert.ToString(chain4Lookup.chunkIds.Length) + ")" + " will be added " + Convert.ToString(chunkList.Count-savedChunkListIdx), ModuleLogLevel);
long[] newChunkList = new long[chain4Lookup.NumberOfEntries + ((LastChunkId + 1) - savedChunkListIdx)];
chain4Lookup.chunkIds.CopyTo(newChunkList, 0);
idx = (int)chain4Lookup.NumberOfEntries;
for (chunkListIdx = savedChunkListIdx; chunkListIdx <= LastChunkId; chunkListIdx++)
{
#if false
if (ChunkIsPresentInChain(chain4Lookup, chunkList[chunkListIdx]))
{
LogUtility.LogUtility.LogFile("SANITY CHECK: chunk " + Convert.ToString(chunkList[chunkListIdx]) + " len " + Convert.ToString(PackChunking.chunkToLen(chunkList[chunkListIdx])) + " is already in chain", ModuleLogLevel);
return(presentChunks);
}
#endif
newChunkList[idx++] = chunkList[chunkListIdx];
}
chain4Lookup.chunkIds = newChunkList;
chain4Lookup.NumberOfEntries = (uint)newChunkList.Length;
GetChainAsByteArray(chain4Lookup.chunkIds, out buf);
//save on disk
chunkAndChainManager.chainId4Lookup = fileManager.AddUpdateChainFile(buf, chunkAndChainManager.chainId4Lookup);
return(presentChunks);
}
void MatchChain(List <ChunkMetaData> predMsg)
{
try
{
uint matchLen = 0;
uint matchChunkCount = 0;
uint firstSenderIdx = 0;
uint firstReceiverIdx = 0;
int senderChunkIdx = 0;
int receiverChunkIdx = 0;
bool match = false;
int offset = 0;
int savedOffset = 0;
while ((senderChunkIdx < m_SenderChunkList.Count) && (receiverChunkIdx < predMsg.Count))
{
byte senderHint = PackChunking.GetChunkHint(m_data, (uint)offset, (uint)PackChunking.chunkToLen(m_SenderChunkList[senderChunkIdx]));
long senderChunk = PackChunking.chunkCode(0, PackChunking.chunkToLen(m_SenderChunkList[senderChunkIdx]));
switch (match)
{
case false:
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " in non-match " + Convert.ToString(PackChunking.chunkToLen(m_SenderChunkList[senderChunkIdx])), ModuleLogLevel);
receiverChunkIdx = (int)FindFirstMatchingChunk(predMsg, senderChunk, senderHint);
if (receiverChunkIdx != predMsg.Count)
{
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + " match " + Convert.ToString(PackChunking.chunkToLen(predMsg[(int)receiverChunkIdx].chunk)), ModuleLogLevel);
match = true;
firstReceiverIdx = (uint)receiverChunkIdx;
firstSenderIdx = (uint)senderChunkIdx;
matchLen = (uint)PackChunking.chunkToLen(m_SenderChunkList[senderChunkIdx]);
matchChunkCount = 1;
savedOffset = offset;
receiverChunkIdx++;
}
else
{
receiverChunkIdx = 0;
}
break;
case true:
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + "in match " + Convert.ToString(PackChunking.chunkToLen(m_SenderChunkList[senderChunkIdx])), ModuleLogLevel);
if ((senderChunk != PackChunking.chunkCode(0, PackChunking.chunkToLen(predMsg[receiverChunkIdx].chunk))) ||
(senderHint != predMsg[receiverChunkIdx].hint))
{
match = false;
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + "stopped. matching sha1 ", ModuleLogLevel);
OnEndOfMatch(predMsg, matchLen, matchChunkCount, firstSenderIdx, firstReceiverIdx, savedOffset);
if (matchLen >= (m_data.Length / 3))
{
//return;
}
}
else
{
matchLen += (uint)PackChunking.chunkToLen(m_SenderChunkList[senderChunkIdx]);
matchChunkCount++;
receiverChunkIdx++;
if (senderChunkIdx == (m_SenderChunkList.Count - 1))
{
LogUtility.LogUtility.LogFile(Convert.ToString(m_Id) + "stopped (end). matching sha1 ", ModuleLogLevel);
OnEndOfMatch(predMsg, matchLen, matchChunkCount, firstSenderIdx, firstReceiverIdx, savedOffset);
if (matchLen >= (m_data.Length / 3))
{
//return;
}
}
}
break;
}
//LogUtility.LogUtility.LogFile("Sender's offset " + Convert.ToString(offset), ModuleLogLevel);
offset += (int)PackChunking.chunkToLen(m_SenderChunkList[senderChunkIdx]);
senderChunkIdx++;
}
}
catch (Exception exc)
{
LogUtility.LogUtility.LogFile("EXCEPTION: " + exc.Message + " " + exc.StackTrace + " " + exc.InnerException, LogUtility.LogLevels.LEVEL_LOG_HIGH);
}
}