Beispiel #1
0
 static uint GetChunkCB(byte[] buffer, uint offset, out ChunkCB chunkCB)
 {
     offset        += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.ByteArray2Uint(buffer, offset, out chunkCB.size);
     offset        += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.ByteArray2Long(buffer, offset, out chunkCB.sha1);
     chunkCB.hint   = buffer[offset++];
     offset        += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.ByteArray2Long(buffer, offset, out chunkCB.fo.file);
     offset        += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.ByteArray2Long(buffer, offset, out chunkCB.fo.offset);
     offset        += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.ByteArray2Uint(buffer, offset, out chunkCB.ChainsListSize);
     chunkCB.chains = new long[chunkCB.ChainsListSize];
     return(offset);
 }
Beispiel #2
0
 static bool ChainIsPresentInChunkCB(ChunkCB chunkCB, long chain4Lookup)
 {
     for (int idx = 0; idx < chunkCB.ChainsListSize; idx++)
     {
         if (chunkCB.chains[idx] == chain4Lookup)
         {
             return(true);
         }
     }
     return(false);
 }
Beispiel #3
0
        static uint GetChunkCBAndChainList(byte[] buffer, uint offset, out ChunkCB chunkCB)
        {
            offset = GetChunkCB(buffer, offset, out chunkCB);
            long [] chains = new long[chunkCB.ChainsListSize];

            for (int idx = 0; idx < chunkCB.ChainsListSize; idx++)
            {
                offset += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.ByteArray2Long(buffer, offset, out chains[idx]);
                ChainsLoaded++;
            }
            chunkCB.chains = chains;
            return(offset);
        }
Beispiel #4
0
        static void GetChunkCBAsByteArray(ChunkCB chunkCB, out byte[] buffer)
        {
            buffer = new byte[ChunkCB.GetSize(chunkCB.ChainsListSize)];
            uint offset = 0;

            offset          += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.Uint2ByteArray(buffer, offset, chunkCB.size);
            offset          += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.Long2ByteArray(buffer, offset, chunkCB.sha1);
            buffer[offset++] = chunkCB.hint;
            offset          += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.Long2ByteArray(buffer, offset, chunkCB.fo.file);
            offset          += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.Long2ByteArray(buffer, offset, chunkCB.fo.offset);
            offset          += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.Uint2ByteArray(buffer, offset, chunkCB.ChainsListSize);
            foreach (long l in chunkCB.chains)
            {
                offset += ByteArrayScalarTypeConversionLib.ByteArrayScalarTypeConversionLib.Long2ByteArray(buffer, offset, l);
            }
        }
Beispiel #5
0
 static bool GetChunkCB(long chunk, out ChunkCB chunkCB)
 {
     //string filename = "datafiles" + "\\"+ FileManager.FileManager.LongName2StringName(chunk, (byte)FileManager.FileTypes.CHUNK_CTRL_FILE_TYPE);
     try
     {
         byte[] buff = fileManager.ReadFile(chunk);
         if (buff == null)
         {
             chunkCB = new ChunkCB();
             return(false);
         }
         if (GetChunkCBAndChainList(buff, 0, out chunkCB) == buff.Length)
         {
             ChunkExists++;
             return(true);
         }
     }
     catch (Exception exc)
     {
         LogUtility.LogUtility.LogFile("EXCEPTION: " + exc.Message + " " + exc.StackTrace, LogUtility.LogLevels.LEVEL_LOG_HIGH);
     }
     chunkCB = new ChunkCB();
     return(false);
 }
Beispiel #6
0
        static public bool SaveChain(List <long> chunkList, int chunkListIdx, int LastChunkId, byte[] data, int offset, ChunkAndChainManager chunkAndChainManager)
        {
            ChunkCB chunkCB;

            byte[] buf;
            bool   presentChunks     = false;
            int    savedChunkListIdx = chunkListIdx;
            int    idx;
            Chain  chain4Lookup;

            if (chunkAndChainManager.chainId4Lookup == 0)
            {
                GetChainAsByteArray(new long[0], out buf);
                chunkAndChainManager.chainId4Lookup = fileManager.AddUpdateChainFile(buf, chunkAndChainManager.chainId4Lookup);
//                LogUtility.LogUtility.LogFile("****created new chain " + Convert.ToString(chainId4Lookup), ModuleLogLevel);
                ChainsCreated++;
            }
            else
            {
//                LogUtility.LogUtility.LogFile("****updating chain " + Convert.ToString(chainId4Lookup), ModuleLogLevel);
                ChainsUpdated++;
            }

            for (; chunkListIdx <= LastChunkId; chunkListIdx++)
            {
#if false
                if (chunkMap.TryGetValue(chunkId, out chunkCB))
#else
                if (!GetChunkCB(chunkList[chunkListIdx], out chunkCB))
#endif
                {
                    chunkCB      = new ChunkCB();
                    chunkCB.sha1 = PackChunking.chunkToSha1(chunkList[chunkListIdx]);
                    chunkCB.size = (uint)PackChunking.chunkToLen(chunkList[chunkListIdx]);
                    fileManager.AddNewChunk(data, (uint)offset, chunkCB.size, chunkList[chunkListIdx], out chunkCB.fo);
#if false
                    {
                        byte[] buff2log = new byte[chunkCB.size];
                        for (int i = 0; i < chunkCB.size; i++)
                        {
                            buff2log[i] = data[i + offset];
                        }
                        LogUtility.LogUtility.LogBinary("_stored", buff2log);
                    }
#endif
                    chunkCB.hint           = PackChunking.GetChunkHint(data, (uint)offset, chunkCB.size);
                    chunkCB.chains         = new long[1];
                    chunkCB.chains[0]      = chunkAndChainManager.chainId4Lookup;
                    chunkCB.ChainsListSize = 1;
//                    LogUtility.LogUtility.LogFile("****creating new chunk CB " + Convert.ToString(chunkCB.size) + " " + Convert.ToString(chunkList[chunkListIdx]) + " len " + Convert.ToString(PackChunking.chunkToLen(chunkList[chunkListIdx])) + " hint " + Convert.ToString(chunkCB.hint) + " sha1 " + Convert.ToString(chunkCB.sha1), ModuleLogLevel);
                    ChunksCreated++;
                }
                else if (!ChainIsPresentInChunkCB(chunkCB, chunkAndChainManager.chainId4Lookup))
                {
                    presentChunks = true;
                    long[] newChainList = new long[chunkCB.chains.Length + 1];
                    chunkCB.chains.CopyTo(newChainList, 0);
                    newChainList[chunkCB.chains.Length] = chunkAndChainManager.chainId4Lookup;
                    chunkCB.chains = newChainList;
                    chunkCB.ChainsListSize++;
//                    LogUtility.LogUtility.LogFile("****updating existing chunk CB " + Convert.ToString(chunkCB.size) + " " + Convert.ToString(chunkList[chunkListIdx]) + " hint " + Convert.ToString(chunkCB.hint) + " sha1 " + Convert.ToString(chunkCB.sha1), ModuleLogLevel);
                    ChainAdded2ExistingChunk++;
                }
                else
                {
//                    LogUtility.LogUtility.LogFile("****chain exists in chunk CB " + Convert.ToString(chunkList[chunkListIdx]) + " " + Convert.ToString(PackChunking.chunkToLen(chunkList[chunkListIdx])) + " chain " + Convert.ToString(chainId4Lookup) + " hint " + Convert.ToString(chunkCB.hint) + " sha1 " + Convert.ToString(chunkCB.sha1), ModuleLogLevel);
                    offset += (int)chunkCB.size;
                    ChainExistInChunkChainsList++;
                    continue;
                }
                offset += (int)chunkCB.size;
                GetChunkCBAsByteArray(chunkCB, out buf);
                fileManager.AddUpdateChunkCtrlFile(chunkList[chunkListIdx], buf, 0, (uint)buf.Length);
            }
            if (!GetChain(chunkAndChainManager.chainId4Lookup, out chain4Lookup))
            {
                LogUtility.LogUtility.LogFile("****cannot get chain!!! " + Convert.ToString(chunkAndChainManager.chainId4Lookup), ModuleLogLevel);
                CannotGetChain++;
                return(presentChunks);
            }
//            LogUtility.LogUtility.LogFile("prepare chain, now " + Convert.ToString(chain4Lookup.NumberOfEntries) + " ( " + Convert.ToString(chain4Lookup.chunkIds.Length) + ")" + " will be added " + Convert.ToString(chunkList.Count-savedChunkListIdx), ModuleLogLevel);
            long[] newChunkList = new long[chain4Lookup.NumberOfEntries + ((LastChunkId + 1) - savedChunkListIdx)];
            chain4Lookup.chunkIds.CopyTo(newChunkList, 0);
            idx = (int)chain4Lookup.NumberOfEntries;
            for (chunkListIdx = savedChunkListIdx; chunkListIdx <= LastChunkId; chunkListIdx++)
            {
#if false
                if (ChunkIsPresentInChain(chain4Lookup, chunkList[chunkListIdx]))
                {
                    LogUtility.LogUtility.LogFile("SANITY CHECK: chunk " + Convert.ToString(chunkList[chunkListIdx]) + " len " + Convert.ToString(PackChunking.chunkToLen(chunkList[chunkListIdx])) + " is already in chain", ModuleLogLevel);
                    return(presentChunks);
                }
#endif
                newChunkList[idx++] = chunkList[chunkListIdx];
            }
            chain4Lookup.chunkIds        = newChunkList;
            chain4Lookup.NumberOfEntries = (uint)newChunkList.Length;
            GetChainAsByteArray(chain4Lookup.chunkIds, out buf);
            //save on disk
            chunkAndChainManager.chainId4Lookup = fileManager.AddUpdateChainFile(buf, chunkAndChainManager.chainId4Lookup);
            return(presentChunks);
        }