//[TestMore]
public bool RemoveData(string key)
{
long iIdx = 0;
int iLen = 0, CurrentIndexSize = GetIndexSize();
if (!ExistsDataKey(key, out iIdx, out iLen))
{
return(false);
}
else
{
//remove index data, make as dirty
IndexObject.Remove(key);
SetKeyCount(GetKeyCount() - 1);
byte[] idxBytes = FileWrapHelper.GetBytes(IndexObject);
//更新索引内容实际大小
if (UpdateDynamicBytes((long)GetNextIndexWriteOffset(), idxBytes, CurrentIndexSize, 24))
{
IncrementIndexSize();
UpdateDynamicBytes((long)GetNextIndexWriteOffset(), idxBytes, CurrentIndexSize, 24);
_currentDatOffset = GetOffSetDat <int>(_internalReader, 12);
CurrentIndexSize = GetIndexSize();
}
#region Update Dirty Block
SortedList <long, DirtyBlock> dObjs = GetStoreDirtyData();
DirtyBlock dBlock = new DirtyBlock {
DataIndex = iIdx, Length = iLen
};
if (DirtyObject.ContainsKey(dBlock.DataIndex))
{
//[impossible]
DirtyObject[dBlock.DataIndex] = dBlock;
}
else
{
DirtyObject.Add(dBlock.DataIndex, dBlock);
}
byte[] dBytes = FileWrapHelper.GetBytes(DirtyObject);
if (UpdateDynamicBytes((long)(HEAD_SUMMARY_BYTES + CurrentIndexSize), dBytes, MAX_DIRTYBLOCK_SIZE, HEAD_SUMMARY_BYTES - 9))
{
ClearDirtyData();
UpdateDynamicBytes((long)(HEAD_SUMMARY_BYTES + CurrentIndexSize), dBytes, MAX_DIRTYBLOCK_SIZE, HEAD_SUMMARY_BYTES - 9);
}
#endregion
return(true);
}
}
internal SortedList <string, KeyValueState> GetIndexObject(int?idxCount)
{
if (IndexObject == null)
{
byte[] idxBytes = ReadData((long)GetNextIndexWriteOffset(),
(idxCount.HasValue && idxCount.Value != 0) ? idxCount.Value : GetIndexRealSize());
//Console.WriteLine("Idx Len: {0}", idxBytes.Length);
if (idxBytes.Length > 0)
{
//Console.WriteLine("Restore");
IndexObject = FileWrapHelper.GetObject <SortedList <string, KeyValueState> >(idxBytes);
}
else
{
IndexObject = new SortedList <string, KeyValueState>(StringComparer.Ordinal);
}
}
return(IndexObject);
}
internal SortedList <long, DirtyBlock> GetStoreDirtyData()
{
if (DirtyObject == null)
{
int dCount = GetDirtyBlockRealSize();
if (dCount > 0)
{
byte[] idxBytes = ReadData((long)(HEAD_SUMMARY_BYTES + GetIndexSize()), dCount);
if (idxBytes.Length > 0)
{
DirtyObject = FileWrapHelper.GetObject <SortedList <long, DirtyBlock> >(idxBytes);
}
}
else
{
DirtyObject = new SortedList <long, DirtyBlock>();
}
}
return(DirtyObject);
}
private void BuildEmptyFile()
{
_internalWriter.Write(Encoding.ASCII.GetBytes("KVS 1.0 ")); //文件版本 +8
_internalWriter.Write(BitConverter.GetBytes(HeadIndexLength)); //索引空间长度 +4
_internalWriter.Write(BitConverter.GetBytes((int)0)); //数据索引偏移量 +4
_internalWriter.Write(BitConverter.GetBytes((long)0)); //下次写入数据索引 +8
byte[] idxBytes = FileWrapHelper.GetBytes(new SortedList <string, KeyValueState>(StringComparer.Ordinal));
KeepPositionWrite(HEAD_SUMMARY_BYTES, idxBytes);
_internalWriter.Write(BitConverter.GetBytes(idxBytes.Length)); //索引有效数据长度 +4
_internalWriter.Write(BitConverter.GetBytes(MAX_DIRTYBLOCK_SIZE)); //DirtyBlock空间长度 +4
byte[] dbBytes = FileWrapHelper.GetBytes(new SortedList <long, DirtyBlock>());
KeepPositionWrite(HEAD_SUMMARY_BYTES + HeadIndexLength, dbBytes);
_internalWriter.Write(BitConverter.GetBytes(dbBytes.Length)); //DirtyBlock有效数据长度 +4
_internalWriter.Write(BitConverter.GetBytes((int)0)); //所有键值总数 +4
_internalWriter.Write('\n'); // +1
}
public static void DecryptStream(Stream fin, Stream fout, string password, WrapProgressCallBack callback)
{
int size = (int)fin.Length; // the size of the file for progress notification
byte[] bytes = new byte[BUFFER_SIZE]; // byte buffer
int read = -1; // the amount of bytes read from the stream
int value = 0;
int outValue = 0; // the amount of bytes written out
// read off the IV and Salt
byte[] IV = new byte[16];
fin.Read(IV, 0, 16);
byte[] salt = new byte[16];
fin.Read(salt, 0, 16);
// create the crypting stream
SymmetricAlgorithm sma = FileWrapHelper.CreateRijndael(password, salt);
sma.IV = IV;
value = 32; // the value for the progress
long lSize = -1; // the size stored in the input stream
// create the hashing object, so that we can verify the file
HashAlgorithm hasher = SHA256.Create();
// create the cryptostreams that will process the file
using (CryptoStream cin = new CryptoStream(fin, sma.CreateDecryptor(), CryptoStreamMode.Read),
chash = new CryptoStream(Stream.Null, hasher, CryptoStreamMode.Write))
{
// read size from file
BinaryReader br = new BinaryReader(cin);
lSize = br.ReadInt64();
ulong tag = br.ReadUInt64();
if (FC_TAG != tag)
{
throw new FileWrapException("File Corrupted!");
}
//determine number of reads to process on the file
long numReads = lSize / BUFFER_SIZE;
// determine what is left of the file, after numReads
long slack = (long)lSize % BUFFER_SIZE;
// read the buffer_sized chunks
for (int i = 0; i < numReads; ++i)
{
read = cin.Read(bytes, 0, bytes.Length);
fout.Write(bytes, 0, read);
chash.Write(bytes, 0, read);
value += read;
outValue += read;
if (callback != null)
{
callback(0, size, value);
}
}
// now read the slack
if (slack > 0)
{
read = cin.Read(bytes, 0, (int)slack);
fout.Write(bytes, 0, read);
chash.Write(bytes, 0, read);
value += read;
outValue += read;
if (callback != null)
{
callback(0, size, value);
}
}
// flush and close the hashing stream
chash.Flush();
chash.Close();
// flush and close the output file
fout.Flush();
fout.Close();
// read the current hash value
byte[] curHash = hasher.Hash;
// get and compare the current and old hash values
byte[] oldHash = new byte[hasher.HashSize / 8];
read = cin.Read(oldHash, 0, oldHash.Length);
if ((oldHash.Length != read) || (!CheckByteArrays(oldHash, curHash)))
{
throw new FileWrapException("File Corrupted!");
}
}
// make sure the written and stored size are equal
if (outValue != lSize)
{
throw new FileWrapException("File Sizes don't match!");
}
}
public static void EncryptStream(Stream fin, Stream fout, string password, WrapProgressCallBack callback)
{
long lSize = fin.Length; // the size of the input file for storing
int size = (int)lSize; // the size of the input file for progress
byte[] bytes = new byte[BUFFER_SIZE]; // the buffer
int read = -1; // the amount of bytes read from the input file
int value = 0; // the amount overall read from the input file for progress
// generate IV and Salt
byte[] IV = GenerateRandomBytes(16);
byte[] salt = GenerateRandomBytes(16);
// create the crypting object
SymmetricAlgorithm sma = FileWrapHelper.CreateRijndael(password, salt);
sma.IV = IV;
// write the IV and salt to the beginning of the file
fout.Write(IV, 0, IV.Length);
fout.Write(salt, 0, salt.Length);
// create the hashing and crypto streams
HashAlgorithm hasher = SHA256.Create();
using (CryptoStream cout = new CryptoStream(fout, sma.CreateEncryptor(), CryptoStreamMode.Write),
chash = new CryptoStream(Stream.Null, hasher, CryptoStreamMode.Write))
{
// write the size of the file to the output file
BinaryWriter bw = new BinaryWriter(cout);
bw.Write(lSize);
// write the file cryptor tag to the file
bw.Write(FC_TAG);
// read and the write the bytes to the crypto stream in BUFFER_SIZEd chunks
while ((read = fin.Read(bytes, 0, bytes.Length)) != 0)
{
cout.Write(bytes, 0, read);
chash.Write(bytes, 0, read);
value += read;
if (callback != null)
{
callback(0, size, value);
}
}
// flush and close the hashing object
chash.Flush();
chash.Close();
// read the hash
byte[] hash = hasher.Hash;
// write the hash to the end of the file
cout.Write(hash, 0, hash.Length);
// flush and close the cryptostream
cout.Flush();
cout.Close();
cout.Dispose();
}
}
//[TestMore]
public bool StoreKeyData(string key, byte[] kDat)
{
long iIdx = 0;
int iLen = 0, CurrentIndexSize = GetIndexSize();
//Read
long nDataIndex = GetOffSetDat <long>(_internalReader, 16);
if (nDataIndex == 0)
{
nDataIndex = HEAD_SUMMARY_BYTES + (long)CurrentIndexSize + MAX_DIRTYBLOCK_SIZE;
}
//Console.WriteLine("Store Index: {0}", nDataIndex);
bool addNew = false, append = false;
SortedList <string, KeyValueState> idxObj = GetIndexObject(null);
KeyValueState kState = new KeyValueState();
kState.Key = key;
kState.Length = kDat.Length;
if (idxObj == null)
{
IndexObject = new SortedList <string, KeyValueState>(StringComparer.Ordinal);
addNew = true;
}
else
{
if (!ExistsDataKey(key, out iIdx, out iLen))
{
addNew = true;
append = true;
//Console.WriteLine("Note Exists Key" + key);
}
else
{
KeyValueState oldState = IndexObject[key];
if ((oldState.Length + oldState.ChipSize) >= kDat.Length)
{
append = false;
//Location Update
nDataIndex = oldState.DataIndex + _currentDatOffset;
kState.DataIndex = oldState.DataIndex;
kState.ChipSize = (oldState.Length + oldState.ChipSize) - kDat.Length;
}
else
{
kState.ChipSize = 0;
Console.WriteLine("old:{0}, new:{1}, other:{2}", oldState.DataIndex, nDataIndex, nDataIndex - _currentDatOffset);
if (nDataIndex == (oldState.DataIndex + _currentDatOffset + oldState.Length + oldState.ChipSize))
{
append = false;
nDataIndex = oldState.DataIndex + _currentDatOffset;
kState.DataIndex = oldState.DataIndex;
//下次写入数据索引
KeepPositionWrite(16, BitConverter.GetBytes(nDataIndex + kDat.LongLength));
//Console.WriteLine("Location Append");
}
else
{
append = true;
SortedList <long, DirtyBlock> dirtyDat = GetStoreDirtyData();
#region Dirty Block
DirtyBlock dBlock = new DirtyBlock {
DataIndex = oldState.DataIndex, Length = oldState.Length + kState.ChipSize
};
//Console.WriteLine("Dirty: {0}+{1}", dBlock.DataIndex, dBlock.Length);
if (DirtyObject.ContainsKey(dBlock.DataIndex))
{
//[impossible]
DirtyObject[dBlock.DataIndex] = dBlock;
}
else
{
DirtyObject.Add(dBlock.DataIndex, dBlock);
}
byte[] dBytes = FileWrapHelper.GetBytes(DirtyObject);
if (UpdateDynamicBytes((long)(HEAD_SUMMARY_BYTES + CurrentIndexSize), dBytes, MAX_DIRTYBLOCK_SIZE, HEAD_SUMMARY_BYTES - 9))
{
ClearDirtyData();
UpdateDynamicBytes((long)(HEAD_SUMMARY_BYTES + CurrentIndexSize), dBytes, MAX_DIRTYBLOCK_SIZE, HEAD_SUMMARY_BYTES - 9);
}
#endregion
}
}
}
}
if (append)
{
//在DirtyBlock总查找可用空间(TODO)
storeStream.SetLength(storeStream.Length + kDat.Length);
//下次写入数据索引
KeepPositionWrite(16, BitConverter.GetBytes(nDataIndex + kDat.LongLength));
//Console.WriteLine("test:{0}, {1}, {2}", nDataIndex, _currentDatOffset, nDataIndex - _currentDatOffset);
kState.DataIndex = nDataIndex - _currentDatOffset;
}
if (addNew)
{
kState.ChipSize = 0;
IndexObject.Add(key, kState);
SetKeyCount(GetKeyCount() + 1);
//Console.WriteLine("ADD New: Set {0}", GetKeyCount());
}
else
{
IndexObject[key] = kState;
}
byte[] idxBytes = FileWrapHelper.GetBytes(IndexObject);
bool blnIndxAdd = false;
//更新索引内容实际大小
if (UpdateDynamicBytes((long)GetNextIndexWriteOffset(), idxBytes, CurrentIndexSize, 24))
{
IncrementIndexSize();
nDataIndex += INDEX_INCREMENT_STEP;
UpdateDynamicBytes((long)GetNextIndexWriteOffset(), idxBytes, CurrentIndexSize, 24);
_currentDatOffset = GetOffSetDat <int>(_internalReader, 12);
blnIndxAdd = true;
}
//Console.WriteLine("DataOffSet: {0}", _currentDatOffset);
//Console.WriteLine("Store Index: {0}", nDataIndex);
KeepPositionWrite(nDataIndex, kDat);
//修改下次写入位置
if (blnIndxAdd == true)
{
KeepPositionWrite(16, BitConverter.GetBytes(nDataIndex + kDat.LongLength));
}
return(true);
}
