private void MoveBufferToFile() { // time to dump buffer var offSet = _file.Length; _file.Seek(offSet, SeekOrigin.Begin); _file.Write(_internalBuffer.Buffer, 0, _internalBuffer.UsedStorage); foreach (var tmpKey in _bufferIndexes.Keys) { BinaryStorageKey bufferKeyValue; if (!_bufferIndexes.TryRemove(tmpKey, out bufferKeyValue)) { throw new Exception("Invalid Buffer State [ " + tmpKey + " ]"); } var newPos = offSet + bufferKeyValue.Position; var newLen = bufferKeyValue.Length; BinaryStorageKey tmp; if (_lstIndexes.TryGetValue(tmpKey, out tmp)) { // adjust pos to reflect location in file ;) tmp.Length = newLen; tmp.Position = newPos; _lstIndexes[tmpKey] = tmp; } else { tmp = new BinaryStorageKey { Length = newLen, Position = newPos }; _lstIndexes.TryAdd(tmpKey, tmp); } } // reset buffer and try again ;) _internalBuffer.ResetBuffer(); }
public void Add(string key, T objToAdd) { lock (_opsLock) { try { if (!_bufferIndexes.ContainsKey(key) && !_lstIndexes.ContainsKey(key)) { _itemCnt++; } byte[] data = objToAdd.ToByteArray(); // we need to add to buffer first ;) if (_internalBuffer.CanFitSegment(data.Length)) { // shove into the internal buffer ;) var bufferIdx = _internalBuffer.InsertSegment(data); _bufferIndexes[key] = new BinaryStorageKey { Length = data.Length, Position = bufferIdx }; } else { // dump the buffer to file ;) MoveBufferToFile(); // once buffer is dumped, check for compaction operation ;) // TODO : Background operation so we can continue to fill buffer ;) if (_file.Length - _lastCompactSize > CompactThresholdSize) { Compact(); } if (_internalBuffer.CanFitSegment(data.Length)) { // shove into the internal buffer ;) var bufferIdx = _internalBuffer.InsertSegment(data); _bufferIndexes[key] = new BinaryStorageKey { Length = data.Length, Position = bufferIdx }; } else { // Too big, dump to file directly ;) BinaryStorageKey tmp; if (_lstIndexes.TryGetValue(key, out tmp)) { tmp.Length = data.Length; tmp.Position = _file.Length; _lstIndexes[key] = tmp; } else { tmp = new BinaryStorageKey { Length = data.Length, Position = _file.Length }; _lstIndexes.TryAdd(key, tmp); } _file.Seek(_file.Length, SeekOrigin.Begin); _file.Write(data, 0, data.Length); } } } catch (Exception e) { Dev2Logger.Log.Error(e); } } }
/// <summary> /// Compacts the backing file. /// </summary> /// <exception cref="System.Exception"> /// Compacting data to the temp file failed. /// or /// Unable to close current file for compacting. /// or /// Unable to backup current file for compacting. /// or /// Unable to swap compacted file with the old file. /// </exception> public void Compact() { /* * This method is way too slow, we need to fix the speed issues ;) */ if (!_hasBeenRemoveSinceLastCompact) { return; } Dev2Logger.Log.Debug("Compacting, things should be even slower now! [ " + DateTime.Now + " ]"); // Get tmp file path string directory = Path.GetDirectoryName(_completeFilename); if (directory == null) { throw new Exception( string.Format("Unable to create compact path. '{0}' doesn't contain a valid directory name.", _completeFilename)); } string tempFile = string.Format("{0}.tmp", Guid.NewGuid()); string tempPath = Path.Combine(directory, tempFile); // Open temp file to write entries to try { using (FileStream tmpFileStream = new FileStream(tempPath, FileMode.Create, FileAccess.Write, FileShare.None)) { // Write entries sequentially into the tmp file, this will exclude any removed entries. foreach (var key in Keys.ToList()) { byte[] data = ReadBytes(key); if (data != null && data.Length > 0) { _lstIndexes[key] = new BinaryStorageKey { Length = data.Length, Position = tmpFileStream.Position }; tmpFileStream.Write(data, 0, data.Length); } else { BinaryStorageKey tmp; _lstIndexes.TryRemove(key, out tmp); } } } } catch (Exception e) { throw new Exception("Compacting data to the temp file failed.", e); } // Swap files string backupPath = string.Format("{0}{1}", _completeFilename, ".bak"); try { _file.Close(); _file.Dispose(); } catch (Exception e) { throw new Exception("Unable to close current file for compacting.", e); } try { File.Move(_completeFilename, backupPath); } catch (Exception e) { throw new Exception("Unable to backup current file for compacting.", e); } try { File.Move(tempPath, _completeFilename); } catch (Exception e) { File.Move(backupPath, _completeFilename); File.Delete(tempPath); throw new Exception("Unable to swap compacted file with the old file.", e); } // Remove old file File.Delete(backupPath); // Open file again _file = new FileStream(_completeFilename, FileMode.OpenOrCreate, FileAccess.ReadWrite); _lastCompactSize = _file.Length; _hasBeenRemoveSinceLastCompact = false; Dev2Logger.Log.Debug("Compacting finished! [ " + DateTime.Now + " ]"); }