public DataSlice CreateNewDataSlice()
{
DataSlice _slice = new DataSlice(_sliceCount);
_sliceCount++;
return(_slice);
}
// think of a better return value
public DataSlice ReadDataSlice(DataSlice dataSlice)
{
try
{
Impersonation impersonation = null;
if (_isSharedPath)
{
impersonation = new Impersonation(_username, _password);
}
if (_fileStream == null)
{
_fileStream = GetFile();
}
byte[] _data = new byte[dataSlice.SliceHeader.TotalSize];
int offset = 0;
foreach (long segID in dataSlice.SliceHeader.SegmentIds)
{
Segment segment = _segmentManager.GetSegment(segID);
byte[] _splitData = segment.ReadDataSlice(_fileStream, dataSlice.SliceHeader.Id);
try
{
System.Buffer.BlockCopy(_splitData, 0, _data, offset, _splitData.Length);
offset += _splitData.Length;
}
catch (Exception exp)
{
if (LoggerManager.Instance.RecoveryLogger != null &&
LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("BackupFile.ReadDataSlice()", exp.ToString());
}
}
}
dataSlice.Data = _data;
if (_isSharedPath)
{
impersonation.Dispose();
}
}
catch (Exception ex)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("BackupFile.ReadDataSlice()", ex.ToString());
}
}
return(dataSlice);
}
internal List <DataSlice> RecreateSliceMetaInfo(Stream stream)
{
foreach (long key in SegmentHeader.SliceMap.Keys)
{
DataSlice _slice = new DataSlice(key);
long _startingOffset = SegmentHeader.SliceMap[key];
try
{
stream.Seek(_startingOffset, SeekOrigin.Begin);
// read header length
byte[] len = new byte[4];
if (stream.Read(len, 0, 4) > 0)
{
int length = BitConverter.ToInt32(len, 0);
byte[] data = new byte[length];
stream.Read(data, 0, length);
_slice.PopulateHeader(CompactBinaryFormatter.FromByteBuffer(data, string.Empty) as DataSlice.Header);
_slice.SliceHeader.SegmentIds.Add(_segmentHeader.Id);
// internal map
_containingDataSlice.Add(_slice.SliceHeader.Id, _slice);
}
}
catch (Exception ex)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("Segment.RecreateSliceMeta()", ex.ToString());
}
}
}
return(_containingDataSlice.Values.ToList());
}
private void WriteData(DataSlice slice)
{
string fileName;
if (slice.Data != null)
{
try
{
if (slice.SliceHeader.ContentType == Common.Recovery.DataSliceType.Data)
{
fileName = slice.SliceHeader.Database;
}
else
{
fileName = RecoveryFolderStructure.CONFIG_SERVER;
}
BackupFile file = Context.GetBackupFile(fileName);
if (file != null)
{
file.SaveDataSlice(slice);
}
else
{
throw new NullReferenceException("File " + fileName + " cannot be null");
}
}
catch (Exception exp)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("RecoveryIOWriter.Write()", exp.ToString());
}
}
}
}
public void WriteDataSlice(Stream stream, DataSlice dataSlice, long offset, long length = -1)
{
// segment starting offset i.e {[(16mb * segments #)+file_header_space] + (current offset+ reserved_space_for_header+1)}
long _startingOffset = ((BackupFile.HeaderCapacity + ((_dataCapacity + _headerCapacity) * SegmentHeader.Id)) + (_headerCapacity + _currentSize + 1));
//add slice id and starting offset of each slice
if (!_segmentHeader.SliceMap.ContainsKey(dataSlice.SliceHeader.Id))
{
_segmentHeader.SliceMap.Add(dataSlice.SliceHeader.Id, _startingOffset);
}
if (length == -1)
{
dataSlice.SliceHeader.MappedData = dataSlice.Size;
}
else
{
dataSlice.SliceHeader.MappedData = length;
}
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("Segment.Write()", _segmentHeader.ToString() + " Writing to file " + dataSlice.SliceHeader.ToString());
}
try
{
//Calculate crc for data
// seek to point
stream.Seek(_startingOffset, SeekOrigin.Begin);
//1. write slice header
byte[] header = CompactBinaryFormatter.ToByteBuffer(dataSlice.SliceHeader, string.Empty);
int _headerLength = header.Length;
stream.Write(BitConverter.GetBytes(_headerLength), 0, 4);
_currentSize += 4;
stream.Write(header, 0, _headerLength);
_currentSize += _headerLength;
//2. write data as per offset, if length is 0 then write entire data
byte[] data = dataSlice.Data;
stream.Write(BitConverter.GetBytes(length == -1 ? dataSlice.Size : length), 0, 4);
_currentSize += 4;
// write complete data
if (length == -1)
{
int _dataLength = data.Length;
stream.Write(data, 0, _dataLength);
_currentSize += _dataLength;
}
else
{
// write truncated data
byte[] truncated = CopySlice(data, offset, length);
int _dataLength = truncated.Length;
stream.Write(truncated, 0, _dataLength);
_currentSize += _dataLength;
}
//3. if segment is completely filled write segment header
if (EmptySpace < 5)
{
SaveHeader(stream);
}
}
catch (Exception ex)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("Segment.WriteSlice()", ex.ToString());
}
}
}
public void SaveDataSlice(DataSlice dataSlice)
{
try
{
Impersonation impersonation = null;
if (_isSharedPath)
{
impersonation = new Impersonation(_username, _password);
}
if (_fileStream == null)
{
_fileStream = GetFile();
}
//1. get segment from segment manager, 8 is padded for length that is added
SliceFacilitator[] _segmentList = _segmentManager.GetFacilitatingSegments(dataSlice.Size,
dataSlice.HeaderSize, 8);
if (_segmentList.Length > 1)
{
dataSlice.SliceHeader.OverFlow = true;
// dataSlice.SliceHeader.SegmentIds = _segmentList.Select(x => x.Segment.SegmentHeader.Id).Cast<long>().ToArray();
int i = 0;
long offset = 0;
foreach (SliceFacilitator _fac in _segmentList)
{
//4. write slice to file
if (i + 1 < _segmentList.Length)
{
dataSlice.SliceHeader.NextSegmentId = _segmentList[i + 1].Segment.SegmentHeader.Id;
}
else
{
dataSlice.SliceHeader.NextSegmentId = -1;
}
_fac.Segment.SegmentHeader.NumberOfSlices++;
lock (_mutex)
{
_fac.Segment.WriteDataSlice(_fileStream, dataSlice, offset, _fac.Size);
}
offset += _fac.Size;
i++;
}
}
else
{
dataSlice.SliceHeader.OverFlow = false;
dataSlice.SliceHeader.NextSegmentId = -1;
_segmentList[0].Segment.SegmentHeader.NumberOfSlices++;
lock (_mutex)
{
_segmentList[0].Segment.WriteDataSlice(_fileStream, dataSlice, 0);
}
}
if (_isSharedPath)
{
impersonation.Dispose();
}
}
catch (Exception ex)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("BackupFile.SaveDataSlice()", ex.ToString());
}
}
}
public void Add(DataSlice slice)
{
documentCollection.Add(slice);
}
public bool TryTake(out DataSlice dataSlice, int timeout)
{
return(documentCollection.TryTake(out dataSlice, timeout));
}
internal override void Run()
{
// 1. based on job type, select file name
// read that file
// and then read the other file and viola
try
{
switch (Context.JobType)
{
case RecoveryJobType.ConfigRestore:
#region config restore
foreach (BackupFile file in Context.FileList)
{
if (file.Name.Contains(RecoveryFolderStructure.CONFIG_SERVER))
{
IDictionary <long, DataSlice> sliceMap = file.RecreateMetaInfo();
foreach (DataSlice sliceID in sliceMap.Values)
{
// Add data to shared queue
Context.SharedQueue.Add(file.ReadDataSlice(sliceID));
}
file.Close();
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("RecoveryIOReader.Run()", "Config file reading complete");
}
}
}
#endregion
break;
case RecoveryJobType.DataRestore:
#region DataRestore
//1. Select data files
List <BackupFile> dataFile = Context.FileList.Where(x => !x.Name.Contains(RecoveryFolderStructure.OPLOG)).ToList <BackupFile>();
//2. iterate and produce them completely
foreach (BackupFile file in dataFile)
{
IDictionary <long, DataSlice> sliceMap = file.RecreateMetaInfo();
Dictionary <string, string[]> inner = Context.PersistenceConfiguration.DbCollectionMap.Where(x => x.Key.Equals(Context.ActiveDB)).Select(x => x.Value).First();
string[] collectionList = inner.First().Value;
//2. based on config select slices to read
foreach (string collection in collectionList)
{
// get all slices holding data of collection to be restored
List <DataSlice> _slicesToRestore = sliceMap.Where(x => x.Value.SliceHeader.Collection.Equals(collection)).Select(x => x.Value).ToList <DataSlice>();
foreach (DataSlice sliceID in _slicesToRestore)
{
// Add data to shared queue
Context.SharedQueue.Add(file.ReadDataSlice(sliceID));
}
}
file.Close();
}
//3. inform consuming job of complete iteration
DataSlice commandSlice = new DataSlice(999998);
commandSlice.SliceHeader.Collection = "Complete";
commandSlice.SliceHeader.Database = "";
commandSlice.SliceHeader.Cluster = "";
commandSlice.SliceHeader.ContentType = DataSliceType.Command;
commandSlice.Data = CompactBinaryFormatter.ToByteBuffer("Complete_Adding_Data", string.Empty);
Context.SharedQueue.Add(commandSlice);
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("RecoveryIOReader.Run()", "Db file reading complete");
}
//4. wait for message to produce next
while (!Context.SharedQueue.PauseProducing)
{
// wait with time out
}
// check if producing allowed
if (!Context.SharedQueue.PauseProducing)
{
}
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("RecoveryIOReader.Run()", "oplog file reading complete");
}
#endregion
break;
}
// Add command slice
DataSlice finalSlice = new DataSlice(999999);
finalSlice.SliceHeader.Collection = "Complete";
finalSlice.SliceHeader.Database = "";
finalSlice.SliceHeader.Cluster = "";
finalSlice.SliceHeader.ContentType = DataSliceType.Command;
finalSlice.Data = CompactBinaryFormatter.ToByteBuffer("Complete_Adding", string.Empty);
Context.SharedQueue.Add(finalSlice);
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("RecoveryIOReader.Run()", "Complete status set");
}
}
catch (ThreadAbortException)
{
Thread.ResetAbort();
}
catch (Exception exp)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("RecoveryIOReader.Run()", exp.ToString());
}
}
}