public const int CacheCheckpointSize = 3; // cached in remote offset, cached in local, remote max
public CacheFetcher(string sas, string stream, DirectoryInfo folder, IMemoryStreamManager streamManager)
{
StreamName = stream;
_streamManager = streamManager;
var raw = CloudSetup.GetReaderRaw(sas);
_remote = raw.Item2;
_remotePos = raw.Item1;
var streamDir = Path.Combine(folder.FullName, stream);
var di = new DirectoryInfo(streamDir);
if (!di.Exists)
{
di.Create();
}
_outputFile = new FileInfo(Path.Combine(di.FullName, CacheStreamName));
_outputCheckpoint = new FileInfo(Path.Combine(di.FullName, CachePositionName));
_outputFile.Refresh();
_cacheWriter = _outputFile.Open(_outputFile.Exists ? FileMode.Open : FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
_writer = new BinaryWriter(_cacheWriter, CacheFormat);
_cacheChk = new FileCheckpointArrayWriter(_outputCheckpoint, CacheCheckpointSize);
}
static void Backup(string cachePath, string target)
{
var streamFile = new FileInfo(Path.Combine(cachePath, CacheFetcher.CacheStreamName));
var checkFile = new FileInfo(Path.Combine(cachePath, CacheFetcher.CachePositionName));
var posReader = new FileCheckpointArrayReader(checkFile, 2);
var pos = posReader.Read();
Console.WriteLine("Pos {0}/{1}", pos[0], pos[1]);
var dataBackup = Path.Combine(target, CacheFetcher.CacheStreamName + ".gzip");
var checkBackup = Path.Combine(target, CacheFetcher.CachePositionName + ".bak");
Console.WriteLine(new DateTime(2016, 4, 6, 12, 0, 0).ToUniversalTime().ToString("O"));
Console.WriteLine("Will backup to {0}/{1}", dataBackup, checkBackup);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
using (var source = streamFile.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
using (var output = File.Create(dataBackup)) {
using (var gzip = new GZipStream(output, CompressionMode.Compress)) {
CopyStream(source, gzip, pos[0]);
}
}
}
var checkWrite = new FileCheckpointArrayWriter(new FileInfo(checkBackup), 2);
checkWrite.GetOrInitPosition();
checkWrite.Update(pos);
Console.WriteLine("Backed up to {0}", target);
}
static void Repair(string cachePath)
{
var streamFile = new FileInfo(Path.Combine(cachePath, CacheFetcher.CacheStreamName));
var checkFile = new FileInfo(Path.Combine(cachePath, CacheFetcher.CachePositionName));
var replace = new FileInfo(Path.Combine(cachePath, CacheFetcher.CachePositionName + ".fix"));
var posReader = new FileCheckpointArrayReader(checkFile, 2);
var vector = posReader.Read();
var sourceStream = streamFile.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
var reader = new CacheReader(new FixedCheckpointArrayReader(vector), sourceStream, posReader);
var prev = 0L;
var pos = 0L;
var i = 0L;
while (true)
{
try {
var result = reader.ReadAll(pos, 1000, (id, position, maxPosition) => { });
if (result.ReadRecords > 0)
{
prev = pos;
pos = result.CurrentCachePosition;
i += 1;
if ((i % 100) == 0)
{
Console.WriteLine("{0}:{1:F1}GB", i, 1F * pos / 1024 / 1024 / 1024);
}
continue;
}
Console.WriteLine("We are good");
return;
}
catch (InvalidStorageFormatException ex) {
Console.WriteLine("Last known position {0}", pos);
Console.WriteLine("Previous position {0} (-{1} bytes)", prev, pos - prev);
var previousBlock = reader.ReadAll(prev, 1);
var msg = previousBlock.Messages.Single();
var offset = msg.Message.Id.GetOffset();
Console.WriteLine("Previous offset {0}", offset);
var replaceCheck = new FileCheckpointArrayWriter(replace, 2);
replaceCheck.GetOrInitPosition();
replaceCheck.Update(new long[] { prev, offset });
Console.WriteLine("Backup written");
}
break;
}
Console.ReadLine();
}
public static CacheReader ReaderInstance(FileInfo dataFile, FileInfo checkpointFile, FileCheckpointArrayWriter fileCheckpointArrayWriter)
{
var cacheReader = dataFile.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
var checkpointReader = new FileCheckpointArrayReader(checkpointFile, CacheCheckpointSize);
return(new CacheReader(fileCheckpointArrayWriter, cacheReader, checkpointReader));
}
