IEnumerable <StorageFrameDecoded> EnumerateHistory()
{
// cleanup old pending files
// load indexes
// build and save missing indexes
var datFiles = _info.EnumerateFiles("*.dat");
foreach (var fileInfo in datFiles.OrderBy(fi => fi.Name))
{
// quick cleanup
if (fileInfo.Length == 0)
{
fileInfo.Delete();
continue;
}
using (var reader = fileInfo.OpenRead())
{
StorageFrameDecoded result;
while (StorageFramesEvil.TryReadFrame(reader, out result))
{
yield return(result);
}
}
}
}
IEnumerable <StorageFrameDecoded> EnumerateHistory()
{
// cleanup old pending files
// load indexes
// build and save missing indexes
var datFiles = _container
.ListBlobs(new BlobRequestOptions()
{
BlobListingDetails = BlobListingDetails.Metadata
})
.OrderBy(s => s.Uri.ToString())
.OfType <CloudPageBlob>()
.Where(s => s.Name.EndsWith(".dat"));
foreach (var fileInfo in datFiles)
{
var bytes = fileInfo.DownloadByteArray();
bool potentiallyNonTruncatedChunk = bytes.Length % _pageSizeMultiplier == 0;
long lastValidPosition = 0;
using (var stream = new MemoryStream(bytes))
{
StorageFrameDecoded result;
while (StorageFramesEvil.TryReadFrame(stream, out result))
{
lastValidPosition = stream.Position;
yield return(result);
}
}
var haveSomethingToTruncate = bytes.Length - lastValidPosition >= 512;
if (potentiallyNonTruncatedChunk & haveSomethingToTruncate)
{
TruncateBlob(lastValidPosition, fileInfo);
}
}
}
public void read_empty_frame()
{
Stream stream = new MemoryStream();
StorageFrameDecoded decoded;
var isreadFrame = StorageFramesEvil.TryReadFrame(stream, out decoded);
Assert.IsFalse(isreadFrame);
}
public void async_read_write_more_frame()
{
//GIVEN
string msg = "test message";
var path = Path.Combine(Path.GetTempPath(), "lokad-cqrs", Guid.NewGuid() + ".pb");
Directory.CreateDirectory(Path.GetDirectoryName(path));
const int maxIndex = 100;
var writeTask = Task.Factory.StartNew(() =>
{
using (var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite))
{
for (int i = 0; i < maxIndex; i++)
{
StorageFramesEvil.WriteFrame("test-key" + i, i, Encoding.UTF8.GetBytes(msg + i), stream);
}
}
});
//WHEN
int index = 0;
var readTask = Task.Factory.StartNew(() =>
{
using (var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Read, FileShare.ReadWrite))
{
while (index < maxIndex)
{
StorageFrameDecoded decoded;
if (StorageFramesEvil.TryReadFrame(stream, out decoded))
{
Assert.AreEqual("test-key" + index, decoded.Name);
Assert.AreEqual(index, decoded.Stamp);
Assert.AreEqual(msg + index, Encoding.UTF8.GetString(decoded.Bytes));
index++;
}
}
}
});
writeTask.Wait();
readTask.Wait();
Assert.AreEqual(maxIndex, index);
}
IEnumerable <StorageFrameDecoded> EnumerateHistory()
{
// cleanup old pending files
// load indexes
// build and save missing indexes
var datFiles = _info.EnumerateFiles("*.dat");
var fileFrameCollections = new List <BlockingCollection <StorageFrameDecoded> >();
foreach (var fileInfo in datFiles.OrderBy(fi => fi.Name))
{
// quick cleanup
if (fileInfo.Length == 0)
{
fileInfo.Delete();
continue;
}
var fileFrames = new BlockingCollection <StorageFrameDecoded>();
fileFrameCollections.Add(fileFrames);
using (var reader = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 64 * 1024, FileOptions.Asynchronous))
{
byte[] buffer = new byte[fileInfo.Length];
Task.Factory.FromAsync(reader.BeginRead(buffer, 0, buffer.Length, null, null),
r =>
{
reader.EndRead(r);
using (var ms = new MemoryStream(buffer))
{
StorageFrameDecoded result;
while (StorageFramesEvil.TryReadFrame(ms, out result))
{
fileFrames.Add(result);
}
}
fileFrames.CompleteAdding();
});
}
}
return(fileFrameCollections.SelectMany(fileFrameCollection => fileFrameCollection.GetConsumingEnumerable()));
}
private async Task LoadTapeFile(CloudPageBlob fileBlob, BlockingCollection <StorageFrameDecoded> fileFrames)
{
var retryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(0.5), 100);
try
{
var tapeStream = await _policy.Get(() => fileBlob.OpenReadAsync(null, new BlobRequestOptions {
RetryPolicy = retryPolicy, MaximumExecutionTime = TimeSpan.FromMinutes(30), ServerTimeout = TimeSpan.FromMinutes(30)
}, null)).ConfigureAwait(false);
using (var ms = new MemoryStream())
{
tapeStream.CopyToAsync(ms).Wait();
ms.Position = 0;
var potentiallyNonTruncatedChunk = ms.Length % this._pageSizeMultiplier == 0;
long lastValidPosition = 0;
StorageFrameDecoded result;
while (StorageFramesEvil.TryReadFrame(ms, out result))
{
lastValidPosition = ms.Position;
fileFrames.Add(result);
}
var haveSomethingToTruncate = ms.Length - lastValidPosition >= 512;
if (potentiallyNonTruncatedChunk & haveSomethingToTruncate)
{
this.TruncateBlob(lastValidPosition, fileBlob, retryPolicy);
}
}
}
catch (Exception x)
{
this.Log().Error(x, "Error loading tape {0}", fileBlob.Name);
throw;
}
finally
{
fileFrames.CompleteAdding();
}
}
IEnumerable <StorageFrameDecoded> EnumerateHistory()
{
// cleanup old pending files
// load indexes
// build and save missing indexes
var datFiles = _container
.ListBlobs()
.OrderBy(s => s.Uri.ToString())
.OfType <CloudPageBlob>();
foreach (var fileInfo in datFiles)
{
using (var stream = new MemoryStream(fileInfo.DownloadByteArray()))
{
StorageFrameDecoded result;
while (StorageFramesEvil.TryReadFrame(stream, out result))
{
yield return(result);
}
}
}
}