/// <summary>
/// Extract the masterstream to mimic the folder structure the AzureAsyncAppender
/// </summary>
/// <returns></returns>
public async Task ExtractMasterStream(DirectoryInfo output, IAppenderNamingPolicy namingPolicy, long afterVersion, int maxCount, bool writeIndexFile = true)
{
var events = await this.ReadRecords(afterVersion, maxCount);
var versions = new Dictionary<string, int>();
var rootindex = new FileInfo(Path.Combine(output.FullName, namingPolicy.GetIndexPath("master")));
if (!output.Exists)
{
output.Create();
}
using (var masterindexwriter = new StreamWriter(System.IO.File.OpenWrite(rootindex.FullName)))
{
foreach (var @event in events)
{
var streampath = namingPolicy.GetStreamPath(@event.Name);
var filename = new FileInfo(Path.Combine(output.FullName, streampath));
if (!versions.ContainsKey(@event.Name))
{
versions[@event.Name] = 1;
}
else
{
versions[@event.Name]++;
}
var record = new FileRecord(@event.Data, @event.Name, versions[@event.Name]);
if (!filename.Directory.Exists)
{
filename.Directory.Create();
}
using (var fs = System.IO.File.OpenWrite(filename.FullName))
{
record.WriteContentToStream(fs);
}
if (writeIndexFile)
{
var indexfile = Path.Combine(output.FullName, namingPolicy.GetIndexPath(@event.Name));
var hash = string.Join("", record.Hash.Select(x => x.ToString("x2")));
FileStream fileStream = System.IO.File.Exists(indexfile) ? System.IO.File.Open(indexfile, FileMode.Append, FileAccess.Write) : System.IO.File.Create(indexfile);
using (var streamindex = new StreamWriter(fileStream))
{
streamindex.WriteLine(String.Join("\t", record.Name, record.Version, hash, streampath));
masterindexwriter.WriteLine(String.Join("\t", record.Name, record.Version, hash, streampath));
streamindex.Flush();
}
}
}
masterindexwriter.Flush();
}
}
public async Task AppendWriteNoLease(FileRecord record, string filename)
{
using (var mem = new MemoryStream())
{
if (lease != null)
{
mem.Write(existingdata, 0, existingdata.Length);
var writer = new StreamWriter(mem);
var hashhex = string.Join(String.Empty, record.Hash.Select(x => x.ToString("x2")));
writer.WriteLine(String.Join("\t", record.Name, record.Version, hashhex, filename));
writer.Flush();
mem.Position = 0;
await this.blob.UploadFromStreamAsync(mem, new AccessCondition() { LeaseId = this.lease }, new BlobRequestOptions(), null);
}
else
{
var writer = new StreamWriter(mem);
var hashhex = string.Join(String.Empty, record.Hash.Select(x => x.ToString("x2")));
writer.WriteLine(String.Join("\t", record.Name, record.Version, hashhex, filename));
writer.Flush();
mem.Position = 0;
await this.blob.AppendBlockAsync(mem);
}
}
}
public async Task AppendWrite(FileRecord record, string filename)
{
await GetLeaseAndRead();
await AppendWriteNoLease(record, filename);
}
public async Task AppendWrite(FileRecord record, string filename)
{
await this.index.AppendWriteNoLease(record, filename);
}
public List<FileRecord> Read(string filename)
{
var list = new List<FileRecord>();
int version = 1;
if (!System.IO.File.Exists(filename))
{
return list;
}
using (var fs = System.IO.File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (var reader = new BinaryReader(fs))
{
while (fs.Position < fs.Length)
{
try
{
var record = new FileRecord(version);
record.ReadContentFromStream(reader);
list.Add(record);
version++;
}
catch (IOException)
{
}
}
}
}
return list;
}
public async Task Append(string streamName, byte[] data, long expectedStreamVersion = -1)
{
var path = new FileInfo(this.options.FileStream(options.MasterStreamName));
var streampath = this.options.FileStream(streamName);
FileStream filestream;
if (!path.Directory.Exists)
{
path.Directory.Create();
}
if (!System.IO.File.Exists(path.FullName))
{
filestream = System.IO.File.Open(path.FullName, FileMode.CreateNew, FileAccess.Write, FileShare.Write);
}
else
{
filestream = System.IO.File.Open(path.FullName, FileMode.Append, FileAccess.Write, FileShare.Write);
}
using (Stream s = filestream)
{
var record = new FileRecord(data, streamName, expectedStreamVersion + 1);
await record.WriteContentToStreamAsync(s);
}
}
public async Task WriteContent(string name, byte[] content, long expectedStreamVersion, int retry = 0)
{
this.index = new FileIndexLock(storage, container, options.NamingPolicy.GetIndexPath(name));
try
{
//if (expectedStreamVersion == 0)
//{
// await this.index.CreateIfNotExist();
// await this.masterindex.CreateIfNotExist();
//}
var canWrite = await index.GetLeaseAndRead(expectedStreamVersion != 0);
if (!canWrite)
{
if (retry < options.RetryPolicy.Length)
{
await Task.Delay(retry);
await this.WriteContent(name, content, expectedStreamVersion, retry++);
return;
}
await this.index.CreateIfNotExist();
await this.index.ReadIndex();
throw new AppendOnlyTimeoutException(expectedStreamVersion, index.OrderedIndex.Any() ? this.index.ReadLast().Version : 0, name);
}
await this.index.ReadIndex();
var last = index.ReadLast();
if ((last == null && expectedStreamVersion != 0) || (last != null && expectedStreamVersion != last.Version) || (last != null && last.Version != expectedStreamVersion ))
{
throw new AppendOnlyStoreConcurrencyException(expectedStreamVersion, last?.Version ?? 0, name);
}
var blobcache = GetStreamCache(name);
if (last == null)
{
//no file, create the stream to append to...
await blobcache.CreateOrReplaceAsync();
}
var master = this.GetMasterCache();
using (var memstream = new MemoryStream())
{
var record = new FileRecord(content, name, expectedStreamVersion + 1);
record.WriteContentToStream(memstream);
var mybytes = memstream.ToArray();
var appendmaster = Task.Run(async () =>
{
using (var masterstream = new MemoryStream(memstream.ToArray()))
{
await master.AppendBlockAsync(masterstream);
}
});
var up1 = blobcache.AppendFromByteArrayAsync(mybytes, 0, mybytes.Length);
var up3 = index.AppendWrite(record, blobcache.Name);
var up4 = masterindex.AppendWrite(record, blobcache.Name);
await index.EnsureLease();
if (!options.DisableMasterIndex)
{
await Task.WhenAll(up1, appendmaster, up3);
}
else
{
await Task.WhenAll(up4, up1, appendmaster, up3);
}
}
}
finally
{
await index.Release();
}
}
private static async Task<List<FileRecord>> ReadStreamToRecords(CloudAppendBlob blob)
{
List<FileRecord> records = new List<FileRecord>();
using (var s = new MemoryStream())
{
if (!await blob.ExistsAsync())
{
return records;
}
await blob.DownloadRangeToStreamAsync(s, 0, blob.Properties.Length);
s.Position = 0;
var reader = new BinaryReader(s);
var record = new FileRecord();
while (record.ReadContentFromStream(reader))
{
records.Add(record);
record = new FileRecord();
}
}
return records;
}