/// <summary> /// Extract the masterstream to mimic the folder structure the AzureAsyncAppender /// </summary> /// <returns></returns> public async Task ExtractMasterStream(DirectoryInfo output, IAppenderNamingPolicy namingPolicy, long afterVersion, int maxCount, bool writeIndexFile = true) { var events = await this.ReadRecords(afterVersion, maxCount); var versions = new Dictionary<string, int>(); var rootindex = new FileInfo(Path.Combine(output.FullName, namingPolicy.GetIndexPath("master"))); if (!output.Exists) { output.Create(); } using (var masterindexwriter = new StreamWriter(System.IO.File.OpenWrite(rootindex.FullName))) { foreach (var @event in events) { var streampath = namingPolicy.GetStreamPath(@event.Name); var filename = new FileInfo(Path.Combine(output.FullName, streampath)); if (!versions.ContainsKey(@event.Name)) { versions[@event.Name] = 1; } else { versions[@event.Name]++; } var record = new FileRecord(@event.Data, @event.Name, versions[@event.Name]); if (!filename.Directory.Exists) { filename.Directory.Create(); } using (var fs = System.IO.File.OpenWrite(filename.FullName)) { record.WriteContentToStream(fs); } if (writeIndexFile) { var indexfile = Path.Combine(output.FullName, namingPolicy.GetIndexPath(@event.Name)); var hash = string.Join("", record.Hash.Select(x => x.ToString("x2"))); FileStream fileStream = System.IO.File.Exists(indexfile) ? System.IO.File.Open(indexfile, FileMode.Append, FileAccess.Write) : System.IO.File.Create(indexfile); using (var streamindex = new StreamWriter(fileStream)) { streamindex.WriteLine(String.Join("\t", record.Name, record.Version, hash, streampath)); masterindexwriter.WriteLine(String.Join("\t", record.Name, record.Version, hash, streampath)); streamindex.Flush(); } } } masterindexwriter.Flush(); } }
public async Task AppendWriteNoLease(FileRecord record, string filename) { using (var mem = new MemoryStream()) { if (lease != null) { mem.Write(existingdata, 0, existingdata.Length); var writer = new StreamWriter(mem); var hashhex = string.Join(String.Empty, record.Hash.Select(x => x.ToString("x2"))); writer.WriteLine(String.Join("\t", record.Name, record.Version, hashhex, filename)); writer.Flush(); mem.Position = 0; await this.blob.UploadFromStreamAsync(mem, new AccessCondition() { LeaseId = this.lease }, new BlobRequestOptions(), null); } else { var writer = new StreamWriter(mem); var hashhex = string.Join(String.Empty, record.Hash.Select(x => x.ToString("x2"))); writer.WriteLine(String.Join("\t", record.Name, record.Version, hashhex, filename)); writer.Flush(); mem.Position = 0; await this.blob.AppendBlockAsync(mem); } } }
public async Task AppendWrite(FileRecord record, string filename) { await GetLeaseAndRead(); await AppendWriteNoLease(record, filename); }
public async Task AppendWrite(FileRecord record, string filename) { await this.index.AppendWriteNoLease(record, filename); }
public List<FileRecord> Read(string filename) { var list = new List<FileRecord>(); int version = 1; if (!System.IO.File.Exists(filename)) { return list; } using (var fs = System.IO.File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { using (var reader = new BinaryReader(fs)) { while (fs.Position < fs.Length) { try { var record = new FileRecord(version); record.ReadContentFromStream(reader); list.Add(record); version++; } catch (IOException) { } } } } return list; }
public async Task Append(string streamName, byte[] data, long expectedStreamVersion = -1) { var path = new FileInfo(this.options.FileStream(options.MasterStreamName)); var streampath = this.options.FileStream(streamName); FileStream filestream; if (!path.Directory.Exists) { path.Directory.Create(); } if (!System.IO.File.Exists(path.FullName)) { filestream = System.IO.File.Open(path.FullName, FileMode.CreateNew, FileAccess.Write, FileShare.Write); } else { filestream = System.IO.File.Open(path.FullName, FileMode.Append, FileAccess.Write, FileShare.Write); } using (Stream s = filestream) { var record = new FileRecord(data, streamName, expectedStreamVersion + 1); await record.WriteContentToStreamAsync(s); } }
public async Task WriteContent(string name, byte[] content, long expectedStreamVersion, int retry = 0) { this.index = new FileIndexLock(storage, container, options.NamingPolicy.GetIndexPath(name)); try { //if (expectedStreamVersion == 0) //{ // await this.index.CreateIfNotExist(); // await this.masterindex.CreateIfNotExist(); //} var canWrite = await index.GetLeaseAndRead(expectedStreamVersion != 0); if (!canWrite) { if (retry < options.RetryPolicy.Length) { await Task.Delay(retry); await this.WriteContent(name, content, expectedStreamVersion, retry++); return; } await this.index.CreateIfNotExist(); await this.index.ReadIndex(); throw new AppendOnlyTimeoutException(expectedStreamVersion, index.OrderedIndex.Any() ? this.index.ReadLast().Version : 0, name); } await this.index.ReadIndex(); var last = index.ReadLast(); if ((last == null && expectedStreamVersion != 0) || (last != null && expectedStreamVersion != last.Version) || (last != null && last.Version != expectedStreamVersion )) { throw new AppendOnlyStoreConcurrencyException(expectedStreamVersion, last?.Version ?? 0, name); } var blobcache = GetStreamCache(name); if (last == null) { //no file, create the stream to append to... await blobcache.CreateOrReplaceAsync(); } var master = this.GetMasterCache(); using (var memstream = new MemoryStream()) { var record = new FileRecord(content, name, expectedStreamVersion + 1); record.WriteContentToStream(memstream); var mybytes = memstream.ToArray(); var appendmaster = Task.Run(async () => { using (var masterstream = new MemoryStream(memstream.ToArray())) { await master.AppendBlockAsync(masterstream); } }); var up1 = blobcache.AppendFromByteArrayAsync(mybytes, 0, mybytes.Length); var up3 = index.AppendWrite(record, blobcache.Name); var up4 = masterindex.AppendWrite(record, blobcache.Name); await index.EnsureLease(); if (!options.DisableMasterIndex) { await Task.WhenAll(up1, appendmaster, up3); } else { await Task.WhenAll(up4, up1, appendmaster, up3); } } } finally { await index.Release(); } }
private static async Task<List<FileRecord>> ReadStreamToRecords(CloudAppendBlob blob) { List<FileRecord> records = new List<FileRecord>(); using (var s = new MemoryStream()) { if (!await blob.ExistsAsync()) { return records; } await blob.DownloadRangeToStreamAsync(s, 0, blob.Properties.Length); s.Position = 0; var reader = new BinaryReader(s); var record = new FileRecord(); while (record.ReadContentFromStream(reader)) { records.Add(record); record = new FileRecord(); } } return records; }