/// <summary>
        /// Extract the masterstream to mimic the folder structure the AzureAsyncAppender
        /// </summary>
        /// <returns></returns>
        public async Task ExtractMasterStream(DirectoryInfo output, IAppenderNamingPolicy namingPolicy, long afterVersion, int maxCount, bool writeIndexFile = true)
        {
            var events = await this.ReadRecords(afterVersion, maxCount);
            var versions = new Dictionary<string, int>();
            
            var rootindex = new FileInfo(Path.Combine(output.FullName, namingPolicy.GetIndexPath("master")));

            if (!output.Exists)
            {
                output.Create();
            }

            using (var masterindexwriter = new StreamWriter(System.IO.File.OpenWrite(rootindex.FullName)))
            {   
                foreach (var @event in events)
                {
                    var streampath = namingPolicy.GetStreamPath(@event.Name);
                    var filename = new FileInfo(Path.Combine(output.FullName, streampath));

                    if (!versions.ContainsKey(@event.Name))
                    {
                        versions[@event.Name] = 1;
                    }
                    else
                    {
                        versions[@event.Name]++;
                    }

                    var record = new FileRecord(@event.Data, @event.Name, versions[@event.Name]);

                    if (!filename.Directory.Exists)
                    {
                        filename.Directory.Create();
                    }
                    
                    using (var fs = System.IO.File.OpenWrite(filename.FullName))
                    {
                        record.WriteContentToStream(fs);
                    }

                    if (writeIndexFile)
                    {
                        var indexfile = Path.Combine(output.FullName, namingPolicy.GetIndexPath(@event.Name));

                        var hash = string.Join("", record.Hash.Select(x => x.ToString("x2")));

                        FileStream fileStream = System.IO.File.Exists(indexfile) ? System.IO.File.Open(indexfile, FileMode.Append, FileAccess.Write) : System.IO.File.Create(indexfile);
                        using (var streamindex = new StreamWriter(fileStream))
                        {
                            streamindex.WriteLine(String.Join("\t", record.Name, record.Version, hash, streampath));
                            masterindexwriter.WriteLine(String.Join("\t", record.Name, record.Version, hash, streampath));
                            streamindex.Flush();
                        }
                    }
                }

                masterindexwriter.Flush();
            }
        }
Ejemplo n.º 2
0
        public async Task AppendWriteNoLease(FileRecord record, string filename)
        {
            using (var mem = new MemoryStream())
            {               
                if (lease != null)
                {
                    mem.Write(existingdata, 0, existingdata.Length);

                    var writer = new StreamWriter(mem);
                    var hashhex = string.Join(String.Empty, record.Hash.Select(x => x.ToString("x2")));
                    writer.WriteLine(String.Join("\t", record.Name, record.Version, hashhex, filename));
                    writer.Flush();
                    mem.Position = 0;

                    await this.blob.UploadFromStreamAsync(mem, new AccessCondition() { LeaseId = this.lease }, new BlobRequestOptions(), null);
                }
                else
                {
                    var writer = new StreamWriter(mem);
                    var hashhex = string.Join(String.Empty, record.Hash.Select(x => x.ToString("x2")));
                    writer.WriteLine(String.Join("\t", record.Name, record.Version, hashhex, filename));
                    writer.Flush();
                    mem.Position = 0;

                    await this.blob.AppendBlockAsync(mem);
                }                
            }
        }
Ejemplo n.º 3
0
 public async Task AppendWrite(FileRecord record, string filename)
 {
     await GetLeaseAndRead();
     await AppendWriteNoLease(record, filename);
 }
Ejemplo n.º 4
0
 public async Task AppendWrite(FileRecord record, string filename)
 {
     await this.index.AppendWriteNoLease(record, filename);
 }
        public List<FileRecord> Read(string filename)
        {
            var list = new List<FileRecord>();
            int version = 1;

            if (!System.IO.File.Exists(filename))
            {
                return list;
            }

            using (var fs = System.IO.File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                using (var reader = new BinaryReader(fs))
                {
                    while (fs.Position < fs.Length)
                    {
                        try
                        {
                            var record = new FileRecord(version);
                            record.ReadContentFromStream(reader);
                            list.Add(record);
                            version++;
                        }
                        catch (IOException)
                        {
                        }
                    }
                }
            }

            return list;
        }
        public async Task Append(string streamName, byte[] data, long expectedStreamVersion = -1)
        {
            var path = new FileInfo(this.options.FileStream(options.MasterStreamName));
            var streampath = this.options.FileStream(streamName);

            FileStream filestream;

            if (!path.Directory.Exists)
            {
                path.Directory.Create();
            }

            if (!System.IO.File.Exists(path.FullName))
            {
                filestream = System.IO.File.Open(path.FullName, FileMode.CreateNew, FileAccess.Write, FileShare.Write);
            }
            else
            {
                filestream = System.IO.File.Open(path.FullName, FileMode.Append, FileAccess.Write, FileShare.Write);
            }

            using (Stream s = filestream)
            {
                var record = new FileRecord(data, streamName, expectedStreamVersion + 1);
                await record.WriteContentToStreamAsync(s);
            }
        }
        public async Task WriteContent(string name, byte[] content, long expectedStreamVersion, int retry = 0)
        {
            this.index = new FileIndexLock(storage, container, options.NamingPolicy.GetIndexPath(name));
            
            try
            {
                //if (expectedStreamVersion == 0)
                //{
                //    await this.index.CreateIfNotExist();
                //    await this.masterindex.CreateIfNotExist();
                //}

                var canWrite = await index.GetLeaseAndRead(expectedStreamVersion != 0);

                if (!canWrite)
                {
                    if (retry < options.RetryPolicy.Length)
                    {
                        await Task.Delay(retry);
                        await this.WriteContent(name, content, expectedStreamVersion, retry++);
                        return;
                    }

                    await this.index.CreateIfNotExist();
                    await this.index.ReadIndex();
                    throw new AppendOnlyTimeoutException(expectedStreamVersion, index.OrderedIndex.Any() ? this.index.ReadLast().Version : 0, name);
                }

                await this.index.ReadIndex();
                var last = index.ReadLast();

                if ((last == null && expectedStreamVersion != 0) || (last != null && expectedStreamVersion != last.Version) || (last != null && last.Version != expectedStreamVersion ))
                {
                    throw new AppendOnlyStoreConcurrencyException(expectedStreamVersion, last?.Version ?? 0, name);
                }

                var blobcache = GetStreamCache(name);
                                
                if (last == null)
                {
                    //no file, create the stream to append to...
                    await blobcache.CreateOrReplaceAsync();
                }

                var master = this.GetMasterCache();

                using (var memstream = new MemoryStream())
                {
                    var record = new FileRecord(content, name, expectedStreamVersion + 1);
                    record.WriteContentToStream(memstream);

                    var mybytes = memstream.ToArray();

                    var appendmaster = Task.Run(async () =>
                    {
                        using (var masterstream = new MemoryStream(memstream.ToArray()))
                        {
                            await master.AppendBlockAsync(masterstream);
                        }
                    });

                    var up1 = blobcache.AppendFromByteArrayAsync(mybytes, 0, mybytes.Length);
                    var up3 = index.AppendWrite(record, blobcache.Name);
                    var up4 = masterindex.AppendWrite(record, blobcache.Name);

                    await index.EnsureLease();
                    if (!options.DisableMasterIndex)
                    {
                        await Task.WhenAll(up1, appendmaster, up3);
                    }
                    else
                    {
                        await Task.WhenAll(up4, up1, appendmaster, up3);
                    }
                }
            }
            finally
            {
                await index.Release();
            }
        }
        private static async Task<List<FileRecord>> ReadStreamToRecords(CloudAppendBlob blob)
        {
            List<FileRecord> records = new List<FileRecord>();

            using (var s = new MemoryStream())
            {
                if (!await blob.ExistsAsync())
                {
                    return records;
                }

                await blob.DownloadRangeToStreamAsync(s, 0, blob.Properties.Length);

                s.Position = 0;
                var reader = new BinaryReader(s);

                var record = new FileRecord();
                while (record.ReadContentFromStream(reader))
                {
                    records.Add(record);
                    record = new FileRecord();
                }
            }

            return records;
        }