public async Task WhenContainer_FullLiveCycle_ShouldPass()
{
var container = new BlobRepository(_blobStore);
await container.CreateContainer(_workContext);
IReadOnlyList <string> filePaths = await container.List(_workContext, "*");
await filePaths.ForEachAsync(x => container.Delete(_workContext, x));
const string filePath = "testBlob";
const string data = "This is data";
await container.Set(_workContext, filePath, data);
filePaths = await container.List(_workContext, "*");
filePaths.Should().NotBeNull();
filePaths.Count.Should().Be(1);
filePaths[0].Should().Be(filePath);
string readData = await container.Get(_workContext, filePath);
readData.Should().NotBeNullOrWhiteSpace();
readData.Should().Be(data);
await container.Delete(_workContext, filePath);
filePaths = await container.List(_workContext, "*");
filePaths.Should().NotBeNull();
filePaths.Count.Should().Be(0);
}
public void Upload_And_Download_Photo()
{
var blobRepository = new BlobRepository();
const string fileName = @"Koala.jpg";
var filePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Images\\{fileName}");
if (!File.Exists(filePath))
{
throw new ArgumentNullException(nameof(filePath));
}
var partitionKey = Guid.NewGuid();
var entity = new PhotoEntity(partitionKey.ToString(), Rowkey)
{
ID = partitionKey,
Title = "Koala",
Owner = Owner,
};
string photoUrl;
using (Stream file = File.OpenRead(filePath))
{
photoUrl = blobRepository.UploadBlob(file, entity);
}
Assert.IsNotNull(photoUrl);
Assert.AreEqual($"https://medya.blob.core.windows.net:443/images/photo-{entity.ID}.jpg", photoUrl);
}
public void Initialize()
{
var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString;
CfCloudStorage = CloudStorageAccount.Parse(connectionString);
blobRepo = new BlobRepository();
CfTrace.InitializeTraceSource(new Instrumentation.CfTraceSource("Cf.CacheServer"));
}
public async Task GivenBlockChain_WhenUsingBuilder_ShouldValidate()
{
const string _blobPath = "Test.sa";
var container = new BlobRepository(_blobStore);
await container.CreateContainer(_workContext);
var blockChain = new BlockChain()
{
new DataBlock <HeaderBlock>("header", "header_1", new HeaderBlock("Master Contract")),
new DataBlock <BlobBlock>("contract", "contract_1", new BlobBlock("contract.docx", "docx", "me", Encoding.UTF8.GetBytes("this is a contract between two people"))),
new DataBlock <TrxBlock>("ContractLedger", "Pmt", new TrxBlock("1", "cr", 100)),
};
blockChain.Blocks.Count.Should().Be(4);
string blockChainHash = blockChain.ToMerkleTree().BuildTree().ToString();
using var zipStream = blockChain.ToZipContainer(_workContext);
BlockChain result = zipStream.ToBlockChain(_workContext);
result.IsValid().Should().BeTrue();
string resultChainHash = result.ToMerkleTree().BuildTree().ToString();
blockChainHash.Should().Be(resultChainHash);
await container.Delete(_workContext, _blobPath);
}
public async Task WhenContainer_FullLiveCycleForFiles_ShouldPass()
{
var container = new BlobRepository(_blobStore);
await container.CreateContainer(_workContext);
IReadOnlyList <string> filePaths = await container.List(_workContext, "*");
await filePaths.ForEachAsync(x => container.Delete(_workContext, x));
const int count = 10;
var dataList = Enumerable.Range(0, count)
.Select(x => new { File = $"File_{x}", Data = $"Data_{x}" })
.ToList();
await dataList.ForEachAsync(x => container.Set(_workContext, x.File, x.Data));
filePaths = await container.List(_workContext, "*");
filePaths.Should().NotBeNull();
filePaths.Count.Should().Be(count);
await dataList.ForEachAsync(x => container.Delete(_workContext, x.File));
filePaths = await container.List(_workContext, "*");
filePaths.Should().NotBeNull();
filePaths.Count.Should().Be(0);
}
public void ObjectDeletionIsAlsoDeletingBlobs(BlobRepository repository, IObjectRepositoryContainer <BlobRepository> container, Signature signature, string message)
{
// Arrange
var a = container.AddRepository(repository, signature, message);
// Act
var b = container.Commit(
repository.With(c => c.Remove(repository, r => r.Cars, repository.Cars[0])),
signature,
message);
// Assert
b.Execute(r =>
{
var changes = r.Diff.Compare <TreeChanges>(
r.Lookup <Commit>(a.CommitId).Tree,
r.Lookup <Commit>(b.CommitId).Tree);
Assert.That(
changes.Deleted.Select(c => c.Path),
Is.EquivalentTo(new[]
{
$"Cars/{a.Cars[0].Id}/blob{FileSystemStorage.BlobExtension}",
$"Cars/{a.Cars[0].Id}/{FileSystemStorage.DataFile}"
}));
});
}
async Task TestLargeBlobPersistence()
{
var db = getDataContext();
IBlobRepository blrepo = new BlobRepository(db);
Random rnd = new Random(0x5555AAC0);
const int count = 4000;
Console.WriteLine("Constructing {0} blobs...", count);
Blob[] bls = new Blob[count];
for (int i = 0; i < count; ++i)
{
byte[] c = new byte[8040];
rnd.NextBytes(c);
bls[i] = new Blob.Builder(c);
}
ImmutableContainer <BlobID, Blob> blobs = new ImmutableContainer <BlobID, Blob>(bl => bl.ID, bls);
Console.WriteLine("Persisting {0} blobs...", count);
Stopwatch sw = Stopwatch.StartNew();
ImmutableContainer <BlobID, Blob> pBlobs = await blrepo.PersistBlobs(blobs);
Console.WriteLine("Waiting...");
sw.Stop();
Console.WriteLine("Completed in {0} ms, {1} blobs/sec, {2} bytes/sec",
sw.ElapsedMilliseconds,
(double)count / (double)sw.ElapsedMilliseconds * 1000d,
(double)(count * 8040) / (double)sw.ElapsedMilliseconds * 1000d);
}
public PeopleController(PeopleRepository peopleRepository,
BlobRepository blobRepository,
IOptions <PeopleOptions> peopleOpts)
{
this.peopleRepository = peopleRepository;
this.blobRepository = blobRepository;
this.peopleOpts = peopleOpts;
}
public BookingController(CompanyRepository companyRepository, BlobRepository blobRepository,
FilmSessionRepository filmSessionRepository, TicketRepository ticketRepository)
{
_companyRepository = companyRepository;
_blobRepository = blobRepository;
_filmSessionRepository = filmSessionRepository;
_ticketRepository = ticketRepository;
}
public BlobsModule(BlobRepository repository)
: base("/api/blobs")
{
this.repository = repository;
Post["/"] = parameters => HandlePostBlob();
Get["/{id}"] = parameters => HandleGetBlob(parameters.Id);
}
public void Initialize()
{
var connectionString = ConfigurationManager.ConnectionStrings["CfCloudStorage"].ConnectionString;
CfCloudStorage = CloudStorageAccount.Parse(connectionString);
blobRepo = new BlobRepository();
CfTrace.InitializeTraceSource(new Instrumentation.CfTraceSource("Cf.CacheServer"));
}
public BlobRepositoryTests()
{
sandbox = new FileSandbox(new EmptyFileLocator());
repository = BlobRepositoryFactory.Create(with =>
{
with.Location = sandbox.Directory;
});
}
public ManagerForm()
{
InitializeComponent();
IRepository repository = new BlobRepository();
_mappingManager = new MappingManager(repository);
}
public FilmController(CompanyRepository companyRepository, FilmRepository filmRepository,
BlobRepository blobRepository, FilmSessionRepository filmSessionRepository)
{
_companyRepository = companyRepository;
_filmRepository = filmRepository;
_blobRepository = blobRepository;
_filmSessionRepository = filmSessionRepository;
}
public FilmController(FilmRepository filmRepository, UserManager <ApplicationUser> userManager,
CompanyRepository companyRepository, UserSessionRepository userSessionRepository, BlobRepository blobRepository)
{
_filmRepository = filmRepository;
_userManager = userManager;
_companyRepository = companyRepository;
_userSessionRepository = userSessionRepository;
_blobRepository = blobRepository;
}
public async Task GivenBlockChain_WhenContainerIsBlob_ShouldRoundTrip()
{
const string _zipPath = "$block";
const string _blobPath = "Test.sa";
var container = new BlobRepository(_blobStore);
await container.CreateContainer(_workContext);
var blockChain = new BlockChain()
{
new DataBlock <HeaderBlock>("header", "header_1", new HeaderBlock("Master Contract")),
new DataBlock <BlobBlock>("contract", "contract_1", new BlobBlock("contract.docx", "docx", "me", Encoding.UTF8.GetBytes("this is a contract between two people"))),
new DataBlock <TrxBlock>("ContractLedger", "Pmt", new TrxBlock("1", "cr", 100)),
};
blockChain.Blocks.Count.Should().Be(4);
blockChain.IsValid().Should().BeTrue();
string blockChainHash = blockChain.ToMerkleTree().BuildTree().ToString();
string json = blockChain.ToJson();
//var buffer = new byte[1000];
using var writeMemoryBuffer = new MemoryStream();
var writer = new ZipContainerWriter(new ZipArchive(writeMemoryBuffer, ZipArchiveMode.Create, leaveOpen: true));
writer.Write(_workContext, _zipPath, json);
writer.Close();
writeMemoryBuffer.Length.Should().BeGreaterThan(0);
writeMemoryBuffer.Seek(0, SeekOrigin.Begin);
await container.Delete(_workContext, _blobPath);
await container.Upload(_workContext, _blobPath, writeMemoryBuffer);
writeMemoryBuffer.Close();
IReadOnlyList <byte> readBlob = await container.Download(_workContext, _blobPath);
using var readMemoryBuffer = new MemoryStream(readBlob.ToArray());
var reader = new ZipContainerReader(new ZipArchive(readMemoryBuffer, ZipArchiveMode.Read));
string readJson = reader.Read(_workContext, _zipPath);
reader.Close();
BlockChain result = readJson.ToBlockChain();
result.IsValid().Should().BeTrue();
string resultChainHash = result.ToMerkleTree().BuildTree().ToString();
blockChainHash.Should().Be(resultChainHash);
await container.Delete(_workContext, _blobPath);
}
public void Get_All_Photos_Count()
{
var tableRepository = new TableRepository();
var blobRepository = new BlobRepository();
var visionService = new VisionService();
var photoService = new PhotoService(tableRepository, blobRepository, visionService);
var photos = photoService.GetAll();
Assert.IsTrue(photos.Count > 0);
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.Configure <CookiePolicyOptions>(options =>
{
// This lambda determines whether user consent for non-essential cookies is needed for a given request.
options.CheckConsentNeeded = context => true;
options.MinimumSameSitePolicy = SameSiteMode.None;
});
services.AddAuthentication(AzureADDefaults.AuthenticationScheme)
.AddAzureAD(options => _config.Bind("AzureAd", options));
services.AddMvc(options =>
{
var policy = new AuthorizationPolicyBuilder()
.RequireAuthenticatedUser()
.Build();
options.Filters.Add(new AuthorizeFilter(policy));
})
.SetCompatibilityVersion(CompatibilityVersion.Version_2_1);
var storageAccountName = _config["StorageAccountName"];
var storageAccountUrl = _config["StorageAccountUrl"];
var keyVaultUrl = _config["KeyVaultUrl"];
var containerName = _config["ContainerName"];
// Add the custom repository class
services.AddScoped <IBlobRepository, BlobRepository>(provider => new BlobRepository(
storageAccountName,
storageAccountUrl,
keyVaultUrl,
containerName));
_logger.LogInformation("Pre-creating container");
try
{
var blobRepository = new BlobRepository(
storageAccountName,
storageAccountUrl,
keyVaultUrl,
containerName);
//Pre-create the blob container
blobRepository.CreateBlobContainer().GetAwaiter().GetResult();
}
catch (Exception oops)
{
_logger.LogError(oops, "Error Pre-creating container. {0}, {1}, {2}, {3}", storageAccountName, storageAccountUrl, keyVaultUrl, containerName);
//This error is non-recoverable.
throw oops;
}
}
public UnitOfWork(
BlogContext context,
IMapper mapper)
{
this._context = context;
Blogs = new BlogRepository(context, mapper);
Tags = new TagRepository(context);
Blobs = new BlobRepository(context, mapper);
Subscribers = new SubscriberRepository(context, mapper);
}
public static async Task <GenreResult> GetGenreResult(Genre genre, string lyrics, ILogger log)
{
var timer = new Stopwatch();
timer.Start();
log.LogInformation($"Loading {genre.ToString()} prediction model");
var mlContext = new MLContext(seed: 0);
ITransformer model;
using (var stream = await BlobRepository <object> .GetAsStream($"{genre.ToString()}GenrePrediction"))
{
var task = new Task <ITransformer>(() => mlContext.Model.Load(stream));
task.Start();
model = await task;
//model = mlContext.Model.Load(stream);
}
log.LogInformation($"Finished loading {genre.ToString()} prediction model. {timer.Elapsed.TotalSeconds}sec");
timer.Restart();
var predEngineTask = new Task <PredictionEngine <Lyric, GenrePrediction> >(() => model.CreatePredictionEngine <Lyric, GenrePrediction>(mlContext));
predEngineTask.Start();
var predEngine = await predEngineTask;
//var predEngine = model.CreatePredictionEngine<Lyric, GenrePrediction>(mlContext);
log.LogInformation($"Created {genre.ToString()} prediction model {timer.Elapsed.TotalSeconds}sec");
timer.Restart();
var lyricObj = new Lyric
{
Text = lyrics
};
log.LogInformation($"Predicting {genre.ToString()}...");
var genreResultTask = new Task <GenrePrediction>(() => predEngine.Predict(lyricObj));
genreResultTask.Start();
var genreResult = await genreResultTask;
//var genreResult = predEngine.Predict(lyricObj);
var result = new GenreResult
{
Genre = genre.ToString(),
Result = genreResult
};
log.LogInformation($"Finished {genre.ToString()} prediction {timer.Elapsed.TotalSeconds}sec");
return(result);
}
public void BlobSerializedAsNestedFile(BlobRepository sut, IObjectRepositoryContainer <BlobRepository> container, Signature signature, string message)
{
// Arrange
sut = container.AddRepository(sut, signature, message);
// Assert
sut.Execute(r =>
{
var blob = (Blob)r.Head[$"blob{FileSystemStorage.BlobExtension}"].Target;
Assert.AreEqual(sut.Blob.Value, blob.GetContentText());
});
}
public void MergeBlobConflictsCanResolvedAndContinued(BlobRepository sut, IObjectRepositoryContainer <BlobRepository> container, Signature signature, string message)
{
// Act
CreateConflictingChange(sut, container, signature, message);
var merge = container.Merge(sut.Id, "master");
merge.ModifiedProperties.Single(c => c.IsInConflict).Resolve(new StringBlob("y\nb\nd"));
merge.Apply(signature);
// Assert
Assert.That(container.Repositories.Single().Blob, Is.EqualTo(new StringBlob("y\nb\nd")));
}
/// <summary>
/// The Main method of the project.
/// </summary>
/// <param name="args">Arguments passed to the Main method.</param>
public static void Main(string[] args)
{
IRepository<ICommand> commandRepository = new CommandRepository();
IHandler<ICommandArguments> commandHandler = new CommandHandler(commandRepository);
IInputReader inputReader = new ConsoleInputReader();
IOutputWriter outputWriter = new ConsoleOutputWriter();
IRepository<IBlob> blobRepository = new BlobRepository();
ICommandArguments commandArguments = new CommandArguments(blobRepository, outputWriter);
IEngine gameEngine = new Engine(commandHandler, commandArguments, inputReader, (IUpdateable)blobRepository);
commandArguments.Stopable = gameEngine;
gameEngine.Run();
}
public void RebaseBlobConflictsCanResolvedAndContinued(BlobRepository sut, IObjectRepositoryContainer <BlobRepository> container, Signature signature, string message)
{
// Act
CreateConflictingChange(sut, container, signature, message);
var rebase = container.Rebase(sut.Id, "master");
rebase.ModifiedProperties.Single(c => c.IsInConflict).Resolve(new StringBlob("y\nb\nd"));
rebase.Continue();
// Assert
Assert.That(rebase.Status, Is.EqualTo(RebaseStatus.Complete));
Assert.That(rebase.TotalStepCount, Is.EqualTo(1));
Assert.That(container.Repositories.Single().Blob, Is.EqualTo(new StringBlob("y\nb\nd")));
}
static void CreateConflictingChange(BlobRepository sut, IObjectRepositoryContainer <BlobRepository> container, Signature signature, string message)
{
// master: A---B
// \ (B & C change same value)
// newBranch: C -> A---B---C
var a = container.AddRepository(sut, signature, message); // A
var updateName = a.With(a, r => r.Blob, new StringBlob("x\nb\nc"));
container.Commit(updateName.Repository, signature, message); // B
container.Checkout(a.Id, "newBranch", createNewBranch: true, "HEAD~1");
var updates = a.With(a, r => r.Blob, new StringBlob("z\nb\nd"));
container.Commit(updates, signature, message);
}
public async Task GivenFileDoesNotExist_WhenGet_ShouldThrow()
{
var container = new BlobRepository(_blobStore);
await container.CreateContainer(_workContext);
IReadOnlyList <string> filePaths = await container.List(_workContext, "*");
await filePaths.ForEachAsync(x => container.Delete(_workContext, x));
const string filePath = "testBlob";
Func <Task> act = async() => await container.Get(_workContext, filePath);
await act.Should().ThrowAsync <RequestFailedException>();
}
public LicensePlateBusService(SecretsConfig config,
LicenseApiService licenseService,
LicensePlateRepository repo,
FileRepository fileRepo,
BlobRepository blob)
{
_config = config;
_connectionString = config.ReadConnectionString;
_readTopicName = config.ReadTopic;
_subscriptionKey = config.SubscriptionKey;
_licenseService = licenseService;
_repo = repo;
_blob = blob;
_fileRepo = fileRepo;
}
public async static Task MainAsync(string[] args)
{
Console.WriteLine($"Loading Full Dataset from db");
DocumentDBRepository <SongRecord> .Initialize(collectionId : "Songs");
var songs = await DocumentDBRepository <SongRecord> .GetItemsAsync(x => true, -1);
foreach (var genre in (Genre[])Enum.GetValues(typeof(Genre)))
{
Console.WriteLine();
Console.WriteLine($"=============== Genre: {genre.ToString()} ===============");
var lyricData = songs
.Select(x =>
new Lyric
{
Genre = x.Genre.Contains(genre.ToString()),
Text = x.Lyrics
});
// Only bother with data with more than 20 examples
if (lyricData.Count(x => x.Genre) > 20)
{
Console.WriteLine($"Loading data");
var mlContext = new MLContext(seed: 0);
IDataView trainingDataView = mlContext.Data.LoadFromEnumerable(lyricData);
TrainCatalogBase.TrainTestData splitDataView = mlContext.BinaryClassification.TrainTestSplit(trainingDataView, testFraction: 0.2);
var model = BuildAndTrainModel(splitDataView.TrainSet, mlContext);
Evaluate(mlContext, model, splitDataView.TestSet);
using (var stream = new MemoryStream())
{
mlContext.Model.Save(model, stream);
await BlobRepository <object> .UploadFromStream(stream, $"{genre.ToString()}GenrePrediction");
}
}
else
{
Console.WriteLine($"Not enough data");
}
}
}
public static void Main(string[] args)
{
const string path = @"d:\\tick-tock";
BlobRepository blobs = BlobRepositoryFactory.Create(with =>
{
with.Location = Path.Combine(path, "blobs");
});
JobExecutionRepository executions = JobExecutionRepositoryFactory.Create(with =>
{
with.Location = Path.Combine(path, "executions");
});
JobRepository jobs = JobRepositoryFactory.Create(with =>
{
with.Location = Path.Combine(path, "jobs");
});
JobTaskRepository tasks = JobTaskRepositoryFactory.Create(with =>
{
with.Blobs = blobs;
with.Executions = executions;
});
JobRunner runner = JobRunnerFactory.Create(with =>
{
with.Executions = executions;
with.Jobs = jobs;
with.Tasks = tasks;
});
Bootstrapper bootstrapper = new Bootstrapper(path);
Uri uri = new Uri("http://localhost:1234");
using (var host = new NancyHost(bootstrapper, uri))
{
host.Start();
while (true)
{
runner.Run();
Thread.Sleep(TimeSpan.FromSeconds(10));
}
}
}
public void ResolveDiffsBlobUpdate(BlobRepository sut, IObjectRepositoryContainer <BlobRepository> container, Signature signature, string message, ComputeTreeChangesFactory computeTreeChangesFactory)
{
// Arrange
sut = container.AddRepository(sut, signature, message);
var modified = sut.With(sut, r => r.Blob, new StringBlob("z\nb\nz"));
var commit = container.Commit(modified.Repository, signature, message);
// Act
var changes = computeTreeChangesFactory(container, sut.RepositoryDescription)
.Compare(sut.CommitId, commit.CommitId)
.SkipIndexChanges();
// Assert
Assert.That(changes, Has.Count.EqualTo(1));
Assert.That(changes[0].Status, Is.EqualTo(ChangeKind.Modified));
Assert.That(((BlobRepository)changes[0].Old).Blob, Is.EqualTo(sut.Blob));
Assert.That(((BlobRepository)changes[0].New).Blob, Is.EqualTo(modified.Blob));
}
public void Test_PhotoService()
{
var tableRepository = new TableRepository();
var blobRepository = new BlobRepository();
var visionService = new VisionService();
var photoService = new PhotoService(tableRepository, blobRepository, visionService);
//Photo
const string fileName = @"Desert.jpg";
var filePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Images\\{fileName}");
if (!File.Exists(filePath))
{
throw new ArgumentNullException(nameof(filePath));
}
//Photo Entity
var partitionKey = Guid.NewGuid();
var entity = new PhotoEntity(partitionKey.ToString(), Rowkey)
{
ID = partitionKey,
Title = "Desert",
Owner = Owner,
};
using (Stream file = File.OpenRead(filePath))
{
photoService.Create(entity, file);
}
var photos = photoService.GetAll();
Assert.IsNotNull(photos);
Assert.IsTrue(photos.Count > 0);
var singleOrDefault = photos.SingleOrDefault(p => p.ID.Equals(entity.ID));
Assert.IsNotNull(singleOrDefault);
Assert.AreEqual("Desert", singleOrDefault.Title);
}
public async Task GivenBlockChain_WhenContainerIsBlobAndBuilder_ShouldRoundTrip()
{
const string _blobPath = "Test.sa";
var container = new BlobRepository(_blobStore);
await container.CreateContainer(_workContext);
var blockChain = new BlockChain()
{
new DataBlock <HeaderBlock>("header", "header_1", new HeaderBlock("Master Contract")),
new DataBlock <BlobBlock>("contract", "contract_1", new BlobBlock("contract.docx", "docx", "me", Encoding.UTF8.GetBytes("this is a contract between two people"))),
new DataBlock <TrxBlock>("ContractLedger", "Pmt", new TrxBlock("1", "cr", 100)),
};
blockChain.Blocks.Count.Should().Be(4);
string blockChainHash = blockChain.ToMerkleTree().BuildTree().ToString();
using (var zipStream = blockChain.ToZipContainer(_workContext))
{
await container.Delete(_workContext, _blobPath);
await container.Upload(_workContext, _blobPath, zipStream);
}
IReadOnlyList <byte> readBlob = await container.Download(_workContext, _blobPath);
using var readMemoryBuffer = new MemoryStream(readBlob.ToArray());
BlockChain result = readMemoryBuffer.ToBlockChain(_workContext);
result.IsValid().Should().BeTrue();
string resultChainHash = result.ToMerkleTree().BuildTree().ToString();
blockChainHash.Should().Be(resultChainHash);
await container.Delete(_workContext, _blobPath);
}
