public void FullFileChunkProcessor_StreamToBlobs_SucceedOnNonEmptyStream()
{
// Arrange
string alreadyExistingContent = "SomeRandomText";
byte[] bytes = Encoding.UTF8.GetBytes(alreadyExistingContent);
SpookyHash spookyHash = new SpookyHash();
string hashString = spookyHash.CalculateHash(bytes);
using (MemoryStream ms = new MemoryStream(bytes))
{
string[] blobIds;
IReadOnlyDictionary <string, IBlob> blobs = new Dictionary <string, IBlob>();
IChunkProcessor chunkProcessor = CreateFullFileChunkProcessor();
IBlobAllocator allocator = CreateBlobAllocator();
// Act
chunkProcessor.StreamToBlobs(ms, allocator, out blobIds, out blobs);
// Assert
Assert.AreEqual(1, blobIds.Length);
Assert.AreEqual(1, blobs.Count);
Assert.AreEqual((ulong)bytes.Length, blobs[blobIds[0]].Length);
Assert.AreEqual(hashString, blobs[blobIds[0]].BlobId);
}
}
public TerrainGenerator(WorldData worldData, IChunkProcessor chunkProcessor, IBatchProcessor <Chunk> batchProcessor, ITerrainGenerationMethod terrainGenerationMethod)
{
m_WorldData = worldData;
m_ChunkProcessor = chunkProcessor;
m_BatchProcessor = batchProcessor;
m_TerrainGenerationMethod = terrainGenerationMethod;
}
public ChunkedPhase(
IStageIdentity stageIdentity,
IGenerationPhaseIdentity phaseIdentity,
IReadOnlyLargeCollection <TKey> chunkKeys,
IAsyncFactory <TKey, IDisposableValue <TChunk> > chunkFactory,
IChunkProcessor <TChunk> chunkProcessor,
GenerationOptions options = null)
: base(options?.CancellationToken ?? DefaultCancellationToken)
{
Contracts.Requires.That(stageIdentity != null);
Contracts.Requires.That(phaseIdentity != null);
Contracts.Requires.That(chunkKeys != null);
Contracts.Requires.That(chunkFactory != null);
Contracts.Requires.That(chunkProcessor != null);
this.StageIdentity = stageIdentity;
this.PhaseIdentity = phaseIdentity;
this.chunkKeys = chunkKeys;
this.chunkFactory = chunkFactory;
this.chunkProcessor = chunkProcessor;
this.Progress = this.progress.AsObservable();
var dataflowOptions = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = options?.MaxDegreeOfParallelism ?? DefaultMaxDegreeOfParallelism,
CancellationToken = this.CancellationToken,
};
this.processChunks = new ActionBlock <TKey>(this.ProcessChunk, dataflowOptions);
this.chunksCompleted = new AsyncLongCountdownEvent(this.ProgressTotalCount);
}
public TerrainGenerator(WorldData worldData,IChunkProcessor chunkProcessor, IBatchProcessor<Chunk> batchProcessor, ITerrainGenerationMethod terrainGenerationMethod)
{
m_WorldData = worldData;
m_ChunkProcessor = chunkProcessor;
m_BatchProcessor = batchProcessor;
m_TerrainGenerationMethod = terrainGenerationMethod;
}
/// <summary>
/// This only exists for testing, not to be called directly by production code.
/// </summary>
public WorldDecorator(WorldData worldData, IBatchProcessor<Chunk> batchProcessor, List<IDecoration> decorations,
IChunkProcessor chunkProcessor)
{
m_WorldData = worldData;
m_BatchProcessor = batchProcessor;
m_ChunkProcessor = chunkProcessor;
m_Decorations = decorations;
}
/// <summary>
/// This only exists for testing, not to be called directly by production code.
/// </summary>
public WorldDecorator(WorldData worldData, IBatchProcessor <Chunk> batchProcessor, List <IDecoration> decorations,
IChunkProcessor chunkProcessor)
{
m_WorldData = worldData;
m_BatchProcessor = batchProcessor;
m_ChunkProcessor = chunkProcessor;
m_Decorations = decorations;
}
public World(WorldData worldData, ITerrainGenerator terrainGenerator, ILightProcessor lightProcessor,
IMeshDataGenerator meshDataGenerator, IWorldDecorator worldDecorator, IChunkProcessor chunkProcessor)
{
m_WorldData = worldData;
m_LightProcessor = lightProcessor;
m_MeshDataGenerator = meshDataGenerator;
m_WorldDecorator = worldDecorator;
m_ChunkProcessor = chunkProcessor;
m_TerrainGenerator = terrainGenerator;
ContinueProcessingChunks = true;
}
public void ChunkProcessorFactory_CreateInstance_ReturnFullFileChunkingProcessor()
{
// Act
IChunkProcessor chunkProcessor = ChunkProcessorFactory.Instance.CreateInstance(ChunkingScheme.FullFile);
// Assert
Type chunkProcessorType = chunkProcessor.GetType();
Assert.IsNotNull(chunkProcessor);
Assert.AreEqual(typeof(FullFileChunkProcessor), chunkProcessorType);
}
public World(WorldData worldData, ITerrainGenerator terrainGenerator, ILightProcessor lightProcessor,
IMeshDataGenerator meshDataGenerator, IWorldDecorator worldDecorator, IChunkProcessor chunkProcessor)
{
m_WorldData = worldData;
m_LightProcessor = lightProcessor;
m_MeshDataGenerator = meshDataGenerator;
m_WorldDecorator = worldDecorator;
m_ChunkProcessor = chunkProcessor;
m_TerrainGenerator = terrainGenerator;
ContinueProcessingChunks = true;
}
public void FullFileChunkProcessor_StreamToBlobs_ThrowOnNullStream()
{
// Arrange
Stream stream = null;
string[] blobIds;
IReadOnlyDictionary <string, IBlob> blobs = new Dictionary <string, IBlob>();
IChunkProcessor chunkProcessor = CreateFullFileChunkProcessor();
IBlobAllocator allocator = CreateBlobAllocator();
// Act
chunkProcessor.StreamToBlobs(stream, allocator, out blobIds, out blobs);
}
public GzipProcessor(
IFileService fileService,
ILogger logger,
IContentReader reader,
IContentWriter writer,
IChunkProcessor processor,
Settings settings)
{
_fileService = fileService;
_logger = logger;
_reader = reader;
_writer = writer;
_processor = processor;
_settings = settings;
}
public void FullFileChunkProcessor_StreamToBlobs_SuccessOnEmptyStream()
{
// Arrange
using (MemoryStream ms = new MemoryStream(new byte[0]))
{
string[] blobIds;
IReadOnlyDictionary <string, IBlob> blobs = new Dictionary <string, IBlob>();
IChunkProcessor chunkProcessor = CreateFullFileChunkProcessor();
IBlobAllocator allocator = CreateBlobAllocator();
// Act
chunkProcessor.StreamToBlobs(ms, allocator, out blobIds, out blobs);
// Assert
Assert.AreEqual(0, blobIds.Length);
Assert.AreEqual(0, blobs.Count);
}
}
public ChunkWordFrequencyCounter(IChunkFileReader chunkFileReader,
IChunkProcessor chunkProcessor,
IChunkResultMerger chunkResultMerger)
{
if (chunkFileReader == null)
{
throw new ArgumentNullException(nameof(chunkFileReader));
}
if (chunkProcessor == null)
{
throw new ArgumentNullException(nameof(chunkProcessor));
}
if (chunkResultMerger == null)
{
throw new ArgumentNullException(nameof(chunkResultMerger));
}
_chunkFileReader = chunkFileReader;
_chunkProcessor = chunkProcessor;
_chunkResultMerger = chunkResultMerger;
}
public WorldData(IChunkProcessor chunkProcessor)
{
m_ChunkProcessor = chunkProcessor;
SetShadesOfLight(m_NumberOfLightShades);
}
public WorldData(IChunkProcessor chunkProcessor)
{
m_ChunkProcessor = chunkProcessor;
SetShadesOfLight(m_NumberOfLightShades);
}
/// <summary>
/// Custom constructor using a chunk provider and a chunk processor.
/// </summary>
/// <param name="chunkProvider"></param>
/// <param name="chunkProcessor"></param>
public ChunkOrientedTasklet(IChunkProvider <TI> chunkProvider, IChunkProcessor <TI> chunkProcessor)
{
_chunkProvider = chunkProvider;
_chunkProcessor = chunkProcessor;
}
private IChunkProcessor CreateFullFileChunkProcessor()
{
IChunkProcessor chunkProcessor = ChunkProcessorFactory.Instance.CreateInstance(ChunkingScheme.FullFile);
return(chunkProcessor);
}
public WorldDecorator(WorldData worldData, IBatchProcessor<Chunk> batchProcessor, IChunkProcessor chunkProcessor)
: this(worldData, batchProcessor, LoadDecorators(worldData), chunkProcessor)
{
}
private void BuildChunkSignaturesAndDeltaBlobDictionary(
string newContent,
string lastKnownHostContent,
string newContentResourceId,
string lastKnownHostContentResourceId,
IResourceManager resourceManager,
ChunkingScheme chunkingScheme,
Dictionary <string, IBlob> deltaBlobs,
out ChunkSignature[] chunkSignatures)
{
string[] newContentBlobIds = new string[0];
IReadOnlyDictionary <string, IBlob> newContentBlobs = new Dictionary <string, IBlob>();
IReadOnlyDictionary <string, IBlob> lastKnownHostContentBlobs = new Dictionary <string, IBlob>();
if (chunkingScheme == ChunkingScheme.FullFile)
{
Stream newContentStream = new MemoryStream(Encoding.UTF8.GetBytes(newContent));
Stream lastKnownHostContentStream = new MemoryStream(Encoding.UTF8.GetBytes(lastKnownHostContent));
IChunkProcessor chunkProcessor = ChunkProcessorFactory.Instance.CreateInstance(chunkingScheme);
chunkProcessor.StreamToBlobs(
newContentStream,
new BlobAllocator(),
out newContentBlobIds,
out newContentBlobs
);
chunkProcessor.StreamToBlobs(
lastKnownHostContentStream,
new BlobAllocator(),
out _,
out lastKnownHostContentBlobs
);
}
else if (chunkingScheme == ChunkingScheme.Zip)
{
IChunkProcessor chunkProcessor = ChunkProcessorFactory.Instance.CreateInstance(chunkingScheme);
chunkProcessor.StreamToBlobs(
resourceManager,
newContentResourceId,
out newContentBlobIds,
out newContentBlobs
);
chunkProcessor.StreamToBlobs(
resourceManager,
lastKnownHostContentResourceId,
out _,
out lastKnownHostContentBlobs
);
}
ChunkSignature[] chunkSignaturesToReturn = new ChunkSignature[newContentBlobIds.Length];
for (int i = 0; i < newContentBlobIds.Length; i++)
{
string newContentBlobId = newContentBlobIds[i];
IBlob newContentBlob = newContentBlobs[newContentBlobId];
chunkSignaturesToReturn[i] = new ChunkSignature(ChunkId: newContentBlobId, Length: newContentBlob.Length);
if (!lastKnownHostContentBlobs.ContainsKey(newContentBlobId) && !deltaBlobs.ContainsKey(newContentBlobId))
{
deltaBlobs.Add(newContentBlobId, newContentBlob);
}
}
chunkSignatures = chunkSignaturesToReturn;
}
public WorldDecorator(WorldData worldData, IBatchProcessor <Chunk> batchProcessor, IChunkProcessor chunkProcessor)
: this(worldData, batchProcessor, LoadDecorators(worldData), chunkProcessor)
{
}
