public void WritableStreamTest3() { using (var expected = new MemoryStream()) { using (var stream = SerializableBytes.CreateWritableStream()) { var random = new Random(Environment.TickCount); for (var i = 0; i < 100; i++) { var position = random.Next(10000); WriteByte(expected, stream, position, i); } StreamEqual(expected, stream); } } }
internal static async Task VerifySnapshotSerializationAsync(ISolutionChecksumService service, Solution solution, SolutionChecksumObject solutionId) { using (var stream = SerializableBytes.CreateWritableStream()) using (var writer = new ObjectWriter(stream)) { await solutionId.WriteToAsync(writer, CancellationToken.None).ConfigureAwait(false); stream.Position = 0; using (var reader = new ObjectReader(stream)) { var serializer = new Serializer(solution.Workspace.Services); var recovered = serializer.Deserialize <SolutionChecksumObject>(SolutionChecksumObject.Name, reader, CancellationToken.None); SnapshotEqual(service, solutionId, recovered); } } }
private static async Task <RemotableData> CloneAssetAsync(ISerializerService serializer, RemotableData asset) { using var stream = SerializableBytes.CreateWritableStream(); using (var writer = new ObjectWriter(stream, leaveOpen: true)) { await asset.WriteObjectToAsync(writer, CancellationToken.None).ConfigureAwait(false); } stream.Position = 0; using var reader = ObjectReader.TryGetReader(stream); var recovered = serializer.Deserialize <object>(asset.Kind, reader, CancellationToken.None); var assetFromStorage = new SolutionAsset(serializer.CreateChecksum(recovered, CancellationToken.None), recovered, serializer); Assert.Equal(asset.Checksum, assetFromStorage.Checksum); return(assetFromStorage); }
public void WritableStreamTest1() { using var expected = new MemoryStream(); for (var i = 0; i < 10000; i++) { expected.WriteByte((byte)(i % byte.MaxValue)); } expected.Position = 0; using var stream = SerializableBytes.CreateWritableStream(); for (var i = 0; i < 10000; i++) { stream.WriteByte((byte)(i % byte.MaxValue)); } StreamEqual(expected, stream); }
private static SolutionAsset CloneAsset(ISerializerService serializer, SolutionAsset asset) { using var stream = SerializableBytes.CreateWritableStream(); using (var writer = new ObjectWriter(stream, leaveOpen: true)) { serializer.Serialize(asset.Value, writer, CancellationToken.None); } stream.Position = 0; using var reader = ObjectReader.TryGetReader(stream); var recovered = serializer.Deserialize <object>(asset.Kind, reader, CancellationToken.None); var assetFromStorage = new SolutionAsset(serializer.CreateChecksum(recovered, CancellationToken.None), recovered); Assert.Equal(asset.Checksum, assetFromStorage.Checksum); return(assetFromStorage); }
public async Task <bool> SerializeAsync(object documentOrProject, string key, ImmutableArray <DiagnosticData> items, CancellationToken cancellationToken) { using (var stream = SerializableBytes.CreateWritableStream()) using (var writer = new StreamObjectWriter(stream, cancellationToken: cancellationToken)) { WriteTo(writer, items, cancellationToken); var solution = GetSolution(documentOrProject); var persistService = solution.Workspace.Services.GetService <IPersistentStorageService>(); using (var storage = persistService.GetStorage(solution)) { stream.Position = 0; return(await WriteStreamAsync(storage, documentOrProject, key, stream, cancellationToken).ConfigureAwait(false)); } } }
public void TestEncodingSerialization() { var hostServices = MefHostServices.Create( MefHostServices.DefaultAssemblies.Add(typeof(Host.TemporaryStorageServiceFactory.TemporaryStorageService).Assembly)); var workspace = new AdhocWorkspace(hostServices); var serializer = new Serializer(workspace); // test with right serializable encoding var sourceText = SourceText.From("Hello", Encoding.UTF8); using (var stream = SerializableBytes.CreateWritableStream()) { using (var objectWriter = new ObjectWriter(stream)) { serializer.Serialize(sourceText, objectWriter, CancellationToken.None); } stream.Position = 0; using (var objectReader = ObjectReader.TryGetReader(stream)) { var newText = serializer.Deserialize <SourceText>(sourceText.GetWellKnownSynchronizationKind(), objectReader, CancellationToken.None); Assert.Equal(sourceText.ToString(), newText.ToString()); } } // test with wrong encoding that doesn't support serialization sourceText = SourceText.From("Hello", new NotSerializableEncoding()); using (var stream = SerializableBytes.CreateWritableStream()) { using (var objectWriter = new ObjectWriter(stream)) { serializer.Serialize(sourceText, objectWriter, CancellationToken.None); } stream.Position = 0; using (var objectReader = ObjectReader.TryGetReader(stream)) { var newText = serializer.Deserialize <SourceText>(sourceText.GetWellKnownSynchronizationKind(), objectReader, CancellationToken.None); Assert.Equal(sourceText.ToString(), newText.ToString()); } } }
public void TestEncodingSerialization() { using var workspace = CreateWorkspace(); var serializer = workspace.Services.GetService <ISerializerService>(); // test with right serializable encoding var sourceText = SourceText.From("Hello", Encoding.UTF8); using (var stream = SerializableBytes.CreateWritableStream()) { using var context = SolutionReplicationContext.Create(); using (var objectWriter = new ObjectWriter(stream, leaveOpen: true)) { serializer.Serialize(sourceText, objectWriter, context, CancellationToken.None); } stream.Position = 0; using var objectReader = ObjectReader.TryGetReader(stream); var newText = serializer.Deserialize <SourceText>(sourceText.GetWellKnownSynchronizationKind(), objectReader, CancellationToken.None); Assert.Equal(sourceText.ToString(), newText.ToString()); } // test with wrong encoding that doesn't support serialization sourceText = SourceText.From("Hello", new NotSerializableEncoding()); using (var stream = SerializableBytes.CreateWritableStream()) { using var context = SolutionReplicationContext.Create(); using (var objectWriter = new ObjectWriter(stream, leaveOpen: true)) { serializer.Serialize(sourceText, objectWriter, context, CancellationToken.None); } stream.Position = 0; using var objectReader = ObjectReader.TryGetReader(stream); var newText = serializer.Deserialize <SourceText>(sourceText.GetWellKnownSynchronizationKind(), objectReader, CancellationToken.None); Assert.Equal(sourceText.ToString(), newText.ToString()); } }
private static async Task <Asset> CloneAssetAsync(Serializer serializer, AssetBuilder assetBuilder, Asset asset) { using (var stream = SerializableBytes.CreateWritableStream()) using (var writer = new ObjectWriter(stream)) { await asset.WriteObjectToAsync(writer, CancellationToken.None).ConfigureAwait(false); stream.Position = 0; using (var reader = new ObjectReader(stream)) { var recovered = serializer.Deserialize <object>(asset.Kind, reader, CancellationToken.None); var assetFromStorage = BuildAsset(assetBuilder, asset.Kind, recovered); Assert.Equal(asset.Checksum, assetFromStorage.Checksum); return(assetFromStorage); } } }
public void WritableStreamTest4() { using var expected = new MemoryStream(); using var stream = SerializableBytes.CreateWritableStream(); var random = new Random(0); for (var i = 0; i < 100; i++) { var position = random.Next(10000); WriteByte(expected, stream, position, i); var position1 = random.Next(10000); var temp = GetInitializedArray(100 + position1); Write(expected, stream, position1, temp); } StreamEqual(expected, stream); }
public static void WriteAllLines( this ITemporaryStreamStorage storage, ImmutableArray <string> values ) { using var stream = SerializableBytes.CreateWritableStream(); using var writer = new StreamWriter(stream); foreach (var value in values) { writer.WriteLine(value); } writer.Flush(); stream.Position = 0; storage.WriteStream(stream); }
private static (byte[] bytes, int length, bool fromPool) GetBytes( Checksum checksumOpt, CancellationToken cancellationToken) { // If we weren't passed a checsum, just pass the singleton empty byte array. // Note: we don't add this to/from our pool. But it likely woudn't be a problem // for us to do that as this instance can't actually be mutated since it's just // an empty array. if (checksumOpt == null) { return(Array.Empty <byte>(), length : 0, fromPool : false); } using var stream = SerializableBytes.CreateWritableStream(); using var writer = new ObjectWriter(stream, cancellationToken: cancellationToken); checksumOpt.WriteTo(writer); stream.Position = 0; return(GetBytes(stream)); }
public async Task TestUnresolvedAnalyzerReference() { using (var workspace = new TestWorkspace()) { var project = workspace.CurrentSolution.AddProject("empty", "empty", LanguageNames.CSharp); using var analyzer = new VisualStudioAnalyzer( @"PathToAnalyzer", hostDiagnosticUpdateSource: null, projectId: project.Id, workspace: workspace, language: project.Language); var analyzerReference = analyzer.GetReference(); project = project.WithAnalyzerReferences(new[] { analyzerReference }); var checksum = await project.State.GetChecksumAsync(CancellationToken.None).ConfigureAwait(false); Assert.NotNull(checksum); var assetBuilder = new CustomAssetBuilder(workspace); var serializer = workspace.Services.GetService <ISerializerService>(); var asset = assetBuilder.Build(analyzerReference, CancellationToken.None); using var stream = SerializableBytes.CreateWritableStream(); using (var writer = new ObjectWriter(stream, leaveOpen: true)) { await asset.WriteObjectToAsync(writer, CancellationToken.None).ConfigureAwait(false); } stream.Position = 0; using (var reader = ObjectReader.TryGetReader(stream)) { var recovered = serializer.Deserialize <AnalyzerReference>(asset.Kind, reader, CancellationToken.None); var assetFromStorage = assetBuilder.Build(recovered, CancellationToken.None); Assert.Equal(asset.Checksum, assetFromStorage.Checksum); // This won't round trip, but we should get an UnresolvedAnalyzerReference, with the same path Assert.Equal(analyzerReference.FullPath, recovered.FullPath); } } }
public static async Task <T> GetValueAsync <T>(this IRemotableDataService service, Checksum checksum) { var syncService = (RemotableDataServiceFactory.Service)service; var syncObject = (await syncService.TestOnly_GetRemotableDataAsync(checksum, CancellationToken.None).ConfigureAwait(false)) !; using var stream = SerializableBytes.CreateWritableStream(); using (var writer = new ObjectWriter(stream, leaveOpen: true)) { await syncObject.WriteObjectToAsync(writer, CancellationToken.None).ConfigureAwait(false); } stream.Position = 0; using var reader = ObjectReader.TryGetReader(stream); // deserialize bits to object var serializer = syncService.Serializer_TestOnly; return(serializer.Deserialize <T>(syncObject.Kind, reader, CancellationToken.None)); }
public async Task <bool> SerializeAsync(IPersistentStorageService persistentService, Project project, Document?document, string key, ImmutableArray <DiagnosticData> items, CancellationToken cancellationToken) { Contract.ThrowIfFalse(document == null || document.Project == project); using var stream = SerializableBytes.CreateWritableStream(); using var writer = new ObjectWriter(stream, cancellationToken: cancellationToken); WriteDiagnosticData(writer, items, cancellationToken); using var storage = persistentService.GetStorage(project.Solution); stream.Position = 0; var writeTask = (document != null) ? storage.WriteStreamAsync(document, key, stream, cancellationToken) : storage.WriteStreamAsync(project, key, stream, cancellationToken); return(await writeTask.ConfigureAwait(false)); }
protected static async Task <bool> SaveAsync <T>( Document document, string persistenceName, string formatVersion, T data, CancellationToken cancellationToken) where T : AbstractPersistableState, IObjectWritable { Contract.Requires(!await document.IsForkedDocumentWithSyntaxChangesAsync(cancellationToken).ConfigureAwait(false)); var persistentStorageService = WorkspaceService.GetService <IPersistentStorageService>(document.Project.Solution.Workspace); // attempt to load from persisted state using (var storage = persistentStorageService.GetStorage(document.Project.Solution)) using (var stream = SerializableBytes.CreateWritableStream()) using (var writer = new ObjectWriter(stream, cancellationToken: cancellationToken)) { data.WriteVersion(writer, formatVersion); data.WriteTo(writer); stream.Position = 0; return(await storage.WriteStreamAsync(document, persistenceName, stream, cancellationToken).ConfigureAwait(false)); } }
private void GetStorageInfoFromTemporaryStorage(FileKey moduleFileKey, out ITemporaryStreamStorage storage, out Stream stream, out IntPtr pImage) { int size; using (var copyStream = SerializableBytes.CreateWritableStream()) { // open a file and let it go as soon as possible using (var fileStream = FileUtilities.OpenRead(moduleFileKey.FullPath)) { var headers = new PEHeaders(fileStream); var offset = headers.MetadataStartOffset; size = headers.MetadataSize; // given metadata contains no metadata info. // throw bad image format exception so that we can show right diagnostic to user. if (size <= 0) { throw new BadImageFormatException(); } StreamCopy(fileStream, copyStream, offset, size); } // copy over the data to temp storage and let pooled stream go storage = _temporaryStorageService.CreateTemporaryStreamStorage(CancellationToken.None); copyStream.Position = 0; storage.WriteStream(copyStream); } // get stream that owns direct access memory stream = storage.ReadStream(CancellationToken.None); // stream size must be same as what metadata reader said the size should be. Contract.ThrowIfFalse(stream.Length == size); // under VS host, direct access should be supported var directAccess = (ISupportDirectMemoryAccess)stream; pImage = directAccess.GetPointer(); }
internal static async Task <T> GetValueAsync <T>(ISolutionChecksumService service, Checksum checksum, string kind) { var snapshotService = (SolutionChecksumServiceFactory.Service)service; var checksumObject = service.GetChecksumObject(checksum, CancellationToken.None); using (var stream = SerializableBytes.CreateWritableStream()) using (var writer = new ObjectWriter(stream)) { // serialize asset to bits await checksumObject.WriteToAsync(writer, CancellationToken.None).ConfigureAwait(false); stream.Position = 0; using (var reader = new ObjectReader(stream)) { // deserialize bits to object var serializer = snapshotService.Serializer_TestOnly; return(serializer.Deserialize <T>(kind, reader, CancellationToken.None)); } } }
public static async Task <T> GetValueAsync <T>(this ISolutionSynchronizationService service, Checksum checksum) { var syncService = (SolutionSynchronizationServiceFactory.Service)service; var syncObject = service.GetRemotableData(checksum, CancellationToken.None); using (var stream = SerializableBytes.CreateWritableStream()) using (var writer = new StreamObjectWriter(stream)) { // serialize asset to bits await syncObject.WriteObjectToAsync(writer, CancellationToken.None).ConfigureAwait(false); stream.Position = 0; using (var reader = new StreamObjectReader(stream)) { // deserialize bits to object var serializer = syncService.Serializer_TestOnly; return(serializer.Deserialize <T>(syncObject.Kind, reader, CancellationToken.None)); } } }
private static async Task VerifyOptionSetsAsync(Workspace workspace, string language) { var assetBuilder = new CustomAssetBuilder(workspace); var serializer = workspace.Services.GetService <ISerializerService>(); var asset = assetBuilder.Build(workspace.Options, language, CancellationToken.None); using var stream = SerializableBytes.CreateWritableStream(); using var writer = new ObjectWriter(stream); await asset.WriteObjectToAsync(writer, CancellationToken.None).ConfigureAwait(false); stream.Position = 0; using var reader = ObjectReader.TryGetReader(stream); var recovered = serializer.Deserialize <OptionSet>(asset.Kind, reader, CancellationToken.None); var assetFromStorage = assetBuilder.Build(recovered, language, CancellationToken.None); Assert.Equal(asset.Checksum, assetFromStorage.Checksum); // option should be exactly same Assert.Equal(0, recovered.GetChangedOptions(workspace.Options).Count()); }
public void Store(SyntaxTree tree, SyntaxNode root, ITemporaryStorageService service, CancellationToken cancellationToken) { ITemporaryStorage storage; if (map.TryGetValue(tree, out storage)) { // we already have it serialized to temporary storage return; } // tree will be always held alive in memory, but nodes come and go. serialize nodes to storage using (var stream = SerializableBytes.CreateWritableStream()) { root.SerializeTo(stream, cancellationToken); stream.Position = 0; storage = service.CreateTemporaryStorage(cancellationToken); storage.WriteStream(stream, cancellationToken); } SaveTreeToMap(tree, storage); }
public async Task <bool> SerializeAsync( IPersistentStorageService persistentService, Project project, TextDocument?textDocument, string key, ImmutableArray <DiagnosticData> items, CancellationToken cancellationToken ) { Contract.ThrowIfFalse(textDocument == null || textDocument.Project == project); using var stream = SerializableBytes.CreateWritableStream(); using (var writer = new ObjectWriter(stream, leaveOpen: true, cancellationToken)) { WriteDiagnosticData(writer, items, cancellationToken); } var storage = await persistentService .GetStorageAsync(project.Solution, cancellationToken) .ConfigureAwait(false); await using var _ = storage.ConfigureAwait(false); stream.Position = 0; var writeTask = (textDocument != null) ? textDocument is Document document ? storage.WriteStreamAsync(document, key, stream, cancellationToken) : storage.WriteStreamAsync( GetSerializationKeyForNonSourceDocument(textDocument, key), stream, cancellationToken ) : storage.WriteStreamAsync(project, key, stream, cancellationToken); return(await writeTask.ConfigureAwait(false)); }
public void TestTemporaryStorageScaling() { // This will churn through 4GB of memory. It validates that we don't // use up our address space in a 32 bit process. if (Environment.Is64BitOperatingSystem && !Environment.Is64BitProcess) { using var workspace = new AdhocWorkspace(); var textFactory = Assert.IsType <TextFactoryService>( workspace.Services.GetService <ITextFactoryService>() ); var service = new TemporaryStorageServiceFactory.TemporaryStorageService( textFactory ); using var data = SerializableBytes.CreateWritableStream(); for (var i = 0; i < 1024 * 128; i++) { data.WriteByte(1); } // Create 4GB of memory mapped files var fileCount = (int)((long)4 * 1024 * 1024 * 1024 / data.Length); var storageHandles = new List <ITemporaryStreamStorage>(fileCount); for (var i = 0; i < fileCount; i++) { var s = service.CreateTemporaryStreamStorage(CancellationToken.None); storageHandles.Add(s); data.Position = 0; s.WriteStreamAsync(data).Wait(); } for (var i = 0; i < 1024 * 5; i++) { using var s = storageHandles[i].ReadStreamAsync().Result; Assert.Equal(1, s.ReadByte()); storageHandles[i].Dispose(); } } }
public void TestTemporaryStorageStream() { var textFactory = new TextFactoryService(); var service = new TemporaryStorageServiceFactory.TemporaryStorageService(textFactory); var temporaryStorage = service.CreateTemporaryStreamStorage(System.Threading.CancellationToken.None); using var data = SerializableBytes.CreateWritableStream(); for (var i = 0; i < SharedPools.ByteBufferSize; i++) { data.WriteByte((byte)(i % 2)); } data.Position = 0; temporaryStorage.WriteStreamAsync(data).Wait(); using var result = temporaryStorage.ReadStreamAsync().Result; Assert.Equal(data.Length, result.Length); for (var i = 0; i < SharedPools.ByteBufferSize; i++) { Assert.Equal(i % 2, result.ReadByte()); } }
private async Task <bool> SaveAsync(Document document, CancellationToken cancellationToken) { var solution = document.Project.Solution; var persistentStorageService = (IChecksummedPersistentStorageService)solution.Workspace.Services.GetRequiredService <IPersistentStorageService>(); try { var storage = await persistentStorageService .GetStorageAsync(solution, checkBranchId : false, cancellationToken) .ConfigureAwait(false); await using var _ = storage.ConfigureAwait(false); using var stream = SerializableBytes.CreateWritableStream(); using (var writer = new ObjectWriter(stream, leaveOpen: true, cancellationToken)) { WriteTo(writer); } stream.Position = 0; return(await storage .WriteStreamAsync( document, PersistenceName, stream, this.Checksum, cancellationToken ) .ConfigureAwait(false)); } catch (Exception e) when(IOUtilities.IsNormalIOException(e)) { // Storage APIs can throw arbitrary exceptions. } return(false); }
public void WritableStreamTest2() { using (var expected = new MemoryStream()) { for (var i = 0; i < 10000; i++) { expected.WriteByte((byte)(i % byte.MaxValue)); } expected.Position = 0; using (var stream = SerializableBytes.CreateWritableStream()) { for (var i = 0; i < 10000; i++) { stream.WriteByte((byte)(i % byte.MaxValue)); } Assert.Equal(expected.Length, stream.Length); stream.Position = 0; int index = 0; int count; var bytes = new byte[1000]; while ((count = stream.Read(bytes, 0, bytes.Length)) > 0) { for (var i = 0; i < count; i++) { Assert.Equal((byte)(index % byte.MaxValue), bytes[i]); index++; } } Assert.Equal(index, stream.Length); } } }
public async Task <Deltas> EmitProjectDeltaAsync(Project project, EmitBaseline baseline, CancellationToken cancellationToken) { try { Debug.Assert(!_stoppedAtException); var changes = await GetProjectChangesAsync(project, cancellationToken).ConfigureAwait(false); var currentCompilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false); var allAddedSymbols = await GetAllAddedSymbols(cancellationToken).ConfigureAwait(false); var pdbStream = new MemoryStream(); var updatedMethods = new List <MethodDefinitionHandle>(); using (var metadataStream = SerializableBytes.CreateWritableStream()) using (var ilStream = SerializableBytes.CreateWritableStream()) { EmitDifferenceResult result = currentCompilation.EmitDifference( baseline, changes.SemanticEdits, s => allAddedSymbols?.Contains(s) ?? false, metadataStream, ilStream, pdbStream, updatedMethods, cancellationToken); int[] updateMethodTokens = updatedMethods.Select(h => MetadataTokens.GetToken(h)).ToArray(); return(new Deltas(ilStream.ToArray(), metadataStream.ToArray(), updateMethodTokens, pdbStream, changes.LineChanges, result)); } } catch (Exception e) when(FatalError.ReportWithoutCrash(e)) { // recover (cancel EnC) return(null); } }
public static Checksum CreateChecksum(AnalyzerReference reference, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); using var stream = SerializableBytes.CreateWritableStream(); using (var writer = new ObjectWriter(stream, leaveOpen: true, cancellationToken)) { switch (reference) { case AnalyzerFileReference file: writer.WriteString(file.FullPath); writer.WriteBoolean(IsAnalyzerReferenceWithShadowCopyLoader(file)); break; default: throw ExceptionUtilities.UnexpectedValue(reference); } } stream.Position = 0; return(Checksum.Create(stream)); }
public void TestTemporaryStorageStream() { using var workspace = new AdhocWorkspace(); var textFactory = Assert.IsType <TextFactoryService>(workspace.Services.GetService <ITextFactoryService>()); var service = Assert.IsType <TemporaryStorageService>(workspace.Services.GetRequiredService <ITemporaryStorageService>()); var temporaryStorage = service.CreateTemporaryStreamStorage(); using var data = SerializableBytes.CreateWritableStream(); for (var i = 0; i < SharedPools.ByteBufferSize; i++) { data.WriteByte((byte)(i % 2)); } data.Position = 0; temporaryStorage.WriteStreamAsync(data).Wait(); using var result = temporaryStorage.ReadStreamAsync().Result; Assert.Equal(data.Length, result.Length); for (var i = 0; i < SharedPools.ByteBufferSize; i++) { Assert.Equal(i % 2, result.ReadByte()); } }
public void TestSourceTextSerialization() { var textService = new TextFactoryService(); var maxSize = SourceTextExtensions.SourceTextLengthThreshold * 3; var sb = new StringBuilder(0, maxSize); for (var i = 0; i < maxSize; i++) { var originalText = CreateSourceText(sb, i); using var stream = SerializableBytes.CreateWritableStream(); using var writer = new ObjectWriter(stream); originalText.WriteTo(writer, CancellationToken.None); stream.Position = 0; using var reader = ObjectReader.TryGetReader(stream); var recovered = SourceTextExtensions.ReadFrom(textService, reader, originalText.Encoding, CancellationToken.None); Assert.Equal(originalText.ToString(), recovered.ToString()); } }