public void should_add_an_item()
{
var setToRemoveFrom = new ConcurrentSet<int>(Enumerable.Range(1, 5));
setToRemoveFrom.Add(6);
setToRemoveFrom.ShouldEqual(new[] { 1, 2, 3, 4, 5, 6 });
}
public void ConcurrentTest()
{
var cset = new ConcurrentSet<int>();
Action a1 = () => { for (int i = 0; i < 1000000; i++) cset.Add(i); };
Action a2 = () => { for (int i = 1000000; i < 2000000; i++) cset.Add(i); };
Action a3 = () => { for (int i = 2000000; i < 3000000; i++) cset.Add(i); };
Action a4 = () => { for (int i = 3000000; i < 4000000; i++) cset.Add(i); };
bool b1 = false;
bool b2 = false;
bool b3 = false;
bool b4 = false;
a1.BeginInvoke(iar =>
{
a1.EndInvoke(iar);
b1 = true;
}, null);
a2.BeginInvoke(iar =>
{
a2.EndInvoke(iar);
b2 = true;
}, null);
a3.BeginInvoke(iar =>
{
a3.EndInvoke(iar);
b3 = true;
}, null);
a4.BeginInvoke(iar =>
{
a4.EndInvoke(iar);
b4 = true;
}, null);
while (!(b1 && b2 && b3 && b4))
{
Thread.Sleep(10);
}
Assert.AreEqual(4000000, cset.Count());
}
private static async Task<ISymbol> ResolveAsync(
Document document, DeclaredSymbolInfo info, ConcurrentSet<SemanticModel> cachedModels,
CancellationToken cancellationToken)
{
var semanticModel = await document.GetSemanticModelAsync(cancellationToken).ConfigureAwait(false);
cachedModels.Add(semanticModel);
return info.Resolve(semanticModel, cancellationToken);
}
private PrefetchingBehavior GetPrefetcherFor(Etag fromEtag, ConcurrentSet<PrefetchingBehavior> usedPrefetchers)
{
foreach (var prefetchingBehavior in prefetchingBehaviors)
{
if (prefetchingBehavior.CanUsePrefetcherToLoadFrom(fromEtag) && usedPrefetchers.TryAdd(prefetchingBehavior))
return prefetchingBehavior;
}
var newPrefetcher = prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Indexer, autoTuner);
prefetchingBehaviors.Add(newPrefetcher);
usedPrefetchers.Add(newPrefetcher);
return newPrefetcher;
}
public void AfterDelete(string key, Guid?lastDocumentEtag)
{
documentsToRemove.Add(new DocumentToRemove(key, lastDocumentEtag));
}
public void RegisterLowMemoryHandler(ILowMemoryHandler handler)
{
_lowMemoryHandlers.Add(new WeakReference <ILowMemoryHandler>(handler));
}
public void RegisterServerHttpTraceTransport(IEventsTransport transport)
{
serverHttpTrace.Add(transport);
transport.Disconnected += () => serverHttpTrace.TryRemove(transport);
}
public void SetDirty(NodeHandle n) => dirtyRegions.Add(Region(n));
private async Task DetermineAllSymbolsCoreAsync(
SymbolAndProjectId symbolAndProjectId, ConcurrentSet<SymbolAndProjectId> result)
{
_cancellationToken.ThrowIfCancellationRequested();
var searchSymbolAndProjectId = MapToAppropriateSymbol(symbolAndProjectId);
// 2) Try to map this back to source symbol if this was a metadata symbol.
var sourceSymbolAndProjectId = await SymbolFinder.FindSourceDefinitionAsync(searchSymbolAndProjectId, _solution, _cancellationToken).ConfigureAwait(false);
if (sourceSymbolAndProjectId.Symbol != null)
{
searchSymbolAndProjectId = sourceSymbolAndProjectId;
}
var searchSymbol = searchSymbolAndProjectId.Symbol;
if (searchSymbol != null && result.Add(searchSymbolAndProjectId))
{
await _progress.OnDefinitionFoundAsync(searchSymbolAndProjectId).ConfigureAwait(false);
// get project to search
var projects = GetProjectScope();
_cancellationToken.ThrowIfCancellationRequested();
List<Task> finderTasks = new List<Task>();
foreach (var f in _finders)
{
finderTasks.Add(Task.Run(async () =>
{
var symbolTasks = new List<Task>();
var symbols = await f.DetermineCascadedSymbolsAsync(
searchSymbolAndProjectId, _solution, projects, _cancellationToken).ConfigureAwait(false);
AddSymbolTasks(result, symbols, symbolTasks);
// Defer to the language to see if it wants to cascade here in some special way.
var symbolProject = _solution.GetProject(searchSymbol.ContainingAssembly);
var service = symbolProject?.LanguageServices.GetService<ILanguageServiceReferenceFinder>();
if (service != null)
{
symbols = await service.DetermineCascadedSymbolsAsync(
searchSymbolAndProjectId, symbolProject, _cancellationToken).ConfigureAwait(false);
AddSymbolTasks(result, symbols, symbolTasks);
}
_cancellationToken.ThrowIfCancellationRequested();
await Task.WhenAll(symbolTasks).ConfigureAwait(false);
}, _cancellationToken));
}
await Task.WhenAll(finderTasks).ConfigureAwait(false);
}
}
public void ExecuteBatch <T>(IList <T> src, Action <T> action, DocumentDatabase database = null, string description = null,
bool allowPartialBatchResumption = false)
{
//, int completedMultiplier = 2, int freeThreadsMultiplier = 2, int maxWaitMultiplier = 1
switch (src.Count)
{
case 0:
return;
case 1:
//if we have only one source to go through,
//we should execute it in the current thread, without using RTP threads
ExecuteSingleBatchSynchronously(src, action, description, database);
return;
}
var now = DateTime.UtcNow;
CountdownEvent countdownEvent;
var itemsCount = 0;
var batch = new BatchStatistics
{
Total = src.Count,
Completed = 0
};
if (_concurrentEvents.TryDequeue(out countdownEvent) == false)
{
countdownEvent = new CountdownEvent(src.Count);
}
else
{
countdownEvent.Reset(src.Count);
}
var exceptions = new ConcurrentSet <Exception>();
var currentBatchTasks = new List <ThreadTask>(src.Count);
for (; itemsCount < src.Count && _ct.IsCancellationRequested == false; itemsCount++)
{
var copy = itemsCount;
var threadTask = new ThreadTask
{
Action = () =>
{
try
{
database?.WorkContext.CancellationToken.ThrowIfCancellationRequested();
action(src[copy]);
}
catch (Exception e)
{
exceptions.Add(e);
throw;
}
finally
{
countdownEvent.Signal();
}
},
Description = new OperationDescription
{
Type = OperationDescription.OperationType.Atomic,
PlainText = description,
From = itemsCount + 1,
To = itemsCount + 1,
Total = src.Count
},
Database = database,
BatchStats = batch,
EarlyBreak = allowPartialBatchResumption,
QueuedAt = now,
DoneEvent = countdownEvent,
Proccessing = 0
};
_tasks.Add(threadTask);
currentBatchTasks.Add(threadTask);
}
if (allowPartialBatchResumption == false)
{
WaitForBatchToCompletion(countdownEvent, currentBatchTasks, database?.WorkContext.CancellationToken);
}
else
{
WaitForBatchAllowingPartialBatchResumption(countdownEvent, batch, currentBatchTasks, database);
}
switch (exceptions.Count)
{
case 0:
return;
case 1:
ExceptionDispatchInfo.Capture(exceptions.First()).Throw();
break;
default:
throw new AggregateException(exceptions);
}
}
/// <summary>
/// Executes given function in batches on received collection
/// </summary>
public void ExecuteBatch <T>(IList <T> src, Action <IEnumerator <T> > action, DocumentDatabase database = null,
int pageSize = defaultPageSize, string description = null)
{
if (src.Count == 0)
{
return;
}
if (src.Count <= pageSize)
{
//if we have only none or less than pageSize,
//we should execute it in the current thread, without using RTP threads
ExecuteSingleBatchSynchronously(src, action, description, database);
return;
}
var ranges = new ConcurrentQueue <Tuple <int, int> >();
var now = DateTime.UtcNow;
var numOfBatches = (src.Count / pageSize) + (src.Count % pageSize == 0 ? 0 : 1);
CountdownEvent batchesCountdown;
if (_concurrentEvents.TryDequeue(out batchesCountdown) == false)
{
batchesCountdown = new CountdownEvent(numOfBatches);
}
else
{
batchesCountdown.Reset(numOfBatches);
}
var localTasks = new List <ThreadTask>();
var exceptions = new ConcurrentSet <Exception>();
for (var i = 0; i < src.Count; i += pageSize)
{
var rangeStart = i;
var rangeEnd = i + pageSize - 1 < src.Count ? i + pageSize - 1 : src.Count - 1;
ranges.Enqueue(Tuple.Create(rangeStart, rangeEnd));
var threadTask = new ThreadTask
{
Action = () =>
{
var numOfBatchesUsed = new Reference <int>();
try
{
database?.WorkContext.CancellationToken.ThrowIfCancellationRequested();
Tuple <int, int> range;
if (ranges.TryDequeue(out range) == false)
{
return;
}
action(YieldFromRange(ranges, range, src, numOfBatchesUsed, database?.WorkContext.CancellationToken));
}
catch (Exception e)
{
exceptions.Add(e);
throw;
}
finally
{
// we will try to release the countdown event if we got here because cancellation token was requested, else wi will signal the countdown event according to the number of batches we've proccessed
if (database?.WorkContext.CancellationToken.IsCancellationRequested ?? false)
{
try
{
while (batchesCountdown.CurrentCount != 0)
{
batchesCountdown.Signal(1);
}
}
catch
{
}
}
else
{
var numOfBatchesUsedValue = numOfBatchesUsed.Value;
//handle the case when we are out of ranges
if (numOfBatchesUsedValue > 0)
{
try
{
batchesCountdown.Signal(numOfBatchesUsedValue);
}
catch (Exception)
{
}
}
}
}
},
Description = new OperationDescription
{
Type = OperationDescription.OperationType.Range,
PlainText = description,
From = rangeStart,
To = rangeEnd,
Total = src.Count
},
Database = database,
BatchStats = new BatchStatistics
{
Total = rangeEnd - rangeStart,
Completed = 0
},
QueuedAt = now,
DoneEvent = batchesCountdown,
};
localTasks.Add(threadTask);
}
// we must add the tasks to the global tasks after we added all the ranges
// to prevent the tasks from completing the range fast enough that it won't
// see the next range, see: http://issues.hibernatingrhinos.com/issue/RavenDB-4829
foreach (var threadTask in localTasks)
{
_tasks.Add(threadTask, _ct);
}
WaitForBatchToCompletion(batchesCountdown, localTasks, database?.WorkContext.CancellationToken);
switch (exceptions.Count)
{
case 0:
return;
case 1:
ExceptionDispatchInfo.Capture(exceptions.First()).Throw();
break;
default:
throw new AggregateException(exceptions);
}
}
static unsafe TestBase()
{
IgnoreProcessorAffinityChanges(ignore: true);
LicenseManager.AddLicenseStatusToLicenseLimitsException = true;
RachisStateMachine.EnableDebugLongCommit = true;
NativeMemory.GetCurrentUnmanagedThreadId = () => (ulong)Pal.rvn_get_current_thread_id();
Lucene.Net.Util.UnmanagedStringArray.Segment.AllocateMemory = NativeMemory.AllocateMemory;
Lucene.Net.Util.UnmanagedStringArray.Segment.FreeMemory = NativeMemory.Free;
BackupTask.DateTimeFormat = "yyyy-MM-dd-HH-mm-ss-fffffff";
RestorePointsBase.BackupFolderRegex = new Regex(@"([0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{2}(-[0-9]{2}-[0-9]{7})?).ravendb-(.+)-([A-Za-z]+)-(.+)$", RegexOptions.Compiled);
RestorePointsBase.FileNameRegex = new Regex(@"([0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{2}(-[0-9]{2}-[0-9]{7})?)", RegexOptions.Compiled);
var packagesPath = new PathSetting(RavenTestHelper.NewDataPath("NuGetPackages", 0, forceCreateDir: true));
GlobalPathsToDelete.Add(packagesPath.FullPath);
MultiSourceNuGetFetcher.Instance.Initialize(packagesPath, "https://api.nuget.org/v3/index.json");
#if DEBUG2
TaskScheduler.UnobservedTaskException += (sender, args) =>
{
if (args.Observed)
{
return;
}
var e = args.Exception.ExtractSingleInnerException();
var sb = new StringBuilder();
sb.AppendLine("===== UNOBSERVED TASK EXCEPTION =====");
sb.AppendLine(e.ExceptionToString(null));
sb.AppendLine("=====================================");
Console.WriteLine(sb.ToString());
};
#endif
if (PlatformDetails.RunningOnPosix == false &&
PlatformDetails.Is32Bits) // RavenDB-13655
{
ThreadPool.SetMinThreads(25, 25);
ThreadPool.SetMaxThreads(125, 125);
}
else
{
ThreadPool.SetMinThreads(250, 250);
}
RequestExecutor.RemoteCertificateValidationCallback += (sender, cert, chain, errors) => true;
var maxNumberOfConcurrentTests = Math.Max(ProcessorInfo.ProcessorCount / 2, 2);
if (int.TryParse(Environment.GetEnvironmentVariable("RAVEN_MAX_RUNNING_TESTS"), out var maxRunningTests))
{
maxNumberOfConcurrentTests = maxRunningTests;
}
else
{
var fileInfo = new FileInfo(XunitConfigurationFile);
if (fileInfo.Exists)
{
using (var file = File.OpenRead(XunitConfigurationFile))
using (var sr = new StreamReader(file))
{
var json = JObject.Parse(sr.ReadToEnd());
if (json.TryGetValue("maxRunningTests", out var testsToken))
{
maxNumberOfConcurrentTests = testsToken.Value <int>();
}
else if (json.TryGetValue("maxParallelThreads", out var threadsToken))
{
maxNumberOfConcurrentTests = threadsToken.Value <int>();
}
}
}
}
Console.WriteLine("Max number of concurrent tests is: " + maxNumberOfConcurrentTests);
ConcurrentTestsSemaphore = new SemaphoreSlim(maxNumberOfConcurrentTests, maxNumberOfConcurrentTests);
}
protected TestCertificatesHolder GenerateAndSaveSelfSignedCertificate(bool createNew = false)
{
var selfSignedCertificatePaths = _selfSignedCertificates;
if (selfSignedCertificatePaths != null && createNew == false)
{
return(ReturnCertificatesHolder(selfSignedCertificatePaths));
}
lock (typeof(TestBase))
{
selfSignedCertificatePaths = _selfSignedCertificates;
if (selfSignedCertificatePaths == null || createNew)
{
_selfSignedCertificates = selfSignedCertificatePaths = Generate();
}
return(ReturnCertificatesHolder(selfSignedCertificatePaths));
}
TestCertificatesHolder ReturnCertificatesHolder(TestCertificatesHolder certificates)
{
return(new TestCertificatesHolder(certificates, GetTempFileName));
}
TestCertificatesHolder Generate()
{
var log = new StringBuilder();
byte[] certBytes;
try
{
certBytes = CertificateUtils.CreateSelfSignedTestCertificate(Environment.MachineName, "RavenTestsServer", log);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to generate the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
X509Certificate2 serverCertificate;
try
{
serverCertificate = new X509Certificate2(certBytes, (string)null, X509KeyStorageFlags.MachineKeySet);
}
catch (Exception e)
{
throw new CryptographicException($"Unable to load the test certificate for the machine '{Environment.MachineName}'. Log: {log}", e);
}
if (certBytes.Length == 0)
{
throw new CryptographicException($"Test certificate length is 0 bytes. Machine: '{Environment.MachineName}', Log: {log}");
}
string serverCertificatePath = null;
try
{
serverCertificatePath = Path.GetTempFileName();
File.WriteAllBytes(serverCertificatePath, certBytes);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to write the test certificate to a temp file." +
$"tempFileName = {serverCertificatePath}" +
$"certBytes.Length = {certBytes.Length}" +
$"MachineName = {Environment.MachineName}.", e);
}
GlobalPathsToDelete.Add(serverCertificatePath);
SecretProtection.ValidatePrivateKey(serverCertificatePath, null, certBytes, out var pk);
var clientCertificate1Path = GenerateClientCertificate(1, serverCertificate, pk);
var clientCertificate2Path = GenerateClientCertificate(2, serverCertificate, pk);
var clientCertificate3Path = GenerateClientCertificate(3, serverCertificate, pk);
return(new TestCertificatesHolder(serverCertificatePath, clientCertificate1Path, clientCertificate2Path, clientCertificate3Path));
}
string GenerateClientCertificate(int index, X509Certificate2 serverCertificate, Org.BouncyCastle.Pkcs.AsymmetricKeyEntry pk)
{
CertificateUtils.CreateSelfSignedClientCertificate(
$"{Environment.MachineName}_CC_{index}",
new RavenServer.CertificateHolder
{
Certificate = serverCertificate,
PrivateKey = pk
},
out var certBytes, DateTime.UtcNow.Date.AddYears(5));
string clientCertificatePath = null;
try
{
clientCertificatePath = Path.GetTempFileName();
File.WriteAllBytes(clientCertificatePath, certBytes);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to write the test certificate to a temp file." +
$"tempFileName = {clientCertificatePath}" +
$"certBytes.Length = {certBytes.Length}" +
$"MachineName = {Environment.MachineName}.", e);
}
GlobalPathsToDelete.Add(clientCertificatePath);
return(clientCertificatePath);
}
}
protected virtual DocumentStore GetDocumentStore(
[CallerMemberName] string caller = null,
string dbSuffixIdentifier = null,
string path = null,
Action <DatabaseDocument> modifyDatabaseDocument = null,
Func <string, string> modifyName = null,
string apiKey = null)
{
var name = caller != null ? $"{caller}_{Interlocked.Increment(ref _counter)}" : Guid.NewGuid().ToString("N");
if (dbSuffixIdentifier != null)
{
name = $"{name}_{dbSuffixIdentifier}";
}
if (modifyName != null)
{
name = modifyName(name);
}
var hardDelete = true;
var runInMemory = true;
if (path == null)
{
path = NewDataPath(name);
}
else
{
hardDelete = false;
runInMemory = false;
}
var doc = MultiDatabase.CreateDatabaseDocument(name);
doc.Settings[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = runInMemory.ToString();
doc.Settings[RavenConfiguration.GetKey(x => x.Core.DataDirectory)] = path;
doc.Settings[RavenConfiguration.GetKey(x => x.Core.ThrowIfAnyIndexOrTransformerCouldNotBeOpened)] = "true";
doc.Settings[RavenConfiguration.GetKey(x => x.Indexing.MinNumberOfMapAttemptsAfterWhichBatchWillBeCanceledIfRunningLowOnMemory)] = int.MaxValue.ToString();
modifyDatabaseDocument?.Invoke(doc);
TransactionOperationContext context;
using (Server.ServerStore.ContextPool.AllocateOperationContext(out context))
{
context.OpenReadTransaction();
if (Server.ServerStore.Read(context, Constants.Database.Prefix + name) != null)
{
throw new InvalidOperationException($"Database '{name}' already exists");
}
}
var store = new DocumentStore
{
Url = UseFiddler(Server.Configuration.Core.ServerUrl),
DefaultDatabase = name,
ApiKey = apiKey
};
ModifyStore(store);
store.Initialize();
store.DatabaseCommands.GlobalAdmin.CreateDatabase(doc);
store.AfterDispose += (sender, args) =>
{
if (CreatedStores.TryRemove(store) == false)
{
return; // can happen if we are wrapping the store inside sharded one
}
if (Server.Disposed == false)
{
var databaseTask = Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(name);
if (databaseTask != null && databaseTask.IsCompleted == false)
{
databaseTask.Wait(); // if we are disposing store before database had chance to load then we need to wait
}
store.DatabaseCommands.GlobalAdmin.DeleteDatabase(name, hardDelete: hardDelete);
}
};
CreatedStores.Add(store);
return(store);
}
public void CanSubscribeToDocumentChanges()
{
using (var store = GetDocumentStore())
{
store.Changes().Task.Result
.ForAllDocuments()
.Subscribe(change =>
{
output.Add("passed_foralldocuments");
});
store.Changes().Task.Result
.ForDocumentsStartingWith("companies")
.Subscribe(change =>
{
output.Add("passed_forfordocumentsstartingwith");
});
store.Changes().Task.Result
.ForDocumentsInCollection("posts")
.Subscribe(change =>
{
output.Add("passed_ForDocumentsInCollection");
});
store.Changes().Task.Result
.ForDocumentsOfType(new Camera().GetType())
.Subscribe(changes =>
{
output.Add("passed_ForDocumentsOfType");
});
store.Changes().Task.Result
.ForDocument("companies/1")
.Subscribe(change =>
{
if (change.Type == DocumentChangeTypes.Delete)
{
output.Add("passed_fordocumentdelete");
}
});
using (var session = store.OpenSession())
{
session.Store(new User
{
Id = "users/1"
});
session.SaveChanges();
WaitUntilOutput("passed_foralldocuments");
session.Store(new Company
{
Id = "companies/1"
});
session.SaveChanges();
WaitUntilOutput("passed_forfordocumentsstartingwith");
session.Store(new Post
{
Id = "posts/1"
});
session.SaveChanges();
WaitUntilOutput("passed_ForDocumentsInCollection");
session.Store(new Camera
{
Id = "cameras/1"
});
session.SaveChanges();
WaitUntilOutput("passed_ForDocumentsOfType");
session.Delete("companies/1");
session.SaveChanges();
WaitUntilOutput("passed_fordocumentdelete");
}
}
}
private void AddExceptionDiagnostic(Diagnostic diagnostic)
{
_exceptionDiagnostics.Add(diagnostic);
}
internal int RunCore(TextWriter consoleOutput, ErrorLogger errorLogger, CancellationToken cancellationToken)
{
Debug.Assert(!Arguments.IsScriptRunner);
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.DisplayVersion)
{
PrintVersion(consoleOutput);
return Succeeded;
}
if (Arguments.DisplayLogo)
{
PrintLogo(consoleOutput);
}
if (Arguments.DisplayHelp)
{
PrintHelp(consoleOutput);
return Succeeded;
}
if (ReportErrors(Arguments.Errors, consoleOutput, errorLogger))
{
return Failed;
}
var touchedFilesLogger = (Arguments.TouchedFilesPath != null) ? new TouchedFileLogger() : null;
Compilation compilation = CreateCompilation(consoleOutput, touchedFilesLogger, errorLogger);
if (compilation == null)
{
return Failed;
}
var diagnosticInfos = new List<DiagnosticInfo>();
ImmutableArray<DiagnosticAnalyzer> analyzers = ResolveAnalyzersFromArguments(diagnosticInfos, MessageProvider);
var additionalTextFiles = ResolveAdditionalFilesFromArguments(diagnosticInfos, MessageProvider, touchedFilesLogger);
if (ReportErrors(diagnosticInfos, consoleOutput, errorLogger))
{
return Failed;
}
var diagnostics = new List<Diagnostic>();
ImmutableArray<EmbeddedText> embeddedTexts = AcquireEmbeddedTexts(compilation, diagnostics);
if (ReportErrors(diagnostics, consoleOutput, errorLogger))
{
return Failed;
}
bool reportAnalyzer = false;
CancellationTokenSource analyzerCts = null;
AnalyzerManager analyzerManager = null;
AnalyzerDriver analyzerDriver = null;
try
{
// Print the diagnostics produced during the parsing stage and exit if there were any errors.
if (ReportErrors(compilation.GetParseDiagnostics(), consoleOutput, errorLogger))
{
return Failed;
}
ConcurrentSet<Diagnostic> analyzerExceptionDiagnostics = null;
if (!analyzers.IsEmpty)
{
analyzerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
analyzerManager = new AnalyzerManager();
analyzerExceptionDiagnostics = new ConcurrentSet<Diagnostic>();
Action<Diagnostic> addExceptionDiagnostic = diagnostic => analyzerExceptionDiagnostics.Add(diagnostic);
var analyzerOptions = new AnalyzerOptions(ImmutableArray<AdditionalText>.CastUp(additionalTextFiles));
analyzerDriver = AnalyzerDriver.CreateAndAttachToCompilation(compilation, analyzers, analyzerOptions, analyzerManager, addExceptionDiagnostic, Arguments.ReportAnalyzer, out compilation, analyzerCts.Token);
reportAnalyzer = Arguments.ReportAnalyzer && !analyzers.IsEmpty;
}
if (ReportErrors(compilation.GetDeclarationDiagnostics(), consoleOutput, errorLogger))
{
return Failed;
}
cancellationToken.ThrowIfCancellationRequested();
string outputName = GetOutputFileName(compilation, cancellationToken);
var finalPeFilePath = Path.Combine(Arguments.OutputDirectory, outputName);
var finalPdbFilePath = Arguments.PdbPath ?? Path.ChangeExtension(finalPeFilePath, ".pdb");
var finalXmlFilePath = Arguments.DocumentationPath;
var diagnosticBag = DiagnosticBag.GetInstance();
Stream sourceLinkStreamOpt = null;
try
{
// NOTE: Unlike the PDB path, the XML doc path is not embedded in the assembly, so we don't need to pass it to emit.
var emitOptions = Arguments.EmitOptions.
WithOutputNameOverride(outputName).
WithPdbFilePath(finalPdbFilePath);
// The PDB path is emitted in it's entirety into the PE. This makes it impossible to have deterministic
// builds that occur in different source directories. To enable this we shave all path information from
// the PDB when specified by the user.
//
// This is a temporary work around to allow us to make progress with determinism. The following issue
// tracks getting an official solution here.
//
// https://github.com/dotnet/roslyn/issues/9813
if (Arguments.ParseOptions.Features.ContainsKey("pdb-path-determinism") && !string.IsNullOrEmpty(emitOptions.PdbFilePath))
{
emitOptions = emitOptions.WithPdbFilePath(Path.GetFileName(emitOptions.PdbFilePath));
}
if (Arguments.SourceLink != null)
{
sourceLinkStreamOpt = OpenFile(Arguments.SourceLink, consoleOutput, FileMode.Open, FileAccess.Read, FileShare.Read);
}
var moduleBeingBuilt = compilation.CheckOptionsAndCreateModuleBuilder(
diagnosticBag,
Arguments.ManifestResources,
emitOptions,
debugEntryPoint: null,
sourceLinkStream: sourceLinkStreamOpt,
embeddedTexts: embeddedTexts,
testData: null,
cancellationToken: cancellationToken);
if (moduleBeingBuilt != null)
{
bool success;
try
{
success = compilation.CompileMethods(
moduleBeingBuilt,
Arguments.EmitPdb,
diagnosticBag,
filterOpt: null,
cancellationToken: cancellationToken);
if (success)
{
// NOTE: as native compiler does, we generate the documentation file
// NOTE: 'in place', replacing the contents of the file if it exists
Stream xmlStreamOpt = null;
if (finalXmlFilePath != null)
{
xmlStreamOpt = OpenFile(finalXmlFilePath,
consoleOutput,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.ReadWrite | FileShare.Delete);
if (xmlStreamOpt == null)
{
return Failed;
}
xmlStreamOpt.SetLength(0);
}
using (xmlStreamOpt)
{
IEnumerable<DiagnosticInfo> errors;
using (var win32ResourceStreamOpt = GetWin32Resources(Arguments, compilation, out errors))
{
if (ReportErrors(errors, consoleOutput, errorLogger))
{
return Failed;
}
success = compilation.GenerateResourcesAndDocumentationComments(
moduleBeingBuilt,
xmlStreamOpt,
win32ResourceStreamOpt,
diagnosticBag,
cancellationToken);
}
// only report unused usings if we have success.
if (success)
{
compilation.ReportUnusedImports(null, diagnosticBag, cancellationToken);
}
}
}
compilation.CompleteTrees(null);
if (analyzerDriver != null)
{
// GetDiagnosticsAsync is called after ReportUnusedImports
// since that method calls EventQueue.TryComplete. Without
// TryComplete, we may miss diagnostics.
var hostDiagnostics = analyzerDriver.GetDiagnosticsAsync(compilation).Result;
diagnosticBag.AddRange(hostDiagnostics);
if (hostDiagnostics.Any(IsReportedError))
{
success = false;
}
}
}
finally
{
moduleBeingBuilt.CompilationFinished();
}
if (success)
{
bool emitPdbFile = Arguments.EmitPdb && emitOptions.DebugInformationFormat != Emit.DebugInformationFormat.Embedded;
using (var peStreamProvider = new CompilerEmitStreamProvider(this, finalPeFilePath))
using (var pdbStreamProviderOpt = emitPdbFile ? new CompilerEmitStreamProvider(this, finalPdbFilePath) : null)
{
success = compilation.SerializeToPeStream(
moduleBeingBuilt,
peStreamProvider,
pdbStreamProviderOpt,
testSymWriterFactory: null,
diagnostics: diagnosticBag,
metadataOnly: emitOptions.EmitMetadataOnly,
cancellationToken: cancellationToken);
if (success && touchedFilesLogger != null)
{
if (pdbStreamProviderOpt != null)
{
touchedFilesLogger.AddWritten(finalPdbFilePath);
}
touchedFilesLogger.AddWritten(finalPeFilePath);
}
}
}
}
var compileAndEmitDiagnostics = diagnosticBag.ToReadOnly();
if (ReportErrors(compileAndEmitDiagnostics, consoleOutput, errorLogger))
{
return Failed;
}
}
finally
{
diagnosticBag.Free();
sourceLinkStreamOpt?.Dispose();
}
cancellationToken.ThrowIfCancellationRequested();
if (analyzerExceptionDiagnostics != null && ReportErrors(analyzerExceptionDiagnostics, consoleOutput, errorLogger))
{
return Failed;
}
bool errorsReadingAdditionalFiles = false;
foreach (var additionalFile in additionalTextFiles)
{
if (ReportErrors(additionalFile.Diagnostics, consoleOutput, errorLogger))
{
errorsReadingAdditionalFiles = true;
}
}
if (errorsReadingAdditionalFiles)
{
return Failed;
}
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.TouchedFilesPath != null)
{
Debug.Assert(touchedFilesLogger != null);
if (finalXmlFilePath != null)
{
touchedFilesLogger.AddWritten(finalXmlFilePath);
}
var readStream = OpenFile(Arguments.TouchedFilesPath + ".read", consoleOutput, mode: FileMode.OpenOrCreate);
if (readStream == null)
{
return Failed;
}
using (var writer = new StreamWriter(readStream))
{
touchedFilesLogger.WriteReadPaths(writer);
}
var writtenStream = OpenFile(Arguments.TouchedFilesPath + ".write", consoleOutput, mode: FileMode.OpenOrCreate);
if (writtenStream == null)
{
return Failed;
}
using (var writer = new StreamWriter(writtenStream))
{
touchedFilesLogger.WriteWrittenPaths(writer);
}
}
}
finally
{
// At this point analyzers are already complete in which case this is a no-op. Or they are
// still running because the compilation failed before all of the compilation events were
// raised. In the latter case the driver, and all its associated state, will be waiting around
// for events that are never coming. Cancel now and let the clean up process begin.
if (analyzerCts != null)
{
analyzerCts.Cancel();
if (analyzerManager != null)
{
// Clear cached analyzer descriptors and unregister exception handlers hooked up to the LocalizableString fields of the associated descriptors.
analyzerManager.ClearAnalyzerState(analyzers);
}
if (reportAnalyzer)
{
ReportAnalyzerExecutionTime(consoleOutput, analyzerDriver, Culture, compilation.Options.ConcurrentBuild);
}
}
}
return Succeeded;
}
private int RunCore(TextWriter consoleOutput, ErrorLogger2 errorLogger, CancellationToken cancellationToken)
{
Debug.Assert(!Arguments.GetFieldOrProperty <bool>("IsScriptRunner"));
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.DisplayLogo)
{
PrintLogo(consoleOutput);
}
if (Arguments.DisplayHelp)
{
PrintHelp(consoleOutput);
return(Succeeded);
}
if (ReportErrors(Arguments.Errors, consoleOutput, errorLogger))
{
return(Failed);
}
var touchedFilesLogger = (Arguments.TouchedFilesPath != null) ? new TouchedFileLogger() : null;
Compilation compilation = CreateCompilation(consoleOutput, touchedFilesLogger, errorLogger);
if (compilation == null)
{
return(Failed);
}
var diagnostics = typeof(List <>).MakeGenericTypeFast(Refl.Type_DiagnosticInfo).InvokeFunction(".ctor");
var analyzers = ResolveAnalyzersFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
var additionalTextFiles = ResolveAdditionalFilesFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
if (ReportErrors((IEnumerable <DiagnosticInfo>)diagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
CancellationTokenSource analyzerCts = null;
AnalyzerManager analyzerManager = null;
AnalyzerDriver analyzerDriver = null;
try
{
Func <ImmutableArray <Diagnostic> > getAnalyzerDiagnostics = null;
ConcurrentSet <Diagnostic> analyzerExceptionDiagnostics = null;
if (!analyzers.IsDefaultOrEmpty)
{
analyzerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
analyzerManager = new AnalyzerManager();
analyzerExceptionDiagnostics = new ConcurrentSet <Diagnostic>();
Action <Diagnostic> addExceptionDiagnostic = diagnostic => analyzerExceptionDiagnostics.Add(diagnostic);
var analyzerOptions = new AnalyzerOptions(ImmutableArray.CreateRange(additionalTextFiles.Select(x => (AdditionalText)x)));
analyzerDriver = ReflAnalyzerDriver.CreateAndAttachToCompilation(
compilation,
analyzers,
analyzerOptions,
analyzerManager,
addExceptionDiagnostic,
Arguments.ReportAnalyzer,
out compilation,
analyzerCts.Token);
getAnalyzerDiagnostics = () => ((Task <ImmutableArray <Diagnostic> >)analyzerDriver.InvokeFunction("GetDiagnosticsAsync", compilation)).Result;
}
// Print the diagnostics produced during the parsing stage and exit if there were any errors.
if (ReportErrors(compilation.GetParseDiagnostics(), consoleOutput, errorLogger))
{
return(Failed);
}
if (ReportErrors(compilation.GetDeclarationDiagnostics(), consoleOutput, errorLogger))
{
return(Failed);
}
consoleOutput.WriteLine("Rewriting LINQ to procedural code...");
var originalCompilation = compilation;
var rewrittenLinqInvocations = 0;
var rewrittenMethods = 0;
foreach (var syntaxTree in compilation.SyntaxTrees)
{
var rewriter = new LinqRewriter(originalCompilation.GetSemanticModel(syntaxTree));
var rewritten = rewriter.Visit(syntaxTree.GetRoot());
compilation = compilation.ReplaceSyntaxTree(syntaxTree, rewritten.SyntaxTree);
rewrittenLinqInvocations += rewriter.RewrittenLinqQueries;
rewrittenMethods += rewriter.RewrittenMethods;
}
consoleOutput.WriteLine(string.Format("Rewritten {0} LINQ queries in {1} methods as procedural code.", rewrittenLinqInvocations, rewrittenMethods));
var k = compilation.GetDiagnostics().Where(x => x.Severity == DiagnosticSeverity.Error).ToList();
if (k.Count != 0)
{
foreach (var err in k)
{
System.Console.WriteLine("Could not compile rewritten method. Consider applying a [Shaman.Runtime.NoLinqRewrite] attribute.");
var m = err.Location.SourceTree.GetRoot().FindNode(err.Location.SourceSpan);
System.Console.WriteLine(err.Location.SourceTree.FilePath);
//var z = m.FirstAncestorOrSelf<CSharp.Syntax.BaseMethodDeclarationSyntax>(x => x.Kind() == CSharp.SyntaxKind.MethodDeclaration);
//compilation.GetSemanticModel().GetEnclosingSymbol(err.Location.SourceSpan.Start)
}
}
if (ReportErrors(compilation.GetParseDiagnostics().Where(x => x.Severity == DiagnosticSeverity.Error), consoleOutput, errorLogger))
{
return(Failed);
}
if (ReportErrors(compilation.GetDeclarationDiagnostics().Where(x => x.Severity == DiagnosticSeverity.Error), consoleOutput, errorLogger))
{
return(Failed);
}
EmitResult emitResult;
// NOTE: as native compiler does, we generate the documentation file
// NOTE: 'in place', replacing the contents of the file if it exists
string finalPeFilePath;
string finalPdbFilePath;
string finalXmlFilePath;
Stream xmlStreamOpt = null;
cancellationToken.ThrowIfCancellationRequested();
finalXmlFilePath = Arguments.DocumentationPath;
if (finalXmlFilePath != null)
{
xmlStreamOpt = OpenFile(finalXmlFilePath, consoleOutput, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete);
if (xmlStreamOpt == null)
{
return(Failed);
}
xmlStreamOpt.SetLength(0);
}
cancellationToken.ThrowIfCancellationRequested();
IEnumerable <DiagnosticInfo> errors;
using (var win32ResourceStreamOpt = GetWin32Resources(Arguments, compilation, out errors))
using (xmlStreamOpt)
{
if (ReportErrors(errors, consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
string outputName = GetOutputFileName(compilation, cancellationToken);
finalPeFilePath = Path.Combine(Arguments.OutputDirectory, outputName);
finalPdbFilePath = Arguments.PdbPath ?? Path.ChangeExtension(finalPeFilePath, ".pdb");
// NOTE: Unlike the PDB path, the XML doc path is not embedded in the assembly, so we don't need to pass it to emit.
var emitOptions = Arguments.EmitOptions.
WithOutputNameOverride(outputName).
WithPdbFilePath(finalPdbFilePath);
// The PDB path is emitted in it's entirety into the PE. This makes it impossible to have deterministic
// builds that occur in different source directories. To enable this we shave all path information from
// the PDB when specified by the user.
//
// This is a temporary work around to allow us to make progress with determinism. The following issue
// tracks getting an official solution here.
//
// https://github.com/dotnet/roslyn/issues/9813
if (Arguments.ParseOptions.Features.ContainsKey("pdb-path-determinism") && !string.IsNullOrEmpty(emitOptions.PdbFilePath))
{
emitOptions = emitOptions.WithPdbFilePath(Path.GetFileName(emitOptions.PdbFilePath));
}
//var dummyCompiler = Refl.Type_Csc.InvokeFunction(".ctor", backup_responseFile, (object)null, backup_args, backup_analyzerLoader);
var constr = Refl.Type_Csc.GetConstructors(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance).First();
var backup_buildPaths2 = Refl.Type_BuildPaths.InvokeFunction(".ctor", backup_buildPaths.ClientDirectory, backup_buildPaths.WorkingDirectory, backup_buildPaths.SdkDirectory);
var dummyCompiler = constr.Invoke(new object[] { backup_responseFile, backup_buildPaths2, backup_args, backup_analyzerLoader });
//var dummyCompiler = Activator.CreateInstance(Refl.Type_Csc, BindingFlags.Public|BindingFlags.NonPublic|BindingFlags.CreateInstance|BindingFlags.Instance, null, new object[]{ backup_responseFile, (object)null, backup_args, backup_analyzerLoader}, null);
using (var peStreamProvider = (IDisposable)Refl.Type_CompilerEmitStreamProvider.InvokeFunction(".ctor", dummyCompiler, finalPeFilePath))
using (var pdbStreamProviderOpt = Arguments.EmitPdb ? (IDisposable)Refl.Type_CompilerEmitStreamProvider.InvokeFunction(".ctor", dummyCompiler, finalPdbFilePath) : null)
{
/*
* internal EmitResult Emit(
* Compilation.EmitStreamProvider peStreamProvider,
* Compilation.EmitStreamProvider pdbStreamProvider,
* Compilation.EmitStreamProvider xmlDocumentationStreamProvider,
* Compilation.EmitStreamProvider win32ResourcesStreamProvider,
* IEnumerable<ResourceDescription> manifestResources,
* EmitOptions options,
* IMethodSymbol debugEntryPoint,
* CompilationTestData testData,
* Func<ImmutableArray<Diagnostic>> getHostDiagnostics,
* CancellationToken cancellationToken)
*/
emitResult = ReflCompilation.Emit(
compilation,
peStreamProvider,
pdbStreamProviderOpt,
(xmlStreamOpt != null) ? Refl.Type_SimpleEmitStreamProvider.InvokeFunction(".ctor", xmlStreamOpt) : null,
(win32ResourceStreamOpt != null) ? Refl.Type_SimpleEmitStreamProvider.InvokeFunction(".ctor", win32ResourceStreamOpt) : null,
Arguments.ManifestResources,
emitOptions,
/*debugEntryPoint:*/ null,
/*testData*/ (object)null,
/*getHostDiagnostics: */ getAnalyzerDiagnostics,
/*cancellationToken: */ cancellationToken);
if (emitResult.Success && touchedFilesLogger != null)
{
if (pdbStreamProviderOpt != null)
{
touchedFilesLogger.InvokeAction("AddWritten", finalPdbFilePath);
}
touchedFilesLogger.InvokeAction("AddWritten", finalPeFilePath);
}
}
}
GenerateSqmData(Arguments.CompilationOptions, emitResult.Diagnostics);
if (ReportErrors(emitResult.Diagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
if (analyzerExceptionDiagnostics != null && ReportErrors(analyzerExceptionDiagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
bool errorsReadingAdditionalFiles = false;
foreach (var additionalFile in additionalTextFiles)
{
if (ReportErrors(additionalFile.GetFieldOrProperty <IEnumerable <Diagnostic> >("Diagnostics"), consoleOutput, errorLogger))
{
errorsReadingAdditionalFiles = true;
}
}
if (errorsReadingAdditionalFiles)
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.TouchedFilesPath != null)
{
Debug.Assert(touchedFilesLogger != null);
if (finalXmlFilePath != null)
{
touchedFilesLogger.InvokeAction("AddWritten", finalXmlFilePath);
}
var readStream = OpenFile(Arguments.TouchedFilesPath + ".read", consoleOutput, mode: FileMode.OpenOrCreate);
if (readStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(readStream))
{
touchedFilesLogger.InvokeAction("WriteReadPaths", writer);
}
var writtenStream = OpenFile(Arguments.TouchedFilesPath + ".write", consoleOutput, mode: FileMode.OpenOrCreate);
if (writtenStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(writtenStream))
{
touchedFilesLogger.InvokeAction("WriteWrittenPaths", writer);
}
}
}
finally
{
// At this point analyzers are already complete in which case this is a no-op. Or they are
// still running because the compilation failed before all of the compilation events were
// raised. In the latter case the driver, and all its associated state, will be waiting around
// for events that are never coming. Cancel now and let the clean up process begin.
if (analyzerCts != null)
{
analyzerCts.Cancel();
if (analyzerManager != null)
{
// Clear cached analyzer descriptors and unregister exception handlers hooked up to the LocalizableString fields of the associated descriptors.
analyzerManager.InvokeAction("ClearAnalyzerState", analyzers);
}
if (Arguments.ReportAnalyzer && analyzerDriver != null && compilation != null)
{
ReportAnalyzerExecutionTime(consoleOutput, analyzerDriver, Culture, compilation.Options.ConcurrentBuild);
}
}
}
return(Succeeded);
}
private int RunCore(TextWriter consoleOutput, ErrorLogger errorLogger, CancellationToken cancellationToken)
{
Debug.Assert(!Arguments.IsScriptRunner);
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.DisplayLogo)
{
PrintLogo(consoleOutput);
}
if (Arguments.DisplayHelp)
{
PrintHelp(consoleOutput);
return Succeeded;
}
if (ReportErrors(Arguments.Errors, consoleOutput, errorLogger))
{
return Failed;
}
var touchedFilesLogger = (Arguments.TouchedFilesPath != null) ? new TouchedFileLogger() : null;
Compilation compilation = CreateCompilation(consoleOutput, touchedFilesLogger, errorLogger);
if (compilation == null)
{
return Failed;
}
var diagnostics = new List<DiagnosticInfo>();
var analyzers = ResolveAnalyzersFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
var additionalTextFiles = ResolveAdditionalFilesFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
if (ReportErrors(diagnostics, consoleOutput, errorLogger))
{
return Failed;
}
cancellationToken.ThrowIfCancellationRequested();
CancellationTokenSource analyzerCts = null;
AnalyzerManager analyzerManager = null;
AnalyzerDriver analyzerDriver = null;
try
{
Func<ImmutableArray<Diagnostic>> getAnalyzerDiagnostics = null;
ConcurrentSet<Diagnostic> analyzerExceptionDiagnostics = null;
if (!analyzers.IsDefaultOrEmpty)
{
analyzerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
analyzerManager = new AnalyzerManager();
analyzerExceptionDiagnostics = new ConcurrentSet<Diagnostic>();
Action<Diagnostic> addExceptionDiagnostic = diagnostic => analyzerExceptionDiagnostics.Add(diagnostic);
var analyzerOptions = new AnalyzerOptions(ImmutableArray<AdditionalText>.CastUp(additionalTextFiles));
analyzerDriver = AnalyzerDriver.CreateAndAttachToCompilation(compilation, analyzers, analyzerOptions, analyzerManager, addExceptionDiagnostic, Arguments.ReportAnalyzer, out compilation, analyzerCts.Token);
getAnalyzerDiagnostics = () => analyzerDriver.GetDiagnosticsAsync(compilation).Result;
}
// Print the diagnostics produced during the parsing stage and exit if there were any errors.
if (ReportErrors(compilation.GetParseDiagnostics(), consoleOutput, errorLogger))
{
return Failed;
}
if (ReportErrors(compilation.GetDeclarationDiagnostics(), consoleOutput, errorLogger))
{
return Failed;
}
EmitResult emitResult;
// NOTE: as native compiler does, we generate the documentation file
// NOTE: 'in place', replacing the contents of the file if it exists
string finalPeFilePath;
string finalPdbFilePath;
string finalXmlFilePath;
Stream xmlStreamOpt = null;
cancellationToken.ThrowIfCancellationRequested();
finalXmlFilePath = Arguments.DocumentationPath;
if (finalXmlFilePath != null)
{
xmlStreamOpt = OpenFile(finalXmlFilePath, consoleOutput, PortableShim.FileMode.OpenOrCreate, PortableShim.FileAccess.Write, PortableShim.FileShare.ReadWriteBitwiseOrDelete);
if (xmlStreamOpt == null)
{
return Failed;
}
xmlStreamOpt.SetLength(0);
}
cancellationToken.ThrowIfCancellationRequested();
IEnumerable<DiagnosticInfo> errors;
using (var win32ResourceStreamOpt = GetWin32Resources(Arguments, compilation, out errors))
using (xmlStreamOpt)
{
if (ReportErrors(errors, consoleOutput, errorLogger))
{
return Failed;
}
cancellationToken.ThrowIfCancellationRequested();
string outputName = GetOutputFileName(compilation, cancellationToken);
finalPeFilePath = Path.Combine(Arguments.OutputDirectory, outputName);
finalPdbFilePath = Arguments.PdbPath ?? Path.ChangeExtension(finalPeFilePath, ".pdb");
// NOTE: Unlike the PDB path, the XML doc path is not embedded in the assembly, so we don't need to pass it to emit.
var emitOptions = Arguments.EmitOptions.
WithOutputNameOverride(outputName).
WithPdbFilePath(finalPdbFilePath);
using (var peStreamProvider = new CompilerEmitStreamProvider(this, finalPeFilePath))
using (var pdbStreamProviderOpt = Arguments.EmitPdb ? new CompilerEmitStreamProvider(this, finalPdbFilePath) : null)
{
emitResult = compilation.Emit(
peStreamProvider,
pdbStreamProviderOpt,
(xmlStreamOpt != null) ? new Compilation.SimpleEmitStreamProvider(xmlStreamOpt) : null,
(win32ResourceStreamOpt != null) ? new Compilation.SimpleEmitStreamProvider(win32ResourceStreamOpt) : null,
Arguments.ManifestResources,
emitOptions,
debugEntryPoint: null,
getHostDiagnostics: getAnalyzerDiagnostics,
cancellationToken: cancellationToken);
if (emitResult.Success && touchedFilesLogger != null)
{
if (pdbStreamProviderOpt != null)
{
touchedFilesLogger.AddWritten(finalPdbFilePath);
}
touchedFilesLogger.AddWritten(finalPeFilePath);
}
}
}
GenerateSqmData(Arguments.CompilationOptions, emitResult.Diagnostics);
if (ReportErrors(emitResult.Diagnostics, consoleOutput, errorLogger))
{
return Failed;
}
cancellationToken.ThrowIfCancellationRequested();
if (analyzerExceptionDiagnostics != null && ReportErrors(analyzerExceptionDiagnostics, consoleOutput, errorLogger))
{
return Failed;
}
bool errorsReadingAdditionalFiles = false;
foreach (var additionalFile in additionalTextFiles)
{
if (ReportErrors(additionalFile.Diagnostics, consoleOutput, errorLogger))
{
errorsReadingAdditionalFiles = true;
}
}
if (errorsReadingAdditionalFiles)
{
return Failed;
}
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.TouchedFilesPath != null)
{
Debug.Assert(touchedFilesLogger != null);
if (finalXmlFilePath != null)
{
touchedFilesLogger.AddWritten(finalXmlFilePath);
}
var readStream = OpenFile(Arguments.TouchedFilesPath + ".read", consoleOutput, mode: PortableShim.FileMode.OpenOrCreate);
if (readStream == null)
{
return Failed;
}
using (var writer = new StreamWriter(readStream))
{
touchedFilesLogger.WriteReadPaths(writer);
}
var writtenStream = OpenFile(Arguments.TouchedFilesPath + ".write", consoleOutput, mode: PortableShim.FileMode.OpenOrCreate);
if (writtenStream == null)
{
return Failed;
}
using (var writer = new StreamWriter(writtenStream))
{
touchedFilesLogger.WriteWrittenPaths(writer);
}
}
}
finally
{
// At this point analyzers are already complete in which case this is a no-op. Or they are
// still running because the compilation failed before all of the compilation events were
// raised. In the latter case the driver, and all its associated state, will be waiting around
// for events that are never coming. Cancel now and let the clean up process begin.
if (analyzerCts != null)
{
analyzerCts.Cancel();
if (analyzerManager != null)
{
// Clear cached analyzer descriptors and unregister exception handlers hooked up to the LocalizableString fields of the associated descriptors.
analyzerManager.ClearAnalyzerState(analyzers);
}
if (Arguments.ReportAnalyzer && analyzerDriver != null && compilation != null)
{
ReportAnalyzerExecutionTime(consoleOutput, analyzerDriver, Culture, compilation.Options.ConcurrentBuild);
}
}
}
return Succeeded;
}
private int RunCore(TextWriter consoleOutput, CancellationToken cancellationToken)
{
Debug.Assert(!Arguments.IsInteractive);
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.DisplayLogo)
{
PrintLogo(consoleOutput);
}
if (Arguments.DisplayHelp)
{
PrintHelp(consoleOutput);
return(Succeeded);
}
if (PrintErrors(Arguments.Errors, consoleOutput))
{
return(Failed);
}
var touchedFilesLogger = (Arguments.TouchedFilesPath != null) ? new TouchedFileLogger() : null;
Compilation compilation = CreateCompilation(consoleOutput, touchedFilesLogger);
if (compilation == null)
{
return(Failed);
}
var diagnostics = new List <DiagnosticInfo>();
var analyzers = ResolveAnalyzersFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
var additionalTextFiles = ResolveAdditionalFilesFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
if (PrintErrors(diagnostics, consoleOutput))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
var analyzerOptions = new AnalyzerOptions(ImmutableArray.Create <AdditionalText, AdditionalTextFile>(additionalTextFiles));
AnalyzerDriver analyzerDriver = null;
AnalyzerManager analyzerManager = null;
ConcurrentSet <Diagnostic> analyzerExceptionDiagnostics = null;
if (!analyzers.IsDefaultOrEmpty)
{
analyzerManager = new AnalyzerManager();
analyzerExceptionDiagnostics = new ConcurrentSet <Diagnostic>();
Action <Diagnostic> addExceptionDiagnostic = diagnostic => analyzerExceptionDiagnostics.Add(diagnostic);
analyzerDriver = AnalyzerDriver.Create(compilation, analyzers, analyzerOptions, analyzerManager, addExceptionDiagnostic, out compilation, cancellationToken);
}
// Print the diagnostics produced during the parsing stage and exit if there were any errors.
if (PrintErrors(compilation.GetParseDiagnostics(), consoleOutput))
{
return(Failed);
}
if (PrintErrors(compilation.GetDeclarationDiagnostics(), consoleOutput))
{
return(Failed);
}
EmitResult emitResult;
// EDMAURER: Don't yet know if there are method body errors. don't overwrite
// any existing output files until the compilation is known to be successful.
string tempExeFilename = null;
string tempPdbFilename = null;
// NOTE: as native compiler does, we generate the documentation file
// NOTE: 'in place', replacing the contents of the file if it exists
try
{
FileStream output = CreateTempFile(consoleOutput, out tempExeFilename);
// Can happen when temp directory is "full"
if (output == null)
{
return(Failed);
}
string finalOutputPath;
string finalPdbFilePath;
string finalXmlFilePath;
using (output)
{
FileStream pdb = null;
FileStream xml = null;
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.EmitPdb)
{
pdb = CreateTempFile(consoleOutput, out tempPdbFilename);
if (pdb == null)
{
return(Failed);
}
}
cancellationToken.ThrowIfCancellationRequested();
finalXmlFilePath = Arguments.DocumentationPath;
if (finalXmlFilePath != null)
{
xml = OpenFile(finalXmlFilePath, consoleOutput, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete);
if (xml == null)
{
return(Failed);
}
xml.SetLength(0);
}
cancellationToken.ThrowIfCancellationRequested();
IEnumerable <DiagnosticInfo> errors;
using (var win32Res = GetWin32Resources(Arguments, compilation, out errors))
using (pdb)
using (xml)
{
if (PrintErrors(errors, consoleOutput))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
string outputName = GetOutputFileName(compilation, cancellationToken);
finalOutputPath = Path.Combine(Arguments.OutputDirectory, outputName);
finalPdbFilePath = Arguments.PdbPath ?? Path.ChangeExtension(finalOutputPath, ".pdb");
// NOTE: Unlike the PDB path, the XML doc path is not embedded in the assembly, so we don't need to pass it to emit.
var emitOptions = Arguments.EmitOptions.
WithOutputNameOverride(outputName).
WithPdbFilePath(finalPdbFilePath);
emitResult = compilation.Emit(output, pdb, xml, win32Res, Arguments.ManifestResources, emitOptions, cancellationToken);
}
}
GenerateSqmData(Arguments.CompilationOptions, emitResult.Diagnostics);
if (PrintErrors(emitResult.Diagnostics, consoleOutput))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
if (analyzerDriver != null)
{
var analyzerDiagnostics = analyzerDriver.GetDiagnosticsAsync().Result;
var allAnalyzerDiagnostics = analyzerDiagnostics.AddRange(analyzerExceptionDiagnostics);
if (PrintErrors(allAnalyzerDiagnostics, consoleOutput))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
}
bool errorsReadingAdditionalFiles = false;
foreach (var additionalFile in additionalTextFiles)
{
if (PrintErrors(additionalFile.Diagnostics, consoleOutput))
{
errorsReadingAdditionalFiles = true;
}
}
if (errorsReadingAdditionalFiles)
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
if (!TryDeleteFile(finalOutputPath, consoleOutput) || !TryMoveFile(tempExeFilename, finalOutputPath, consoleOutput))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
if (tempPdbFilename != null)
{
if (!TryDeleteFile(finalPdbFilePath, consoleOutput) || !TryMoveFile(tempPdbFilename, finalPdbFilePath, consoleOutput))
{
return(Failed);
}
}
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.TouchedFilesPath != null)
{
Debug.Assert(touchedFilesLogger != null);
touchedFilesLogger.AddWritten(tempExeFilename);
touchedFilesLogger.AddWritten(finalOutputPath);
if (tempPdbFilename != null)
{
touchedFilesLogger.AddWritten(tempPdbFilename);
touchedFilesLogger.AddWritten(finalPdbFilePath);
}
if (finalXmlFilePath != null)
{
touchedFilesLogger.AddWritten(finalXmlFilePath);
}
var readStream = OpenFile(Arguments.TouchedFilesPath + ".read", consoleOutput, FileMode.OpenOrCreate);
if (readStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(readStream))
{
touchedFilesLogger.WriteReadPaths(writer);
}
var writtenStream = OpenFile(Arguments.TouchedFilesPath + ".write", consoleOutput, FileMode.OpenOrCreate);
if (writtenStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(writtenStream))
{
touchedFilesLogger.WriteWrittenPaths(writer);
}
}
return(Succeeded);
}
finally
{
if (tempExeFilename != null)
{
TryDeleteFile(tempExeFilename, consoleOutput: null);
}
if (tempPdbFilename != null)
{
TryDeleteFile(tempPdbFilename, consoleOutput: null);
}
}
}
public static void Register(Action action)
{
_releaseMemoryBeforeGC.Add(new WeakReference <Action>(action));
}
private void CreateUploadTaskIfNeeded <S, T>(S settings, Action <S, FileStream, Progress> uploadToServer, T uploadStatus, string targetName)
where S : BackupSettings
where T : CloudUploadStatus
{
if (PeriodicBackupConfiguration.CanBackupUsing(settings) == false)
{
return;
}
Debug.Assert(uploadStatus != null);
var localUploadStatus = uploadStatus;
var thread = PoolOfThreads.GlobalRavenThreadPool.LongRunning(_ =>
{
try
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
NativeMemory.EnsureRegistered();
using (localUploadStatus.UpdateStats(_isFullBackup))
using (var fileStream = File.OpenRead(_backupUploaderSettings.BackupPath))
{
var uploadProgress = localUploadStatus.UploadProgress;
try
{
localUploadStatus.Skipped = false;
uploadProgress.ChangeState(UploadState.PendingUpload);
uploadProgress.SetTotal(fileStream.Length);
AddInfo($"Starting the upload of backup file to {targetName}.");
var bytesPutsPerSec = new MeterMetric();
long lastUploadedInBytes = 0;
var totalToUpload = new Size(uploadProgress.TotalInBytes, SizeUnit.Bytes).ToString();
var sw = Stopwatch.StartNew();
var progress = new Progress(uploadProgress)
{
OnUploadProgress = () =>
{
if (sw.ElapsedMilliseconds <= 1000)
{
return;
}
var totalUploadedInBytes = uploadProgress.UploadedInBytes;
bytesPutsPerSec.MarkSingleThreaded(totalUploadedInBytes - lastUploadedInBytes);
lastUploadedInBytes = totalUploadedInBytes;
var uploaded = new Size(totalUploadedInBytes, SizeUnit.Bytes);
uploadProgress.BytesPutsPerSec = bytesPutsPerSec.MeanRate;
AddInfo($"Uploaded: {uploaded} / {totalToUpload}");
sw.Restart();
}
};
uploadToServer(settings, fileStream, progress);
AddInfo($"Total uploaded: {totalToUpload}, took: {MsToHumanReadableString(uploadProgress.UploadTimeInMs)}");
}
finally
{
uploadProgress.ChangeState(UploadState.Done);
}
}
}
catch (OperationCanceledException e)
{
// shutting down
localUploadStatus.Exception = e.ToString();
_exceptions.Add(e);
}
catch (Exception e)
{
localUploadStatus.Exception = e.ToString();
_exceptions.Add(new InvalidOperationException($"Failed to upload the backup file to {targetName}.", e));
}
}, null, $"Upload backup file of database '{_backupUploaderSettings.DatabaseName}' to {targetName} (task: '{_backupUploaderSettings.TaskName}')");
_threads.Add(thread);
}
public void RegisterTransactionalStorageNotificationHandler(ITransactionalStorageNotificationHandler handler)
{
lowMemoryHandlers.Add(new WeakReference <ITransactionalStorageNotificationHandler>(handler));
}
protected virtual DocumentStore GetDocumentStore(Options options = null, [CallerMemberName] string caller = null)
{
try
{
lock (_getDocumentStoreSync)
{
options = options ?? Options.Default;
var serverToUse = options.Server ?? Server;
var name = GetDatabaseName(caller);
if (options.ModifyDatabaseName != null)
{
name = options.ModifyDatabaseName(name) ?? name;
}
var hardDelete = true;
var runInMemory = options.RunInMemory;
var pathToUse = options.Path;
if (runInMemory == false && options.ReplicationFactor > 1)
{
if (pathToUse == null)
{
// the folders will be assigned automatically
}
else
{
throw new InvalidOperationException($"You cannot set {nameof(Options)}.{nameof(Options.Path)} when, {nameof(Options)}.{nameof(Options.ReplicationFactor)} > 1 and {nameof(Options)}.{nameof(Options.RunInMemory)} == false.");
}
}
else if (pathToUse == null)
{
pathToUse = NewDataPath(name);
}
else
{
hardDelete = false;
runInMemory = false;
}
var doc = new DatabaseRecord(name)
{
Settings =
{
[RavenConfiguration.GetKey(x => x.Replication.ReplicationMinimalHeartbeat)] = "1",
[RavenConfiguration.GetKey(x => x.Replication.RetryReplicateAfter)] = "1",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = runInMemory.ToString(),
[RavenConfiguration.GetKey(x => x.Core.ThrowIfAnyIndexCannotBeOpened)] = "true",
[RavenConfiguration.GetKey(x => x.Indexing.MinNumberOfMapAttemptsAfterWhichBatchWillBeCanceledIfRunningLowOnMemory)] = int.MaxValue.ToString(),
}
};
if (pathToUse != null)
{
doc.Settings.Add(RavenConfiguration.GetKey(x => x.Core.DataDirectory), pathToUse);
}
if (options.Encrypted)
{
SetupForEncryptedDatabase(options, name, serverToUse, doc);
}
options.ModifyDatabaseRecord?.Invoke(doc);
var store = new DocumentStore
{
Urls = UseFiddler(serverToUse.WebUrl),
Database = name,
Certificate = options.ClientCertificate
};
options.ModifyDocumentStore?.Invoke(store);
//This gives too much error details in most cases, we don't need this now
store.RequestExecutorCreated += (sender, executor) =>
{
executor.AdditionalErrorInformation += sb => sb.AppendLine().Append(GetLastStatesFromAllServersOrderedByTime());
};
store.Initialize();
if (options.CreateDatabase)
{
if (Servers.Contains(serverToUse))
{
Servers.ForEach(server => CheckIfDatabaseExists(server, name));
}
else
{
CheckIfDatabaseExists(serverToUse, name);
}
long raftCommand;
try
{
if (options.AdminCertificate != null)
{
using (var adminStore = new DocumentStore
{
Urls = UseFiddler(serverToUse.WebUrl),
Database = name,
Certificate = options.AdminCertificate
}.Initialize())
{
raftCommand = adminStore.Maintenance.Server.Send(new CreateDatabaseOperation(doc, options.ReplicationFactor)).RaftCommandIndex;
}
}
else
{
raftCommand = store.Maintenance.Server.Send(new CreateDatabaseOperation(doc, options.ReplicationFactor)).RaftCommandIndex;
}
}
catch (ConcurrencyException)
{
var record = store.Maintenance.Server.Send(new GetDatabaseRecordOperation(name));
Assert.Equal(options.ReplicationFactor, record.Topology.ReplicationFactor);
raftCommand = record.Etag;
}
Assert.True(raftCommand > 0); //sanity check
if (Servers.Contains(serverToUse))
{
var timeout = TimeSpan.FromMinutes(Debugger.IsAttached ? 5 : 1);
AsyncHelpers.RunSync(async() => await WaitForRaftIndexToBeAppliedInCluster(raftCommand, timeout));
// skip 'wait for requests' on DocumentDatabase dispose
Servers.ForEach(server => ApplySkipDrainAllRequestsToDatabase(server, name));
}
else
{
ApplySkipDrainAllRequestsToDatabase(serverToUse, name);
}
}
store.BeforeDispose += (sender, args) =>
{
var realException = Context.GetException();
try
{
if (CreatedStores.TryRemove(store) == false)
{
return; // can happen if we are wrapping the store inside sharded one
}
DeleteDatabaseResult result = null;
if (options.DeleteDatabaseOnDispose)
{
result = DeleteDatabase(options, serverToUse, name, hardDelete, store);
}
if (Servers.Contains(serverToUse) && result != null)
{
var timeout = options.DeleteTimeout ?? TimeSpan.FromSeconds(Debugger.IsAttached ? 5 : 1);
AsyncHelpers.RunSync(async() => await WaitForRaftIndexToBeAppliedInCluster(result.RaftCommandIndex, timeout));
}
}
catch (Exception e)
{
if (realException != null)
{
throw new AggregateException(realException, e);
}
throw;
}
};
CreatedStores.Add(store);
return(store);
}
}
catch (TimeoutException te)
{
throw new TimeoutException($"{te.Message} {Environment.NewLine} {te.StackTrace}{Environment.NewLine}Servers states:{Environment.NewLine}{GetLastStatesFromAllServersOrderedByTime()}");
}
}
private void MaybeAddFutureBatch(JsonResults past, int indexingAge)
{
if (context.Configuration.DisableDocumentPreFetchingForIndexing || context.RunIndexing == false)
{
return;
}
if (context.Configuration.MaxNumberOfParallelIndexTasks == 1)
{
return;
}
if (past.Results.Length == 0 || past.LoadedFromDisk == false)
{
return;
}
if (futureIndexBatches.Count > 5) // we limit the number of future calls we do
{
var alreadyLoaded = futureIndexBatches.Sum(x =>
{
if (x.Task.IsCompleted)
{
return(x.Task.Result.Results.Length);
}
return(0);
});
if (alreadyLoaded > autoTuner.NumberOfItemsToIndexInSingleBatch)
{
return;
}
}
// ensure we don't do TOO much future cachings
if (MemoryStatistics.AvailableMemory <
context.Configuration.AvailableMemoryForRaisingIndexBatchSizeLimit)
{
return;
}
// we loaded the maximum amount, there are probably more items to read now.
var nextEtag = GetNextDocEtag(GetNextHighestEtag(past.Results));
var nextBatch = futureIndexBatches.FirstOrDefault(x => x.StartingEtag == nextEtag);
if (nextBatch != null)
{
return;
}
var futureBatchStat = new FutureBatchStats
{
Timestamp = SystemTime.UtcNow,
};
var sp = Stopwatch.StartNew();
context.AddFutureBatch(futureBatchStat);
futureIndexBatches.Add(new FutureIndexBatch
{
StartingEtag = nextEtag,
Age = indexingAge,
Task = System.Threading.Tasks.Task.Factory.StartNew(() =>
{
JsonResults jsonDocuments = null;
int localWork = workCounter;
while (context.RunIndexing)
{
jsonDocuments = GetJsonDocuments(nextEtag);
if (jsonDocuments.Results.Length > 0)
{
break;
}
futureBatchStat.Retries++;
context.WaitForWork(TimeSpan.FromMinutes(10), ref localWork, "PreFetching");
}
futureBatchStat.Duration = sp.Elapsed;
futureBatchStat.Size = jsonDocuments == null ? 0 : jsonDocuments.Results.Length;
if (jsonDocuments != null)
{
MaybeAddFutureBatch(jsonDocuments, indexingAge);
}
return(jsonDocuments);
})
});
}
public static void AddConnection(TrafficWatchConnection connection)
{
ServerHttpTrace.Add(connection);
}
private static async Task<ImmutableArray<SymbolAndProjectId<INamedTypeSymbol>>> FindImmediatelyInheritingTypesInDocumentAsync(
Document document,
SymbolAndProjectIdSet typesToSearchFor,
InheritanceQuery inheritanceQuery,
ConcurrentSet<SemanticModel> cachedModels,
ConcurrentSet<IDeclarationInfo> cachedInfos,
Func<SymbolAndProjectIdSet, INamedTypeSymbol, bool> typeImmediatelyMatches,
CancellationToken cancellationToken)
{
var declarationInfo = await document.GetDeclarationInfoAsync(cancellationToken).ConfigureAwait(false);
cachedInfos.Add(declarationInfo);
var result = CreateSymbolAndProjectIdSet();
foreach (var symbolInfo in declarationInfo.DeclaredSymbolInfos)
{
await ProcessSymbolInfo(
document, symbolInfo,
typesToSearchFor,
inheritanceQuery, cachedModels,
typeImmediatelyMatches, result, cancellationToken).ConfigureAwait(false);
}
return ToImmutableAndFree(result);
}
internal int RunCore(TextWriter consoleOutput, ErrorLogger errorLogger, CancellationToken cancellationToken)
{
Debug.Assert(!Arguments.IsScriptRunner);
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.DisplayVersion)
{
PrintVersion(consoleOutput);
return(Succeeded);
}
if (Arguments.DisplayLogo)
{
PrintLogo(consoleOutput);
}
if (Arguments.DisplayHelp)
{
PrintHelp(consoleOutput);
return(Succeeded);
}
if (ReportErrors(Arguments.Errors, consoleOutput, errorLogger))
{
return(Failed);
}
var touchedFilesLogger = (Arguments.TouchedFilesPath != null) ? new TouchedFileLogger() : null;
Compilation compilation = CreateCompilation(consoleOutput, touchedFilesLogger, errorLogger);
if (compilation == null)
{
return(Failed);
}
var diagnosticInfos = new List <DiagnosticInfo>();
ImmutableArray <DiagnosticAnalyzer> analyzers = ResolveAnalyzersFromArguments(diagnosticInfos, MessageProvider);
var additionalTextFiles = ResolveAdditionalFilesFromArguments(diagnosticInfos, MessageProvider, touchedFilesLogger);
if (ReportErrors(diagnosticInfos, consoleOutput, errorLogger))
{
return(Failed);
}
var diagnostics = new List <Diagnostic>();
ImmutableArray <EmbeddedText> embeddedTexts = AcquireEmbeddedTexts(compilation, diagnostics);
if (ReportErrors(diagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
bool reportAnalyzer = false;
CancellationTokenSource analyzerCts = null;
AnalyzerManager analyzerManager = null;
AnalyzerDriver analyzerDriver = null;
try
{
// Print the diagnostics produced during the parsing stage and exit if there were any errors.
if (ReportErrors(compilation.GetParseDiagnostics(), consoleOutput, errorLogger))
{
return(Failed);
}
ConcurrentSet <Diagnostic> analyzerExceptionDiagnostics = null;
if (!analyzers.IsEmpty)
{
analyzerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
analyzerManager = new AnalyzerManager();
analyzerExceptionDiagnostics = new ConcurrentSet <Diagnostic>();
Action <Diagnostic> addExceptionDiagnostic = diagnostic => analyzerExceptionDiagnostics.Add(diagnostic);
var analyzerOptions = new AnalyzerOptions(ImmutableArray <AdditionalText> .CastUp(additionalTextFiles));
analyzerDriver = AnalyzerDriver.CreateAndAttachToCompilation(compilation, analyzers, analyzerOptions, analyzerManager, addExceptionDiagnostic, Arguments.ReportAnalyzer, out compilation, analyzerCts.Token);
reportAnalyzer = Arguments.ReportAnalyzer && !analyzers.IsEmpty;
}
if (ReportErrors(compilation.GetDeclarationDiagnostics(), consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
string outputName = GetOutputFileName(compilation, cancellationToken);
var finalPeFilePath = Path.Combine(Arguments.OutputDirectory, outputName);
var finalPdbFilePath = Arguments.PdbPath ?? Path.ChangeExtension(finalPeFilePath, ".pdb");
var finalXmlFilePath = Arguments.DocumentationPath;
var diagnosticBag = DiagnosticBag.GetInstance();
Stream sourceLinkStreamOpt = null;
try
{
// NOTE: Unlike the PDB path, the XML doc path is not embedded in the assembly, so we don't need to pass it to emit.
var emitOptions = Arguments.EmitOptions.
WithOutputNameOverride(outputName).
WithPdbFilePath(finalPdbFilePath);
// The PDB path is emitted in it's entirety into the PE. This makes it impossible to have deterministic
// builds that occur in different source directories. To enable this we shave all path information from
// the PDB when specified by the user.
//
// This is a temporary work around to allow us to make progress with determinism. The following issue
// tracks getting an official solution here.
//
// https://github.com/dotnet/roslyn/issues/9813
if (Arguments.ParseOptions.Features.ContainsKey("pdb-path-determinism") && !string.IsNullOrEmpty(emitOptions.PdbFilePath))
{
emitOptions = emitOptions.WithPdbFilePath(Path.GetFileName(emitOptions.PdbFilePath));
}
if (Arguments.SourceLink != null)
{
sourceLinkStreamOpt = OpenFile(Arguments.SourceLink, consoleOutput, FileMode.Open, FileAccess.Read, FileShare.Read);
}
var moduleBeingBuilt = compilation.CheckOptionsAndCreateModuleBuilder(
diagnosticBag,
Arguments.ManifestResources,
emitOptions,
debugEntryPoint: null,
sourceLinkStream: sourceLinkStreamOpt,
embeddedTexts: embeddedTexts,
testData: null,
cancellationToken: cancellationToken);
if (moduleBeingBuilt != null)
{
bool success;
try
{
success = compilation.CompileMethods(
moduleBeingBuilt,
Arguments.EmitPdb,
diagnosticBag,
filterOpt: null,
cancellationToken: cancellationToken);
if (success)
{
// NOTE: as native compiler does, we generate the documentation file
// NOTE: 'in place', replacing the contents of the file if it exists
Stream xmlStreamOpt = null;
if (finalXmlFilePath != null)
{
xmlStreamOpt = OpenFile(finalXmlFilePath,
consoleOutput,
FileMode.OpenOrCreate,
FileAccess.Write,
FileShare.ReadWrite | FileShare.Delete);
if (xmlStreamOpt == null)
{
return(Failed);
}
xmlStreamOpt.SetLength(0);
}
using (xmlStreamOpt)
{
IEnumerable <DiagnosticInfo> errors;
using (var win32ResourceStreamOpt = GetWin32Resources(Arguments, compilation, out errors))
{
if (ReportErrors(errors, consoleOutput, errorLogger))
{
return(Failed);
}
success = compilation.GenerateResourcesAndDocumentationComments(
moduleBeingBuilt,
xmlStreamOpt,
win32ResourceStreamOpt,
diagnosticBag,
cancellationToken);
}
// only report unused usings if we have success.
if (success)
{
compilation.ReportUnusedImports(null, diagnosticBag, cancellationToken);
}
}
}
compilation.CompleteTrees(null);
if (analyzerDriver != null)
{
// GetDiagnosticsAsync is called after ReportUnusedImports
// since that method calls EventQueue.TryComplete. Without
// TryComplete, we may miss diagnostics.
var hostDiagnostics = analyzerDriver.GetDiagnosticsAsync(compilation).Result;
diagnosticBag.AddRange(hostDiagnostics);
if (hostDiagnostics.Any(IsReportedError))
{
success = false;
}
}
}
finally
{
moduleBeingBuilt.CompilationFinished();
}
if (success)
{
bool emitPdbFile = Arguments.EmitPdb && emitOptions.DebugInformationFormat != Emit.DebugInformationFormat.Embedded;
using (var peStreamProvider = new CompilerEmitStreamProvider(this, finalPeFilePath))
using (var pdbStreamProviderOpt = emitPdbFile ? new CompilerEmitStreamProvider(this, finalPdbFilePath) : null)
{
success = compilation.SerializeToPeStream(
moduleBeingBuilt,
peStreamProvider,
pdbStreamProviderOpt,
testSymWriterFactory: null,
diagnostics: diagnosticBag,
metadataOnly: emitOptions.EmitMetadataOnly,
cancellationToken: cancellationToken);
if (success && touchedFilesLogger != null)
{
if (pdbStreamProviderOpt != null)
{
touchedFilesLogger.AddWritten(finalPdbFilePath);
}
touchedFilesLogger.AddWritten(finalPeFilePath);
}
}
}
}
var compileAndEmitDiagnostics = diagnosticBag.ToReadOnly();
if (ReportErrors(compileAndEmitDiagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
}
finally
{
diagnosticBag.Free();
sourceLinkStreamOpt?.Dispose();
}
cancellationToken.ThrowIfCancellationRequested();
if (analyzerExceptionDiagnostics != null && ReportErrors(analyzerExceptionDiagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
bool errorsReadingAdditionalFiles = false;
foreach (var additionalFile in additionalTextFiles)
{
if (ReportErrors(additionalFile.Diagnostics, consoleOutput, errorLogger))
{
errorsReadingAdditionalFiles = true;
}
}
if (errorsReadingAdditionalFiles)
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.TouchedFilesPath != null)
{
Debug.Assert(touchedFilesLogger != null);
if (finalXmlFilePath != null)
{
touchedFilesLogger.AddWritten(finalXmlFilePath);
}
var readStream = OpenFile(Arguments.TouchedFilesPath + ".read", consoleOutput, mode: FileMode.OpenOrCreate);
if (readStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(readStream))
{
touchedFilesLogger.WriteReadPaths(writer);
}
var writtenStream = OpenFile(Arguments.TouchedFilesPath + ".write", consoleOutput, mode: FileMode.OpenOrCreate);
if (writtenStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(writtenStream))
{
touchedFilesLogger.WriteWrittenPaths(writer);
}
}
}
finally
{
// At this point analyzers are already complete in which case this is a no-op. Or they are
// still running because the compilation failed before all of the compilation events were
// raised. In the latter case the driver, and all its associated state, will be waiting around
// for events that are never coming. Cancel now and let the clean up process begin.
if (analyzerCts != null)
{
analyzerCts.Cancel();
if (analyzerManager != null)
{
// Clear cached analyzer descriptors and unregister exception handlers hooked up to the LocalizableString fields of the associated descriptors.
analyzerManager.ClearAnalyzerState(analyzers);
}
if (reportAnalyzer)
{
ReportAnalyzerExecutionTime(consoleOutput, analyzerDriver, Culture, compilation.Options.ConcurrentBuild);
}
}
}
return(Succeeded);
}
private static async Task<IEnumerable<INamedTypeSymbol>> FindImmediatelyInheritingTypesInDocumentAsync(
Document document,
HashSet<INamedTypeSymbol> typesToSearchFor,
InheritanceQuery inheritanceQuery,
ConcurrentSet<SemanticModel> cachedModels,
ConcurrentSet<IDeclarationInfo> cachedInfos,
Func<HashSet<INamedTypeSymbol>, INamedTypeSymbol, bool> typeImmediatelyMatches,
CancellationToken cancellationToken)
{
var declarationInfo = await document.GetDeclarationInfoAsync(cancellationToken).ConfigureAwait(false);
cachedInfos.Add(declarationInfo);
HashSet<INamedTypeSymbol> result = null;
foreach (var symbolInfo in declarationInfo.DeclaredSymbolInfos)
{
result = await ProcessSymbolInfo(
document, symbolInfo,
typesToSearchFor,
inheritanceQuery, cachedModels,
typeImmediatelyMatches, result, cancellationToken).ConfigureAwait(false);
}
return result;
}
public void Receive(Packet packet)
{
if (Seen(packet))
{
return;
}
seenTokens.Add(packet.Token);
string result = packet.Data.Aggregate("", (current, data) => current + (" " + data));
TraceHelper.WriteLine("{2}: {0}: {1}", packet.Type, result, packet.Token);
switch (packet.Type)
{
case NetMessageType.InLobby:
bool newLobby = (bool)packet.Data[0];
if (InLobby == newLobby)
{
break;
}
InLobby = newLobby;
if (OnLobbyUpdate != null)
{
OnLobbyUpdate(this, new LobbyEventArgs(InLobby));
}
break;
case NetMessageType.Chat:
if (OnChatMessage != null)
{
OnChatMessage(this, new NetChatEventArgs((string)packet.Data[0]));
}
break;
case NetMessageType.Name:
string newName = (string)packet.Data[0];
if (Name == newName)
{
break;
}
Name = newName;
if (OnNameUpdate != null)
{
OnNameUpdate(this, new NetNameEventArgs(Name));
}
break;
case NetMessageType.GameRequest:
IncomingRequest = new NetGameRequestEventArgs(
(int)packet.Data[0],
(bool)packet.Data[1]
);
if (OnGameRequest != null)
{
OnGameRequest(this, IncomingRequest);
}
break;
case NetMessageType.GameResponse:
if (OnGameResponse != null)
{
bool challengeAccepted = (bool)packet.Data[0];
NetGameResponseEventArgs response;
if (OutgoingRequest == null)
{
response = new NetGameResponseEventArgs(0, true, challengeAccepted);
}
else
{
response = new NetGameResponseEventArgs(
OutgoingRequest.BoardNum,
OutgoingRequest.ChallengerGoesFirst,
challengeAccepted
);
}
OnGameResponse(this, response);
}
break;
case NetMessageType.PlayerMove:
if (OnPlayerMove != null)
{
OnPlayerMove(this, new NetPlayerMoveEventArgs(
(Move)packet.Data[0]
));
}
break;
case NetMessageType.Leave:
if (OnPlayerLeave != null)
{
OnPlayerLeave(this, new EventArgs());
}
break;
}
}
private PrefetchingBehavior GetPrefetcherFor(Etag fromEtag, ConcurrentSet<PrefetchingBehavior> usedPrefetchers)
{
foreach (var prefetchingBehavior in prefetchingBehaviors)
{
if (prefetchingBehavior.CanUsePrefetcherToLoadFrom(fromEtag) && usedPrefetchers.TryAdd(prefetchingBehavior))
return prefetchingBehavior;
}
var newPrefetcher = prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Indexer, autoTuner,string.Format("Etags from: {0}", fromEtag));
var recentEtag = Etag.Empty;
context.Database.TransactionalStorage.Batch(accessor =>
{
recentEtag = accessor.Staleness.GetMostRecentDocumentEtag();
});
if (recentEtag.Restarts != fromEtag.Restarts || Math.Abs(recentEtag.Changes - fromEtag.Changes) > context.CurrentNumberOfItemsToIndexInSingleBatch)
{
// If the distance between etag of a recent document in db and etag to index from is greater than NumberOfItemsToProcessInSingleBatch
// then prevent the prefetcher from loading newly added documents. For such prefetcher we will relay only on future batches to prefetch docs to avoid
// large memory consumption by in-memory prefetching queue that would hold all the new documents, but it would be a long time before we can reach them.
newPrefetcher.DisableCollectingDocumentsAfterCommit = true;
}
prefetchingBehaviors.Add(newPrefetcher);
usedPrefetchers.Add(newPrefetcher);
return newPrefetcher;
}
private void DisableProvider(object provider)
{
disabledProviders.Add(provider);
}
private PrefetchingBehavior GetPrefetcherFor(Etag fromEtag, ConcurrentSet<PrefetchingBehavior> usedPrefetchers)
{
foreach (var prefetchingBehavior in prefetchingBehaviors)
{
// at this point we've already verified that we can't use the default prefetcher
// if it's empty, we don't need to use it
if (prefetchingBehavior.IsDefault == false && prefetchingBehavior.IsEmpty() && usedPrefetchers.TryAdd(prefetchingBehavior))
return prefetchingBehavior;
}
var newPrefetcher = prefetcher.CreatePrefetchingBehavior(PrefetchingUser.Indexer, autoTuner, string.Format("Etags from: {0}", fromEtag));
prefetchingBehaviors.Add(newPrefetcher);
usedPrefetchers.Add(newPrefetcher);
return newPrefetcher;
}
protected virtual DocumentStore GetDocumentStore(Options options = null, [CallerMemberName] string caller = null)
{
try
{
lock (_getDocumentStoreSync)
{
options = options ?? Options.Default;
var serverToUse = options.Server ?? Server;
var name = GetDatabaseName(caller);
if (options.ModifyDatabaseName != null)
{
name = options.ModifyDatabaseName(name) ?? name;
}
var hardDelete = true;
var runInMemory = true;
var pathToUse = options.Path;
if (pathToUse == null)
{
pathToUse = NewDataPath(name);
}
else
{
hardDelete = false;
runInMemory = false;
}
var doc = new DatabaseRecord(name)
{
Settings =
{
[RavenConfiguration.GetKey(x => x.Replication.ReplicationMinimalHeartbeat)] = "1",
[RavenConfiguration.GetKey(x => x.Replication.RetryReplicateAfter)] = "1",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = runInMemory.ToString(),
[RavenConfiguration.GetKey(x => x.Core.DataDirectory)] = pathToUse,
[RavenConfiguration.GetKey(x => x.Core.ThrowIfAnyIndexCannotBeOpened)] = "true",
[RavenConfiguration.GetKey(x => x.Indexing.MinNumberOfMapAttemptsAfterWhichBatchWillBeCanceledIfRunningLowOnMemory)] = int.MaxValue.ToString(),
}
};
options.ModifyDatabaseRecord?.Invoke(doc);
var store = new DocumentStore
{
Urls = UseFiddler(serverToUse.WebUrl),
Database = name,
Certificate = options.ClientCertificate
};
options.ModifyDocumentStore?.Invoke(store);
//This gives too much error details in most cases, we don't need this now
store.RequestExecutorCreated += (sender, executor) =>
{
executor.AdditionalErrorInformation += sb => sb.AppendLine().Append(GetLastStatesFromAllServersOrderedByTime());
};
store.Initialize();
if (options.CreateDatabase)
{
foreach (var server in Servers)
{
using (server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
context.OpenReadTransaction();
if (server.ServerStore.Cluster.Read(context, Constants.Documents.Prefix + name) != null)
{
throw new InvalidOperationException($"Database '{name}' already exists");
}
}
}
DatabasePutResult result;
if (options.AdminCertificate != null)
{
using (var adminStore = new DocumentStore
{
Urls = UseFiddler(serverToUse.WebUrl),
Database = name,
Certificate = options.AdminCertificate
}.Initialize())
{
result = adminStore.Maintenance.Server.Send(new CreateDatabaseOperation(doc, options.ReplicationFactor));
}
}
else
{
result = store.Maintenance.Server.Send(new CreateDatabaseOperation(doc, options.ReplicationFactor));
}
Assert.True(result.RaftCommandIndex > 0); //sanity check
var timeout = TimeSpan.FromMinutes(Debugger.IsAttached ? 5 : 1);
AsyncHelpers.RunSync(async() =>
{
await WaitForRaftIndexToBeAppliedInCluster(result.RaftCommandIndex, timeout);
});
}
store.BeforeDispose += (sender, args) =>
{
if (CreatedStores.TryRemove(store) == false)
{
return; // can happen if we are wrapping the store inside sharded one
}
foreach (var server in Servers)
{
if (server.Disposed)
{
continue;
}
var serverUrl = UseFiddler(server.WebUrl);
if (store.Urls.Any(url => serverUrl.Contains(url)) == false)
{
continue;
}
try
{
var databaseTask = server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(name, options.IgnoreDisabledDatabase);
if (databaseTask != null && databaseTask.IsCompleted == false)
{
// if we are disposing store before database had chance to load then we need to wait
databaseTask.Wait();
}
}
catch (DatabaseDisabledException)
{
// ignoring
}
catch (DatabaseNotRelevantException)
{
continue;
}
if (options.DeleteDatabaseOnDispose)
{
DeleteDatabaseResult result;
try
{
if (options.AdminCertificate != null)
{
using (var adminStore = new DocumentStore
{
Urls = UseFiddler(serverToUse.WebUrl),
Database = name,
Certificate = options.AdminCertificate
}.Initialize())
{
result = adminStore.Maintenance.Server.Send(new DeleteDatabasesOperation(name, hardDelete));
}
}
else
{
result = store.Maintenance.Server.Send(new DeleteDatabasesOperation(name, hardDelete));
}
}
catch (DatabaseDoesNotExistException)
{
continue;
}
catch (NoLeaderException)
{
continue;
}
}
}
};
CreatedStores.Add(store);
return(store);
}
}
catch (TimeoutException te)
{
throw new TimeoutException($"{te.Message} {Environment.NewLine} {te.StackTrace}{Environment.NewLine}Servers states:{Environment.NewLine}{GetLastStatesFromAllServersOrderedByTime()}");
}
}
private PrefetchingBehavior GetPrefetcherFor(Etag fromEtag, ConcurrentSet<PrefetchingBehavior> usedPrefetchers)
{
foreach (var prefetchingBehavior in prefetchingBehaviors)
{
if (prefetchingBehavior.CanUsePrefetcherToLoadFrom(fromEtag) && usedPrefetchers.TryAdd(prefetchingBehavior))
return prefetchingBehavior;
}
var newPrefetcher = Database.Prefetcher.CreatePrefetchingBehavior(PrefetchingUser.SqlReplicator, null, "SqlReplication");
prefetchingBehaviors.Add(newPrefetcher);
usedPrefetchers.Add(newPrefetcher);
return newPrefetcher;
}
private int RunCore(TextWriter consoleOutput, ErrorLogger errorLogger, CancellationToken cancellationToken)
{
Debug.Assert(!Arguments.IsInteractive);
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.DisplayLogo)
{
PrintLogo(consoleOutput);
}
if (Arguments.DisplayHelp)
{
PrintHelp(consoleOutput);
return(Succeeded);
}
if (ReportErrors(Arguments.Errors, consoleOutput, errorLogger))
{
return(Failed);
}
var touchedFilesLogger = (Arguments.TouchedFilesPath != null) ? new TouchedFileLogger() : null;
Compilation compilation = CreateCompilation(consoleOutput, touchedFilesLogger, errorLogger);
if (compilation == null)
{
return(Failed);
}
var diagnostics = new List <DiagnosticInfo>();
var analyzers = ResolveAnalyzersFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
var additionalTextFiles = ResolveAdditionalFilesFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
if (ReportErrors(diagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
CancellationTokenSource analyzerCts = null;
AnalyzerManager analyzerManager = null;
try
{
Func <ImmutableArray <Diagnostic> > getAnalyzerDiagnostics = null;
if (!analyzers.IsDefaultOrEmpty)
{
analyzerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
analyzerManager = new AnalyzerManager();
var analyzerExceptionDiagnostics = new ConcurrentSet <Diagnostic>();
Action <Diagnostic> addExceptionDiagnostic = diagnostic => analyzerExceptionDiagnostics.Add(diagnostic);
var analyzerOptions = new AnalyzerOptions(ImmutableArray.Create <AdditionalText, AdditionalTextFile>(additionalTextFiles));
var analyzerDriver = AnalyzerDriver.Create(compilation, analyzers, analyzerOptions, analyzerManager, addExceptionDiagnostic, out compilation, analyzerCts.Token);
getAnalyzerDiagnostics = () =>
{
var analyzerDiagnostics = analyzerDriver.GetDiagnosticsAsync().Result;
return(analyzerDiagnostics.AddRange(analyzerExceptionDiagnostics));
};
}
// Print the diagnostics produced during the parsing stage and exit if there were any errors.
if (ReportErrors(compilation.GetParseDiagnostics(), consoleOutput, errorLogger))
{
return(Failed);
}
if (ReportErrors(compilation.GetDeclarationDiagnostics(), consoleOutput, errorLogger))
{
return(Failed);
}
EmitResult emitResult;
// NOTE: as native compiler does, we generate the documentation file
// NOTE: 'in place', replacing the contents of the file if it exists
string finalOutputPath;
string finalPdbFilePath;
string finalXmlFilePath;
FileStream xmlStreamOpt = null;
cancellationToken.ThrowIfCancellationRequested();
finalXmlFilePath = Arguments.DocumentationPath;
if (finalXmlFilePath != null)
{
xmlStreamOpt = OpenFile(finalXmlFilePath, consoleOutput, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete);
if (xmlStreamOpt == null)
{
return(Failed);
}
xmlStreamOpt.SetLength(0);
}
cancellationToken.ThrowIfCancellationRequested();
IEnumerable <DiagnosticInfo> errors;
using (var win32ResourceStreamOpt = GetWin32Resources(Arguments, compilation, out errors))
using (xmlStreamOpt)
{
if (ReportErrors(errors, consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
string outputName = GetOutputFileName(compilation, cancellationToken);
finalOutputPath = Path.Combine(Arguments.OutputDirectory, outputName);
finalPdbFilePath = Arguments.PdbPath ?? Path.ChangeExtension(finalOutputPath, ".pdb");
// NOTE: Unlike the PDB path, the XML doc path is not embedded in the assembly, so we don't need to pass it to emit.
var emitOptions = Arguments.EmitOptions.
WithOutputNameOverride(outputName).
WithPdbFilePath(finalPdbFilePath);
using (var peStreamProvider = new CompilerEmitStreamProvider(this, finalOutputPath))
using (var pdbStreamProviderOpt = Arguments.EmitPdb ? new CompilerEmitStreamProvider(this, finalPdbFilePath) : null)
{
emitResult = compilation.Emit(
peStreamProvider,
pdbStreamProviderOpt,
(xmlStreamOpt != null) ? new Compilation.SimpleEmitStreamProvider(xmlStreamOpt) : null,
(win32ResourceStreamOpt != null) ? new Compilation.SimpleEmitStreamProvider(win32ResourceStreamOpt) : null,
Arguments.ManifestResources,
emitOptions,
getAnalyzerDiagnostics,
cancellationToken);
if (emitResult.Success && touchedFilesLogger != null)
{
if (pdbStreamProviderOpt != null)
{
touchedFilesLogger.AddWritten(finalPdbFilePath);
}
touchedFilesLogger.AddWritten(finalOutputPath);
}
}
}
GenerateSqmData(Arguments.CompilationOptions, emitResult.Diagnostics);
if (ReportErrors(emitResult.Diagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
bool errorsReadingAdditionalFiles = false;
foreach (var additionalFile in additionalTextFiles)
{
if (ReportErrors(additionalFile.Diagnostics, consoleOutput, errorLogger))
{
errorsReadingAdditionalFiles = true;
}
}
if (errorsReadingAdditionalFiles)
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.TouchedFilesPath != null)
{
Debug.Assert(touchedFilesLogger != null);
if (finalXmlFilePath != null)
{
touchedFilesLogger.AddWritten(finalXmlFilePath);
}
var readStream = OpenFile(Arguments.TouchedFilesPath + ".read", consoleOutput, FileMode.OpenOrCreate);
if (readStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(readStream))
{
touchedFilesLogger.WriteReadPaths(writer);
}
var writtenStream = OpenFile(Arguments.TouchedFilesPath + ".write", consoleOutput, FileMode.OpenOrCreate);
if (writtenStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(writtenStream))
{
touchedFilesLogger.WriteWrittenPaths(writer);
}
}
}
finally
{
// At this point analyzers are already complete in which case this is a no-op. Or they are
// still running because the compilation failed before all of the compilation events were
// raised. In the latter case the driver, and all its associated state, will be waiting around
// for events that are never coming. Cancel now and let the clean up process begin.
if (analyzerCts != null)
{
analyzerCts.Cancel();
// Clear cached analyzer descriptors and unregister exception handlers hooked up to the LocalizableString fields of the associated descriptors.
analyzerManager.ClearAnalyzerState(analyzers);
}
}
return(Succeeded);
}
private static async Task GetDependentTypesInProjectAsync(
INamedTypeSymbol type, Project project, Solution solution, Func<INamedTypeSymbol, INamedTypeSymbol, bool> predicate, ConditionalWeakTable<Compilation, ConcurrentDictionary<SymbolKey, List<SymbolKey>>> cache, bool locationsInMetadata, ConcurrentSet<ISymbol> results, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
var typeId = type.GetSymbolKey();
List<SymbolKey> dependentTypeIds;
if (!TryGetDependentTypes(cache, compilation, typeId, out dependentTypeIds))
{
List<INamedTypeSymbol> allTypes;
if (locationsInMetadata)
{
// From metadata, have to check other (non private) metadata types, as well as
// source types.
allTypes = GetAllSourceAndAccessibleTypesInCompilation(compilation, cancellationToken);
}
else
{
// It's from source, so only other source types could derive from it.
allTypes = GetAllSourceTypesInCompilation(compilation, cancellationToken);
}
dependentTypeIds = new List<SymbolKey>();
foreach (var t in allTypes)
{
cancellationToken.ThrowIfCancellationRequested();
if (predicate(t, type))
{
dependentTypeIds.Add(t.GetSymbolKey());
}
}
dependentTypeIds = GetOrAddDependentTypes(cache, compilation, typeId, dependentTypeIds);
}
foreach (var id in dependentTypeIds)
{
cancellationToken.ThrowIfCancellationRequested();
var resolvedSymbols = id.Resolve(compilation, cancellationToken: cancellationToken).GetAllSymbols();
foreach (var resolvedSymbol in resolvedSymbols)
{
var mappedSymbol = await SymbolFinder.FindSourceDefinitionAsync(resolvedSymbol, solution, cancellationToken).ConfigureAwait(false) ?? resolvedSymbol;
results.Add(mappedSymbol);
}
}
}
public IDisposable Subscribe(IObserver <T> observer)
{
observers.Add(observer);
return(Disposable.Create(() => observers.TryRemove(observer)));
}
private async Task DetermineAllSymbolsCoreAsync(ISymbol symbol, ConcurrentSet<ISymbol> result)
{
this.cancellationToken.ThrowIfCancellationRequested();
var searchSymbol = MapToAppropriateSymbol(symbol);
// 2) Try to map this back to source symbol if this was a metadata symbol.
searchSymbol = await SymbolFinder.FindSourceDefinitionAsync(searchSymbol, solution, cancellationToken).ConfigureAwait(false) ?? searchSymbol;
if (searchSymbol != null && result.Add(searchSymbol))
{
this.progress.OnDefinitionFound(searchSymbol);
this.foundReferences.GetOrAdd(searchSymbol, createSymbolLocations);
// get project to search
var projects = GetProjectScope();
this.cancellationToken.ThrowIfCancellationRequested();
List<Task> finderTasks = new List<Task>();
foreach (var f in finders)
{
finderTasks.Add(Task.Run(async () =>
{
var symbols = await f.DetermineCascadedSymbolsAsync(searchSymbol, solution, projects, cancellationToken).ConfigureAwait(false) ?? SpecializedCollections.EmptyEnumerable<ISymbol>();
this.cancellationToken.ThrowIfCancellationRequested();
List<Task> symbolTasks = new List<Task>();
foreach (var child in symbols)
{
symbolTasks.Add(Task.Run(() => DetermineAllSymbolsCoreAsync(child, result), this.cancellationToken));
}
await Task.WhenAll(symbolTasks).ConfigureAwait(false);
}, this.cancellationToken));
}
await Task.WhenAll(finderTasks).ConfigureAwait(false);
}
}
private int RunCore(TextWriter consoleOutput, ErrorLogger errorLogger, CancellationToken cancellationToken)
{
Debug.Assert(!Arguments.IsScriptRunner);
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.DisplayLogo)
{
PrintLogo(consoleOutput);
}
if (Arguments.DisplayHelp)
{
PrintHelp(consoleOutput);
return(Succeeded);
}
if (ReportErrors(Arguments.Errors, consoleOutput, errorLogger))
{
return(Failed);
}
var touchedFilesLogger = (Arguments.TouchedFilesPath != null) ? new TouchedFileLogger() : null;
Compilation compilation = CreateCompilation(consoleOutput, touchedFilesLogger, errorLogger);
if (compilation == null)
{
return(Failed);
}
var diagnostics = new List <DiagnosticInfo>();
var analyzers = ResolveAnalyzersFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
var additionalTextFiles = ResolveAdditionalFilesFromArguments(diagnostics, MessageProvider, touchedFilesLogger);
if (ReportErrors(diagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
CancellationTokenSource analyzerCts = null;
AnalyzerManager analyzerManager = null;
AnalyzerDriver analyzerDriver = null;
try
{
Func <ImmutableArray <Diagnostic> > getAnalyzerDiagnostics = null;
ConcurrentSet <Diagnostic> analyzerExceptionDiagnostics = null;
if (!analyzers.IsDefaultOrEmpty)
{
analyzerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
analyzerManager = new AnalyzerManager();
analyzerExceptionDiagnostics = new ConcurrentSet <Diagnostic>();
Action <Diagnostic> addExceptionDiagnostic = diagnostic => analyzerExceptionDiagnostics.Add(diagnostic);
var analyzerOptions = new AnalyzerOptions(ImmutableArray <AdditionalText> .CastUp(additionalTextFiles));
analyzerDriver = AnalyzerDriver.CreateAndAttachToCompilation(compilation, analyzers, analyzerOptions, analyzerManager, addExceptionDiagnostic, Arguments.ReportAnalyzer, out compilation, analyzerCts.Token);
getAnalyzerDiagnostics = () => analyzerDriver.GetDiagnosticsAsync(compilation).Result;
}
// Print the diagnostics produced during the parsing stage and exit if there were any errors.
if (ReportErrors(compilation.GetParseDiagnostics(), consoleOutput, errorLogger))
{
return(Failed);
}
if (ReportErrors(compilation.GetDeclarationDiagnostics(), consoleOutput, errorLogger))
{
return(Failed);
}
EmitResult emitResult;
// NOTE: as native compiler does, we generate the documentation file
// NOTE: 'in place', replacing the contents of the file if it exists
string finalPeFilePath;
string finalPdbFilePath;
string finalXmlFilePath;
Stream xmlStreamOpt = null;
cancellationToken.ThrowIfCancellationRequested();
finalXmlFilePath = Arguments.DocumentationPath;
if (finalXmlFilePath != null)
{
xmlStreamOpt = OpenFile(finalXmlFilePath, consoleOutput, PortableShim.FileMode.OpenOrCreate, PortableShim.FileAccess.Write, PortableShim.FileShare.ReadWriteBitwiseOrDelete);
if (xmlStreamOpt == null)
{
return(Failed);
}
xmlStreamOpt.SetLength(0);
}
cancellationToken.ThrowIfCancellationRequested();
IEnumerable <DiagnosticInfo> errors;
using (var win32ResourceStreamOpt = GetWin32Resources(Arguments, compilation, out errors))
using (xmlStreamOpt)
{
if (ReportErrors(errors, consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
string outputName = GetOutputFileName(compilation, cancellationToken);
finalPeFilePath = Path.Combine(Arguments.OutputDirectory, outputName);
finalPdbFilePath = Arguments.PdbPath ?? Path.ChangeExtension(finalPeFilePath, ".pdb");
// NOTE: Unlike the PDB path, the XML doc path is not embedded in the assembly, so we don't need to pass it to emit.
var emitOptions = Arguments.EmitOptions.
WithOutputNameOverride(outputName).
WithPdbFilePath(finalPdbFilePath);
// The PDB path is emitted in it's entirety into the PE. This makes it impossible to have deterministic
// builds that occur in different source directories. To enable this we shave all path information from
// the PDB when specified by the user.
//
// This is a temporary work around to allow us to make progress with determinism. The following issue
// tracks getting an official solution here.
//
// https://github.com/dotnet/roslyn/issues/9813
if (Arguments.ParseOptions.Features.ContainsKey("pdb-path-determinism") && !string.IsNullOrEmpty(emitOptions.PdbFilePath))
{
emitOptions = emitOptions.WithPdbFilePath(Path.GetFileName(emitOptions.PdbFilePath));
}
using (var peStreamProvider = new CompilerEmitStreamProvider(this, finalPeFilePath))
using (var pdbStreamProviderOpt = Arguments.EmitPdb ? new CompilerEmitStreamProvider(this, finalPdbFilePath) : null)
{
emitResult = compilation.Emit(
peStreamProvider,
pdbStreamProviderOpt,
(xmlStreamOpt != null) ? new Compilation.SimpleEmitStreamProvider(xmlStreamOpt) : null,
(win32ResourceStreamOpt != null) ? new Compilation.SimpleEmitStreamProvider(win32ResourceStreamOpt) : null,
Arguments.ManifestResources,
emitOptions,
debugEntryPoint: null,
getHostDiagnostics: getAnalyzerDiagnostics,
cancellationToken: cancellationToken);
if (emitResult.Success && touchedFilesLogger != null)
{
if (pdbStreamProviderOpt != null)
{
touchedFilesLogger.AddWritten(finalPdbFilePath);
}
touchedFilesLogger.AddWritten(finalPeFilePath);
}
}
}
GenerateSqmData(Arguments.CompilationOptions, emitResult.Diagnostics);
if (ReportErrors(emitResult.Diagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
if (analyzerExceptionDiagnostics != null && ReportErrors(analyzerExceptionDiagnostics, consoleOutput, errorLogger))
{
return(Failed);
}
bool errorsReadingAdditionalFiles = false;
foreach (var additionalFile in additionalTextFiles)
{
if (ReportErrors(additionalFile.Diagnostics, consoleOutput, errorLogger))
{
errorsReadingAdditionalFiles = true;
}
}
if (errorsReadingAdditionalFiles)
{
return(Failed);
}
cancellationToken.ThrowIfCancellationRequested();
if (Arguments.TouchedFilesPath != null)
{
Debug.Assert(touchedFilesLogger != null);
if (finalXmlFilePath != null)
{
touchedFilesLogger.AddWritten(finalXmlFilePath);
}
var readStream = OpenFile(Arguments.TouchedFilesPath + ".read", consoleOutput, mode: PortableShim.FileMode.OpenOrCreate);
if (readStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(readStream))
{
touchedFilesLogger.WriteReadPaths(writer);
}
var writtenStream = OpenFile(Arguments.TouchedFilesPath + ".write", consoleOutput, mode: PortableShim.FileMode.OpenOrCreate);
if (writtenStream == null)
{
return(Failed);
}
using (var writer = new StreamWriter(writtenStream))
{
touchedFilesLogger.WriteWrittenPaths(writer);
}
}
}
finally
{
// At this point analyzers are already complete in which case this is a no-op. Or they are
// still running because the compilation failed before all of the compilation events were
// raised. In the latter case the driver, and all its associated state, will be waiting around
// for events that are never coming. Cancel now and let the clean up process begin.
if (analyzerCts != null)
{
analyzerCts.Cancel();
if (analyzerManager != null)
{
// Clear cached analyzer descriptors and unregister exception handlers hooked up to the LocalizableString fields of the associated descriptors.
analyzerManager.ClearAnalyzerState(analyzers);
}
if (Arguments.ReportAnalyzer && analyzerDriver != null && compilation != null)
{
ReportAnalyzerExecutionTime(consoleOutput, analyzerDriver, Culture, compilation.Options.ConcurrentBuild);
}
}
}
return(Succeeded);
}