/// <summary>
/// Gets events from the DB based on the eventKey.
/// </summary>
/// <remarks>
/// Since 0 isn't serialized by Protobuf, a PrefixSearch for RewriteCount = 0 would match everything.
/// Thus to avoid that, we set it to -1 to "match everything", else we look for specific rewrite counts.
/// Similarly, a PrefixSearch for WorkerID = 0 would match everything.
/// Thus to avoid that, we set it to uint.MaxValue to "match everything", else we look for specific rewrite counts.
/// </remarks>
/// <returns>List of events, empty if no such events exist</returns>
private IEnumerable <IMessage> GetEventsByKey(EventKey eventKey)
{
Contract.Requires(m_accessor != null, "XldbDataStore is not initialized");
var storedEvents = new List <IMessage>();
if (!m_eventParserDictionary.TryGetValue(eventKey.EventTypeID, out var parser))
{
Contract.Assert(false, "No parser found for EventTypeId passed in");
}
var matchAllRewriteCounts = false;
var matchAllWorkerIDs = false;
if (eventKey.FileRewriteCount == s_fileRewriteCountDefaultValue)
{
matchAllRewriteCounts = true;
// Set it to 0 to match everything
eventKey.FileRewriteCount = 0;
}
if (eventKey.WorkerID == s_workerIDDefaultValue)
{
matchAllWorkerIDs = true;
// Set it to 0 to match everything
eventKey.WorkerID = 0;
}
var maybeFound = m_accessor.Use(database =>
{
foreach (var kvp in database.PrefixSearch(eventKey.ToByteArray(), EventColumnFamilyName))
{
var kvpKey = EventKey.Parser.ParseFrom(kvp.Key);
// MatchAllWorker IDs and MatchAllRewriteCounts are true, so just add everything
if (matchAllWorkerIDs && matchAllRewriteCounts)
{
storedEvents.Add(parser.ParseFrom(kvp.Value));
}
// Else if matching all WorkerIDs, check for specific RewriteCounts OR
// if matching all RewriteCounts, check for specific worker ID
else if ((matchAllWorkerIDs && kvpKey.FileRewriteCount == eventKey.FileRewriteCount) ||
(matchAllRewriteCounts && kvpKey.WorkerID == eventKey.WorkerID))
{
storedEvents.Add(parser.ParseFrom(kvp.Value));
}
// Else both worker ID and RewriteCounts are unique so the prefix search matches the right one
else
{
storedEvents.Add(parser.ParseFrom(kvp.Value));
}
}
});
if (!maybeFound.Succeeded)
{
maybeFound.Failure.Throw();
}
return(storedEvents);
}
/// <summary>
/// Ingest all of the pips to RocksDB
/// </summary>
private void IngestAllPips()
{
var totalNumberOfPips = CachedGraph.PipTable.StableKeys.Count;
var pipIds = CachedGraph.PipTable.StableKeys;
var concurrency = (int)(Environment.ProcessorCount * s_innerConcurrencyMultiplier);
var partitionSize = totalNumberOfPips / concurrency;
Console.WriteLine("Ingesting pips now ...");
Parallel.For(0, concurrency, i =>
{
// Hold only one lock while inserting all of these keys into the DB
var maybeInserted = m_accessor.Use(database =>
{
var start = i * partitionSize;
var end = (i + 1) == concurrency ? totalNumberOfPips : (i + 1) * partitionSize;
var pipsIngested = 0;
var pipSemistableMap = new Dictionary <byte[], byte[]>();
var pipIdMap = new Dictionary <byte[], byte[]>();
for (int j = start; j < end; j++)
{
var pipId = pipIds[j];
pipsIngested++;
if (pipsIngested % s_pipOutputBatchLogSize == 0)
{
Console.Write(".");
database.ApplyBatch(pipSemistableMap);
database.ApplyBatch(pipIdMap, XldbDataStore.PipColumnFamilyName);
pipSemistableMap.Clear();
pipIdMap.Clear();
}
var hydratedPip = CachedGraph.PipTable.HydratePip(pipId, BuildXL.Pips.PipQueryContext.PipGraphRetrieveAllPips);
var pipType = hydratedPip.PipType;
if (pipType == PipType.Value || pipType == PipType.HashSourceFile || pipType == PipType.SpecFile || pipType == PipType.Module)
{
continue;
}
var xldbPip = hydratedPip.ToPip(CachedGraph);
var pipIdKey = new PipIdKey()
{
PipId = hydratedPip.PipId.Value,
PipType = (Xldb.Proto.PipType)(pipType + 1)
};
var pipIdKeyArr = pipIdKey.ToByteArray();
IMessage xldbSpecificPip = xldbPip;
if (pipType == PipType.Ipc)
{
var ipcPip = (Pips.Operations.IpcPip)hydratedPip;
xldbSpecificPip = ipcPip.ToIpcPip(PathTable, xldbPip, m_nameExpander);
foreach (var fileArtifact in ipcPip.FileDependencies)
{
AddToFileConsumerMap(fileArtifact, pipId);
}
foreach (var directoryArtifact in ipcPip.DirectoryDependencies)
{
AddToDirectoryConsumerMap(directoryArtifact, pipId);
}
AddToDbStorageDictionary(DBStoredTypes.IpcPip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else if (pipType == PipType.SealDirectory)
{
var sealDirectoryPip = (Pips.Operations.SealDirectory)hydratedPip;
xldbSpecificPip = sealDirectoryPip.ToSealDirectory(PathTable, xldbPip, m_nameExpander);
// If it is a shared opaque, then flatten the list of composted directories
if (sealDirectoryPip.Kind == Pips.Operations.SealDirectoryKind.SharedOpaque)
{
var directoryQueue = new Queue <Utilities.DirectoryArtifact>();
foreach (var initialDirectory in sealDirectoryPip.ComposedDirectories)
{
directoryQueue.Enqueue(initialDirectory);
}
while (directoryQueue.Count > 0)
{
var nestedDirectory = directoryQueue.Dequeue();
var nestedPipId = CachedGraph.PipGraph.GetSealedDirectoryNode(nestedDirectory).ToPipId();
if (CachedGraph.PipTable.IsSealDirectoryComposite(nestedPipId))
{
var nestedPip = (Pips.Operations.SealDirectory)CachedGraph.PipGraph.GetSealedDirectoryPip(nestedDirectory, BuildXL.Pips.PipQueryContext.SchedulerExecuteSealDirectoryPip);
foreach (var pendingDirectory in nestedPip.ComposedDirectories)
{
directoryQueue.Enqueue(pendingDirectory);
}
}
else
{
Contract.Assert(nestedDirectory.IsOutputDirectory());
AddToDirectoryConsumerMap(nestedDirectory, pipId);
}
}
}
foreach (var fileArtifact in sealDirectoryPip.Contents)
{
AddToFileConsumerMap(fileArtifact, pipId);
}
AddToDbStorageDictionary(DBStoredTypes.SealDirectoryPip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else if (pipType == PipType.CopyFile)
{
var copyFilePip = (Pips.Operations.CopyFile)hydratedPip;
xldbSpecificPip = copyFilePip.ToCopyFile(PathTable, xldbPip, m_nameExpander);
AddToFileConsumerMap(copyFilePip.Source, pipId);
AddToDbStorageDictionary(DBStoredTypes.CopyFilePip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else if (pipType == PipType.WriteFile)
{
var writeFilePip = (Pips.Operations.WriteFile)hydratedPip;
xldbSpecificPip = writeFilePip.ToWriteFile(PathTable, xldbPip, m_nameExpander);
AddToDbStorageDictionary(DBStoredTypes.WriteFilePip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else if (pipType == PipType.Process)
{
var processPip = (Pips.Operations.Process)hydratedPip;
xldbSpecificPip = processPip.ToProcessPip(PathTable, xldbPip, m_nameExpander);
AddToFileConsumerMap(processPip.StandardInputFile, pipId);
AddToFileConsumerMap(processPip.Executable, pipId);
foreach (var fileArtifact in processPip.Dependencies)
{
AddToFileConsumerMap(fileArtifact, pipId);
}
foreach (var directoryArtifact in processPip.DirectoryDependencies)
{
AddToDirectoryConsumerMap(directoryArtifact, pipId);
}
AddToDbStorageDictionary(DBStoredTypes.ProcessPip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else
{
Contract.Assert(false, "Unknown pip type parsed");
}
// If the SemiStableHash != 0, then we want to create the SemistableHash -> PipId indirection.
// Else we do not want that since the key would no longer be unique
if (hydratedPip.SemiStableHash != 0)
{
var pipSemistableHashKey = new PipSemistableHashKey()
{
SemiStableHash = hydratedPip.SemiStableHash,
};
pipSemistableMap.Add(pipSemistableHashKey.ToByteArray(), pipIdKeyArr);
}
pipIdMap.Add(pipIdKeyArr, xldbSpecificPip.ToByteArray());
}
database.ApplyBatch(pipSemistableMap);
database.ApplyBatch(pipIdMap, XldbDataStore.PipColumnFamilyName);
});
if (!maybeInserted.Succeeded)
{
Console.WriteLine("Failed to insert pip data into RocksDb. Exiting analyzer now ...");
maybeInserted.Failure.Throw();
}
});
}