/// <summary>
/// Gets all scheduled pips (ie. all the non meta pips). Must return pair of PipType, IMessage so user knows how to
/// cast the IMessage object
/// </summary>
public IEnumerable <(PipType, IMessage)> GetAllScheduledPips()
{
Contract.Requires(Accessor != null, "XldbDataStore is not initialized");
var allPips = new List <(PipType, IMessage)>();
// Empty key to prefix match all pips stored in DB
var pipIdKey = new PipIdKey();
var maybeFound = Accessor.Use(database =>
{
foreach (var kvp in database.PrefixSearch(pipIdKey.ToByteArray(), PipColumnFamilyName))
{
var pipType = PipIdKey.Parser.ParseFrom(kvp.Key).PipType;
if (m_pipParserDictionary.TryGetValue(pipType, out var parser))
{
var xldbPip = parser.ParseFrom(kvp.Value);
allPips.Add((pipType, xldbPip));
}
else
{
Contract.Assert(false, "No parser found for PipId");
}
}
});
if (!maybeFound.Succeeded)
{
maybeFound.Failure.Throw();
}
return(allPips);
}
/// <summary>
/// Gets all pips of a certain type.
/// </summary>
/// <returns>Returns list of all pips of certain type, empty if no such pips exist.</returns>
public IEnumerable <IMessage> GetAllPipsByType(PipType pipType)
{
Contract.Requires(Accessor != null, "XldbDataStore is not initialized");
var storedPips = new List <IMessage>();
if (!m_pipParserDictionary.TryGetValue(pipType, out var parser))
{
Contract.Assert(false, "No parser found for PipId");
}
// Empty key will match all pips in the prefix search, and then we grab only the ones that match the type we want
var pipIdKey = new PipIdKey();
var maybeFound = Accessor.Use(database =>
{
foreach (var kvp in database.PrefixSearch(pipIdKey.ToByteArray(), PipColumnFamilyName))
{
var pipKey = PipIdKey.Parser.ParseFrom(kvp.Key);
if (pipKey.PipType == pipType)
{
storedPips.Add(parser.ParseFrom(kvp.Value));
}
}
});
if (!maybeFound.Succeeded)
{
maybeFound.Failure.Throw();
}
return(storedPips);
}
/// <summary>
/// Gets the pip stored based on the pip id
/// </summary>
/// <returns>Returns null if no such pip is found</returns>
public IMessage GetPipByPipId(uint pipId, out PipType pipType)
{
Contract.Requires(Accessor != null, "XldbDataStore is not initialized");
IMessage foundPip = null;
var pipIdKey = new PipIdKey()
{
PipId = pipId
};
PipType outPipType = 0;
var maybeFound = Accessor.Use(database =>
{
foreach (var kvp in database.PrefixSearch(pipIdKey.ToByteArray(), PipColumnFamilyName))
{
var pipKey = PipIdKey.Parser.ParseFrom(kvp.Key);
if (m_pipParserDictionary.TryGetValue(pipKey.PipType, out var parser))
{
foundPip = parser.ParseFrom(kvp.Value);
outPipType = pipKey.PipType;
}
else
{
Contract.Assert(false, "No parser found for PipId");
}
}
});
if (!maybeFound.Succeeded)
{
maybeFound.Failure.Throw();
}
pipType = outPipType;
return(foundPip);
}
/// <summary>
/// Ingest all of the pips to RocksDB
/// </summary>
private void IngestAllPips()
{
var totalNumberOfPips = CachedGraph.PipTable.StableKeys.Count;
var pipIds = CachedGraph.PipTable.StableKeys;
var concurrency = (int)(Environment.ProcessorCount * s_innerConcurrencyMultiplier);
var partitionSize = totalNumberOfPips / concurrency;
Console.WriteLine("Ingesting pips now ...");
Parallel.For(0, concurrency, i =>
{
// Hold only one lock while inserting all of these keys into the DB
var maybeInserted = m_accessor.Use(database =>
{
var start = i * partitionSize;
var end = (i + 1) == concurrency ? totalNumberOfPips : (i + 1) * partitionSize;
var pipsIngested = 0;
var pipSemistableMap = new Dictionary <byte[], byte[]>();
var pipIdMap = new Dictionary <byte[], byte[]>();
for (int j = start; j < end; j++)
{
var pipId = pipIds[j];
pipsIngested++;
if (pipsIngested % s_pipOutputBatchLogSize == 0)
{
Console.Write(".");
database.ApplyBatch(pipSemistableMap);
database.ApplyBatch(pipIdMap, XldbDataStore.PipColumnFamilyName);
pipSemistableMap.Clear();
pipIdMap.Clear();
}
var hydratedPip = CachedGraph.PipTable.HydratePip(pipId, BuildXL.Pips.PipQueryContext.PipGraphRetrieveAllPips);
var pipType = hydratedPip.PipType;
if (pipType == PipType.Value || pipType == PipType.HashSourceFile || pipType == PipType.SpecFile || pipType == PipType.Module)
{
continue;
}
var xldbPip = hydratedPip.ToPip(CachedGraph);
var pipIdKey = new PipIdKey()
{
PipId = hydratedPip.PipId.Value,
PipType = (Xldb.Proto.PipType)(pipType + 1)
};
var pipIdKeyArr = pipIdKey.ToByteArray();
IMessage xldbSpecificPip = xldbPip;
if (pipType == PipType.Ipc)
{
var ipcPip = (Pips.Operations.IpcPip)hydratedPip;
xldbSpecificPip = ipcPip.ToIpcPip(PathTable, xldbPip, m_nameExpander);
foreach (var fileArtifact in ipcPip.FileDependencies)
{
AddToFileConsumerMap(fileArtifact, pipId);
}
foreach (var directoryArtifact in ipcPip.DirectoryDependencies)
{
AddToDirectoryConsumerMap(directoryArtifact, pipId);
}
AddToDbStorageDictionary(DBStoredTypes.IpcPip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else if (pipType == PipType.SealDirectory)
{
var sealDirectoryPip = (Pips.Operations.SealDirectory)hydratedPip;
xldbSpecificPip = sealDirectoryPip.ToSealDirectory(PathTable, xldbPip, m_nameExpander);
// If it is a shared opaque, then flatten the list of composted directories
if (sealDirectoryPip.Kind == Pips.Operations.SealDirectoryKind.SharedOpaque)
{
var directoryQueue = new Queue <Utilities.DirectoryArtifact>();
foreach (var initialDirectory in sealDirectoryPip.ComposedDirectories)
{
directoryQueue.Enqueue(initialDirectory);
}
while (directoryQueue.Count > 0)
{
var nestedDirectory = directoryQueue.Dequeue();
var nestedPipId = CachedGraph.PipGraph.GetSealedDirectoryNode(nestedDirectory).ToPipId();
if (CachedGraph.PipTable.IsSealDirectoryComposite(nestedPipId))
{
var nestedPip = (Pips.Operations.SealDirectory)CachedGraph.PipGraph.GetSealedDirectoryPip(nestedDirectory, BuildXL.Pips.PipQueryContext.SchedulerExecuteSealDirectoryPip);
foreach (var pendingDirectory in nestedPip.ComposedDirectories)
{
directoryQueue.Enqueue(pendingDirectory);
}
}
else
{
Contract.Assert(nestedDirectory.IsOutputDirectory());
AddToDirectoryConsumerMap(nestedDirectory, pipId);
}
}
}
foreach (var fileArtifact in sealDirectoryPip.Contents)
{
AddToFileConsumerMap(fileArtifact, pipId);
}
AddToDbStorageDictionary(DBStoredTypes.SealDirectoryPip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else if (pipType == PipType.CopyFile)
{
var copyFilePip = (Pips.Operations.CopyFile)hydratedPip;
xldbSpecificPip = copyFilePip.ToCopyFile(PathTable, xldbPip, m_nameExpander);
AddToFileConsumerMap(copyFilePip.Source, pipId);
AddToDbStorageDictionary(DBStoredTypes.CopyFilePip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else if (pipType == PipType.WriteFile)
{
var writeFilePip = (Pips.Operations.WriteFile)hydratedPip;
xldbSpecificPip = writeFilePip.ToWriteFile(PathTable, xldbPip, m_nameExpander);
AddToDbStorageDictionary(DBStoredTypes.WriteFilePip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else if (pipType == PipType.Process)
{
var processPip = (Pips.Operations.Process)hydratedPip;
xldbSpecificPip = processPip.ToProcessPip(PathTable, xldbPip, m_nameExpander);
AddToFileConsumerMap(processPip.StandardInputFile, pipId);
AddToFileConsumerMap(processPip.Executable, pipId);
foreach (var fileArtifact in processPip.Dependencies)
{
AddToFileConsumerMap(fileArtifact, pipId);
}
foreach (var directoryArtifact in processPip.DirectoryDependencies)
{
AddToDirectoryConsumerMap(directoryArtifact, pipId);
}
AddToDbStorageDictionary(DBStoredTypes.ProcessPip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
}
else
{
Contract.Assert(false, "Unknown pip type parsed");
}
// If the SemiStableHash != 0, then we want to create the SemistableHash -> PipId indirection.
// Else we do not want that since the key would no longer be unique
if (hydratedPip.SemiStableHash != 0)
{
var pipSemistableHashKey = new PipSemistableHashKey()
{
SemiStableHash = hydratedPip.SemiStableHash,
};
pipSemistableMap.Add(pipSemistableHashKey.ToByteArray(), pipIdKeyArr);
}
pipIdMap.Add(pipIdKeyArr, xldbSpecificPip.ToByteArray());
}
database.ApplyBatch(pipSemistableMap);
database.ApplyBatch(pipIdMap, XldbDataStore.PipColumnFamilyName);
});
if (!maybeInserted.Succeeded)
{
Console.WriteLine("Failed to insert pip data into RocksDb. Exiting analyzer now ...");
maybeInserted.Failure.Throw();
}
});
}