/// <summary>
/// Creates a new change tracker in the <see cref="FileChangeTrackingState.BuildingInitialChangeTrackingSet"/> state.
/// The caller may then add tracking for full set of files of interest, for later re-use by
/// <see cref="ResumeTrackingChanges(LoggingContext,BuildXL.Utilities.FileEnvelopeId,VolumeMap,IChangeJournalAccessor,FileChangeTrackingSet,string)"/>.
/// </summary>
public static FileChangeTracker StartTrackingChanges(
LoggingContext loggingContext,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
string buildEngineFingerprint,
FileEnvelopeId?correlatedId = default)
{
Contract.Requires(loggingContext != null);
Contract.Requires(volumeMap != null);
Contract.Requires(journal != null);
Contract.Ensures(Contract.Result <FileChangeTracker>().TrackingState == FileChangeTrackingState.BuildingInitialChangeTrackingSet);
var tracker = new FileChangeTracker(
loggingContext,
correlatedId ?? FileEnvelopeId.Create(),
FileChangeTrackingState.BuildingInitialChangeTrackingSet,
volumeMap,
journal,
FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, journal),
buildEngineFingerprint);
foreach (var gvfsProjectionFile in volumeMap.GvfsProjections)
{
var maybeTracking = tracker.TryProbeAndTrackPath(gvfsProjectionFile);
if (!maybeTracking.Succeeded)
{
Logger.Log.TrackChangesToGvfsProjectionFailed(loggingContext, gvfsProjectionFile, maybeTracking.Failure.DescribeIncludingInnerFailures());
}
}
return(tracker);
}
/// <summary>
/// Loads <see cref="FileChangeTracker"/>, and if successful, the <see cref="FileChangeTracker"/> is a disabled one.
/// </summary>
public static LoadingTrackerResult LoadTrackingChanges(
LoggingContext loggingContext,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
string path,
string buildEngineFingerprint,
out FileChangeTracker tracker,
bool loadForAllCapableVolumes = true)
{
Contract.Requires(!loadForAllCapableVolumes || volumeMap != null);
Contract.Requires(!loadForAllCapableVolumes || journal != null);
Contract.Requires(path != null);
tracker = null;
using (var pm = BuildXL.Tracing.PerformanceMeasurement.StartWithoutStatistic(
loggingContext,
loggingContext1 => Logger.Log.StartLoadingChangeTracker(loggingContext1, path),
loggingContext1 => Logger.Log.EndLoadingChangeTracker(loggingContext1)))
{
// Note that TryLoad may throw in the event of spooky I/O errors.
var loadingTrackerResult = TryLoad(
pm.LoggingContext,
path,
volumeMap,
journal,
buildEngineFingerprint,
loadForAllCapableVolumes: loadForAllCapableVolumes);
if (loadingTrackerResult.Succeeded)
{
Contract.Assert(loadingTrackerResult.ChangeTrackingSet != null);
tracker = new FileChangeTracker(
pm.LoggingContext,
loadingTrackerResult.FileId,
FileChangeTrackingState.TrackingChanges,
loadingTrackerResult.ChangeTrackingSet.VolumeMap,
journal ?? new InProcChangeJournalAccessor(),
loadingTrackerResult.ChangeTrackingSet,
buildEngineFingerprint);
}
else
{
Contract.Assert(loadingTrackerResult.ChangeTrackingSet == null);
}
Logger.Log.LoadingChangeTracker(
pm.LoggingContext,
path,
loadingTrackerResult.FileId.ToString(),
loadingTrackerResult.Status.ToString(),
loadingTrackerResult.StatusAsString,
loadingTrackerResult.TrackedVolumesCount,
(long)loadingTrackerResult.TrackedJournalsSizeBytes,
loadingTrackerResult.DurationMs);
return(loadingTrackerResult);
}
}
private ChangeTrackerSupport(
LoggingContext loggingContext,
PathTable pathTable,
FileChangeTracker fileChangeTracker,
FileContentTable fileContentTable,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
AbsolutePath temporaryDirectory,
string buildEngineFingerprint)
{
Contract.Requires(loggingContext != null);
Contract.Requires(pathTable != null);
Contract.Requires(fileChangeTracker != null);
Contract.Requires(fileContentTable != null);
Contract.Requires(temporaryDirectory.IsValid);
m_loggingContext = loggingContext;
m_pathTable = pathTable;
m_fileChangeTracker = fileChangeTracker;
m_fileContentTable = fileContentTable;
m_volumeMap = volumeMap;
m_journal = journal;
m_fileContentTablePath = temporaryDirectory.Combine(m_pathTable, FileContentTableName);
m_fileChangeTrackerPath = temporaryDirectory.Combine(m_pathTable, FileChangeTrackerName);
m_buildEngineFingerprint = buildEngineFingerprint;
}
private FileChangeTracker(
LoggingContext loggingContext,
FileEnvelopeId fileEnvelopeId,
FileChangeTrackingState initialState,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
FileChangeTrackingSet currentChangeTrackingSet,
string buildEngineFingerprint)
{
Contract.Requires(loggingContext != null);
Contract.Requires(fileEnvelopeId.IsValid);
Contract.Requires(
initialState == FileChangeTrackingState.BuildingInitialChangeTrackingSet || initialState == FileChangeTrackingState.TrackingChanges);
Contract.Requires(volumeMap != null);
Contract.Requires(journal != null);
Contract.Requires(currentChangeTrackingSet != null);
FileEnvelopeId = fileEnvelopeId;
m_loggingContext = loggingContext;
m_volumeMap = volumeMap;
m_journal = journal;
m_changeTrackingSet = currentChangeTrackingSet;
m_trackingStateValue = (int)initialState;
Counters = m_changeTrackingSet.Counters;
m_buildEngineFingerprint = buildEngineFingerprint;
}
/// <summary>
/// Creates a state with enabled journal.
/// </summary>
public static JournalState CreateEnabledJournal(VolumeMap volumeMap, IChangeJournalAccessor journal)
{
Contract.Requires(volumeMap != null);
Contract.Requires(journal != null);
return(new JournalState(volumeMap, journal));
}
/// <summary>
/// Tries to load a <see cref="FileChangeTrackingSet"/> and resume tracking changes.
/// If loading is successful (including matching the provided atomic save token), the returned tracker begins in the <see cref="FileChangeTrackingState.TrackingChanges"/> state.
/// The caller may then query for new changes since the tracking set was last checkpointed and persisted, and may track additional files.
/// Otherwise, a tracker with a new and empty change tracking set is created, initially in the <see cref="FileChangeTrackingState.BuildingInitialChangeTrackingSet"/> state.
/// In that case, attempting to query changes will fail (the caller should fall back to doing complete rather than incremental work, thus populating the new change tracking set).
/// </summary>
/// <exception cref="BuildXLException">Thrown in the event of an I/O error other than the given path being absent.</exception>
public static LoadingTrackerResult ResumeOrRestartTrackingChanges(
LoggingContext loggingContext,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
string path,
string buildEngineFingerprint,
out FileChangeTracker tracker)
{
Contract.Requires(volumeMap != null);
Contract.Requires(journal != null);
Contract.Requires(path != null);
using (var pm = BuildXL.Tracing.PerformanceMeasurement.StartWithoutStatistic(
loggingContext,
loggingContext1 => Logger.Log.StartLoadingChangeTracker(loggingContext1, path),
loggingContext1 => Logger.Log.EndLoadingChangeTracker(loggingContext1)))
{
// Note that TryLoad may throw in the event of spooky I/O errors.
var loadingTrackerResult = TryLoad(pm.LoggingContext, path, volumeMap, journal, buildEngineFingerprint);
if (loadingTrackerResult.Succeeded)
{
Contract.Assert(loadingTrackerResult.ChangeTrackingSet != null);
// Ideally, we reload prior state so that the caller can query changes and do incremental work.
// In this case, we already validated the correlating atomic save token, and so the state is safe to reuse.
tracker = ResumeTrackingChanges(
pm.LoggingContext,
loadingTrackerResult.FileId,
volumeMap,
journal,
loadingTrackerResult.ChangeTrackingSet,
buildEngineFingerprint);
}
else
{
Contract.Assert(loadingTrackerResult.ChangeTrackingSet == null);
// Or, we might be unable to re-use the persisted state. In that case we start over. Note that there's nothing to do with the correlating save token here;
// on save, a new or existing token will be provided as appropriate.
// The reason of the failure is already logged in the TryLoad() method above.
tracker = StartTrackingChanges(pm.LoggingContext, volumeMap, journal, buildEngineFingerprint);
}
Logger.Log.LoadingChangeTracker(
pm.LoggingContext,
path,
loadingTrackerResult.FileId.ToString(),
loadingTrackerResult.Status.ToString(),
loadingTrackerResult.StatusAsString,
loadingTrackerResult.TrackedVolumesCount,
(long)loadingTrackerResult.TrackedJournalsSizeBytes,
loadingTrackerResult.DurationMs);
return(loadingTrackerResult);
}
}
private FileChangeTracker(LoggingContext loggingContext)
{
Contract.Requires(loggingContext != null);
m_loggingContext = loggingContext;
m_volumeMap = null;
m_journal = null;
m_changeTrackingSet = null;
m_trackingStateValue = (int)FileChangeTrackingState.DisabledSinceTrackingIsIncomplete;
Counters = new CounterCollection <FileChangeTrackingCounter>();
m_buildEngineFingerprint = null;
}
public void StartTracking()
{
var loggingContext = new LoggingContext("JournalTesting", "Dummy");
VolumeMap volumeMap = JournalUtils.TryCreateMapOfAllLocalVolumes(loggingContext);
XAssert.IsNotNull(volumeMap);
var maybeJournal = JournalUtils.TryGetJournalAccessorForTest(volumeMap);
XAssert.IsTrue(maybeJournal.Succeeded);
m_journal = maybeJournal.Result;
m_changeTracker = FileChangeTrackingSet.CreateForAllCapableVolumes(
loggingContext,
volumeMap,
m_journal);
}
/// <summary>
/// Creates a new change tracker in the <see cref="FileChangeTrackingState.BuildingInitialChangeTrackingSet"/> state.
/// The caller may then add tracking for full set of files of interest, for later re-use by
/// <see cref="ResumeTrackingChanges(LoggingContext,BuildXL.Utilities.FileEnvelopeId,VolumeMap,IChangeJournalAccessor,FileChangeTrackingSet,string)"/>.
/// </summary>
public static FileChangeTracker StartTrackingChanges(
LoggingContext loggingContext,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
string buildEngineFingerprint,
FileEnvelopeId?correlatedId = default)
{
Contract.Requires(loggingContext != null);
Contract.Requires(volumeMap != null);
Contract.Requires(journal != null);
Contract.Ensures(Contract.Result <FileChangeTracker>().TrackingState == FileChangeTrackingState.BuildingInitialChangeTrackingSet);
return(new FileChangeTracker(
loggingContext,
correlatedId ?? FileEnvelopeId.Create(),
FileChangeTrackingState.BuildingInitialChangeTrackingSet,
volumeMap,
journal,
FileChangeTrackingSet.CreateForAllCapableVolumes(loggingContext, volumeMap, journal),
buildEngineFingerprint));
}
/// <summary>
/// Creates a new change tracker in the <see cref="FileChangeTrackingState.TrackingChanges"/> state.
/// The caller may query for new changes since the tracking set was last checkpointed and persisted, and may track additional files.
/// </summary>
private static FileChangeTracker ResumeTrackingChanges(
LoggingContext loggingContext,
FileEnvelopeId fileEnvelopeId,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
FileChangeTrackingSet previousChangeTrackingSet,
string buildEngineFingerprint)
{
Contract.Requires(loggingContext != null);
Contract.Requires(volumeMap != null);
Contract.Requires(journal != null);
Contract.Requires(previousChangeTrackingSet != null);
Contract.Requires(fileEnvelopeId.IsValid);
return(new FileChangeTracker(
loggingContext,
fileEnvelopeId,
FileChangeTrackingState.TrackingChanges,
volumeMap,
journal,
previousChangeTrackingSet,
buildEngineFingerprint));
}
public static LoadingTrackerResult TryLoad(
LoggingContext loggingContext,
string path,
VolumeMap volumeMap,
IChangeJournalAccessor journal,
string buildEngineFingerprint,
bool loadForAllCapableVolumes = true)
{
Contract.Requires(loggingContext != null);
Contract.Requires(!loadForAllCapableVolumes || volumeMap != null);
Contract.Requires(!loadForAllCapableVolumes || journal != null);
Stopwatch stopwatch = Stopwatch.StartNew();
SafeFileHandle handle;
OpenFileResult result = FileUtilities.TryCreateOrOpenFile(
path,
FileDesiredAccess.GenericRead,
FileShare.Read | FileShare.Delete,
FileMode.Open,
// Ok to evict the file from standby since the file will be overwritten and never reread from disk after this point.
FileFlagsAndAttributes.FileFlagSequentialScan,
out handle);
if (result.Succeeded)
{
Contract.Assume(handle != null);
Contract.Assume(!handle.IsInvalid);
using (handle)
{
using (var stream = new FileStream(handle, FileAccess.Read))
{
FileEnvelopeId fileEnvelopeId;
try
{
fileEnvelopeId = FileEnvelope.ReadHeader(stream);
}
catch (BuildXLException e)
{
return(LoadingTrackerResult.FailBadFormatMarker(e.Message));
}
try
{
return(ExceptionUtilities.HandleRecoverableIOException(
() =>
{
using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true))
{
bool wasTrackerDisabled = reader.ReadBoolean();
if (wasTrackerDisabled)
{
return LoadingTrackerResult.FailPriorTrackerDisabled();
}
var previousFingerprint = reader.ReadNullable(r => r.ReadString());
// only check for fingerprints match if the supplied fingerprint is valid
// this is to support special cases where we might want to load ChangeTracker
// regardless of the previously stored fingerprint value
if (buildEngineFingerprint != null && !string.Equals(previousFingerprint, buildEngineFingerprint, StringComparison.Ordinal))
{
return LoadingTrackerResult.FailBuildEngineFingerprintMismatch();
}
return FileChangeTrackingSet.TryLoad(
loggingContext,
fileEnvelopeId,
reader,
volumeMap,
journal,
stopwatch,
loadForAllCapableVolumes);
}
},
ex =>
{
throw new BuildXLException(ex.Message);
}));
}
catch (Exception e)
{
// Catch any exception. Failure in loading FileChangeTracker should not
// cause build break, or worse, make people stuck on erroneous state.
// In such a case, BuildXL simply has to start tracking from scratch.
return(LoadingTrackerResult.FailLoadException(e.GetLogEventMessage()));
}
}
}
}
Contract.Assume(handle == null);
return(LoadingTrackerResult.FailTrackingSetCannotBeOpened(result.CreateFailureForError().DescribeIncludingInnerFailures()));
}
private JournalState(VolumeMap volumeMap, IChangeJournalAccessor journal)
{
Journal = journal;
VolumeMap = volumeMap;
}
