/// <summary>
/// Add snapshot to buffer if snapshot with same timestamp exists in buffer.
/// </summary>
public void addSnapshot(Snapshot snapshot)
{
if (!Snapshots.ContainsKey(snapshot.Time))
{
Snapshots.Add(snapshot.Time, snapshot);
}
}
public Repository AddSnapshot(IRepositorySnapshot snapshot) =>
new Repository(RepositoryName,
Snapshots.Match(s => s.Concat(new[] { snapshot })
.ToArray(),
new[] { snapshot }),
CoverageExecutionCommand,
CoverageOutputLocation);
/// <summary>
/// update snapshorts and size of link block.
/// </summary>
/// <param name="count">Expansion cards</param>
public void Cards(int count)
{
if (count > 2)
{
count = 2;
}
if (count < 0)
{
count = 0;
}
_size = new Point(Width, RowHeight * 4 + UnitHeight + (RowHeight * 5 * count)
+ _main.DataModel.ExpansionCards * InnerSpace);
RaisePropertyChanged(() => Size);
//set bottom snapshot location
var t = Snapshots.FirstOrDefault(s => s.RowId == 30);
if (t != null)
{
t.Offset.Y = Size.Y;
}
foreach (var snapshot in Snapshots)
{
snapshot.Calculate();
}
UpdateLinkOptions(count);
}
public BlLink(MainUnitViewModel main)
{
LinkOptions = new ObservableCollection <LinkOption>();
_main = main;
Location.X = XLocation;
Cards(main.DataModel.ExpansionCards);
for (var x = 0; x < 15; x++)
{
Snapshots.Add(new SnapShot(this)
{
Offset = { X = 0, Y = SnapshotHeight + RowHeight * x + _innerSpace(x) },
SnapType = SnapType.Gray,
RowId = x
});
Snapshots.Add(new SnapShot(this)
{
Offset = { X = Size.X, Y = SnapshotHeight + RowHeight * x + _innerSpace(x) },
SnapType = SnapType.Red,
RowId = x
});
}
Snapshots.Add(new SnapShot(this)
{
Offset = { X = SnapshotWidth, Y = Size.Y }, RowId = 30
});
}
/// <summary>
/// constructs the snapshot data for camera position and rays
/// </summary>
private static Snapshots[] UpdateSnapshots(int frames, float radius, Vector3 origin, bool isHalf = true)
{
var snapshots = new Snapshots[frames * frames];
float framesMinusOne = frames - 1;
var i = 0;
for (var y = 0; y < frames; y++)
{
for (var x = 0; x < frames; x++)
{
var vec = new Vector2(
x / framesMinusOne * 2f - 1f,
y / framesMinusOne * 2f - 1f
);
var ray = isHalf ? OctahedralCoordToVectorHemisphere(vec) : OctahedralCoordToVector(vec);
ray = ray.normalized;
snapshots[i].Position = origin + ray * radius;
snapshots[i].Ray = -ray;
i++;
}
}
return(snapshots);
}
/// <summary>
/// Returns the matching snapshot contained in the directory, otherwise null.
/// </summary>
/// <param name="directory"></param>
/// <param name="progressHandler">The optional handler to report progress to.</param>
/// <param name="progressStatus">The optional progress status description.</param>
/// <returns></returns>
public Snapshot FindSnapshotFromDirectory(string directory, ProgressChangedEventHandler progressHandler = null, string progressStatus = null)
{
// searches snapshots by file count in descending order to find the most specific match possible
Snapshot snapshot = new Snapshot(directory, progressHandler, progressStatus); // NOTE: this needs to be separated to prevent multiple evaluations
return(Snapshots.OrderByDescending(s => s.Files.Count).FirstOrDefault(s => snapshot.Contains(s)));
}
public void Snapshot(double fiat, double security, double securityPrice, double benchmarkPrice = 0)
{
if (benchmarkPrice == 0)
{
benchmarkPrice = securityPrice;
}
double portfolioValue = fiat + security * securityPrice;
bool hasOne = Snapshots.Count > 0;
var newSnapshot = new PortfolioSnapshot()
{
Time = DateTime.Now,
Fiat = fiat,
Security = security,
SecurityPrice = securityPrice,
PortfolioValue = portfolioValue,
PL = Snapshots.Count > 0 ? portfolioValue / Snapshots.Last().PortfolioValue - 1 : 0.0,
CumulativePL = hasOne ? portfolioValue / Snapshots.First().PortfolioValue - 1 : 0.0,
BenchmarkPrice = benchmarkPrice,
BenchmarkPL = hasOne ? benchmarkPrice / Snapshots.Last().BenchmarkPrice - 1 : 0.0,
BenchmarkCumulativePL = hasOne ? benchmarkPrice / Snapshots.First().BenchmarkPrice - 1 : 0.0
};
Snapshots.Add(newSnapshot);
SnapshotEvent?.Invoke(this, newSnapshot);
CachedSharpeRatio = null;
}
public void review_snapshot_is_taken()
{
Snapshots.Should()
.BeEquivalentTo(new ReviewSnapshot(Guid.Empty, ReviewId, 1, NewCaption, NewContent, Status.Approved),
o => o.ExcludingFields()
.Excluding(q => q.Id)
);
}
/// <summary>
/// Add a snapshot to the list.
/// </summary>
/// <param name="snapshot">Snapshot to add.</param>
public void Add(VMWareSnapshot snapshot)
{
if (snapshot.Parent != null && snapshot.Parent != _parent)
{
throw new InvalidOperationException("Snapshot already belongs to another collection.");
}
Snapshots.Add(snapshot);
}
/// <inheritdoc />
public void AddSnapshot(ITableEntriesSnapshot newSnapshot, bool removeAllSnapshots = false)
{
if (removeAllSnapshots)
{
RemoveAllSnapshots();
}
Snapshots.Add(newSnapshot);
}
/// <inheritdoc />
public void ReplaceSnapshot(ITableEntriesSnapshot oldSnapshot, ITableEntriesSnapshot newSnapshot)
{
var index = Snapshots.IndexOf(oldSnapshot);
if (index != -1)
{
Snapshots[index] = newSnapshot;
}
}
protected internal virtual void PrepareIndexAndSnapshots(SnapshotDeletionPolicy sdp, IndexWriter writer, int numSnapshots)
{
for (int i = 0; i < numSnapshots; i++)
{
// create dummy document to trigger commit.
writer.AddDocument(new Document());
writer.Commit();
Snapshots.Add(sdp.Snapshot());
}
}
public override void Dispose()
{
Snapshots.ForEach(s => s.Dispose());
Snapshots.Clear();
SnapshotSummary.Dispose();
SnapshotSummary = null;
base.Dispose();
}
public double SharpeRatio()
{
if (CachedSharpeRatio == null)
{
int timeBetweenTrades = (int)(Snapshots[1].Time - Snapshots[0].Time).TotalMinutes;
int periodsPerYear = 525600 / timeBetweenTrades; //number of minutes in 1 year / number of minutes between each trade
CachedSharpeRatio = SharpeRatio(periodsPerYear, Snapshots.Select(snapshot => snapshot.PL), Snapshots.Select(snapshot => snapshot.BenchmarkPL));
}
return(CachedSharpeRatio.Value);
}
public void RebuildSnapshotIndexes()
{
var i = 0;
foreach (var snapshot in Snapshots.OrderBy(o => o.Index))
{
snapshot.Index = i;
i++;
}
Snapshots.Sort((x, y) => x.Index.CompareTo(y.Index));
}
/// <summary>
/// Take a snapshot of the current state of the blueprint and add it to the snapshots collection
/// </summary>
public void TakeSnapshot()
{
if (Snapshots != null)
{
var snap = new SquareCellMap <CellGenerationType>(_Blueprint.SizeX, _Blueprint.SizeY);
for (int i = 0; i < Blueprint.CellCount; i++)
{
snap[i] = Blueprint[i].GenerationType;
}
Snapshots.Add(snap);
}
}
public void SaveCollectedDataToDatabase(TMDBSnapshot tmdbSnapshot)
{
var snapshot = new Snapshots
{
LanguageCode = tmdbSnapshot.LanguageCode,
RegionCode = tmdbSnapshot.RegionCode,
CompressedBase64JSONData = tmdbSnapshot.CompressedBase64JSONData
};
_context.Snapshots.Add(snapshot);
_context.SaveChanges();
}
private IEnumerable <string> Compare(DataSnapshot dataSnapshot)
{
var missing = false;
//TODO: do we need to create a buffer for the hash sets?
HashSet <string> changedKeys = null;
foreach (var(key, value) in Snapshots)
{
if (dataSnapshot.Snapshots.TryGetValue(key, out var snapShotValue))
{
if (snapShotValue < value)
{
if (changedKeys == null)
{
changedKeys = new HashSet <string>();
}
changedKeys.Add(key);
}
}
else
{
missing = true;
if (changedKeys == null)
{
changedKeys = new HashSet <string>();
}
changedKeys.Add(key);
}
}
if (missing || Snapshots.Count != dataSnapshot.Snapshots.Count
) // snapshot contains at least one removed key or has a different size, we need to notify the player about that
{
//TODO: is the removed key also just a changed key, because then we have to deliver a null mvalue as well, yeah why not
foreach (var(key, _) in dataSnapshot.Snapshots)
{
if (!Snapshots.ContainsKey(key))
{
if (changedKeys == null)
{
changedKeys = new HashSet <string>();
}
changedKeys.Add(key);
}
}
}
return(changedKeys);
}
public async Task with_snapshot_but_without_stream_should_throw_stale_aggregate_exception()
{
var ticketState = new TicketState();
var snapshot = new SnapshotInfo("Ticket_1", 2, ticketState, "1");
await Snapshots.AddAsync("Ticket_1", snapshot).ConfigureAwait(false);
var ex = await Assert.ThrowsAsync <StaleSnapshotException>(() =>
Repository.GetByIdAsync <Ticket>("Ticket_1")
).ConfigureAwait(false);
Assert.Equal("Ticket_1", ex.AggregateId);
Assert.Equal(2, ex.AggregateVersion);
}
public void Execute(IJobExecutionContext context)
{
Thread.CurrentThread.Priority = ThreadPriority.Lowest;
var snapshot = Snapshots.GenerateSnapshot();
Sample sample = RrdDB.createSample((Int64)snapshot.Get(SnapshotValue.Timestamp));
for (int a = 1; a < Snapshot.SnapshotCount; a++)
{
sample.setValue(a + "", snapshot.Get((SnapshotValue)a));
}
sample.update();
}
public async Task saving_new_aggregate_should_create_snapshot()
{
var ticket = await Repository.GetByIdAsync <Ticket>("new_ticket").ConfigureAwait(false);
ticket.Sale();
await Repository.SaveAsync(ticket, "save_snap").ConfigureAwait(false);
var snapshot = await Snapshots.GetAsync("new_ticket", int.MaxValue).ConfigureAwait(false);
Assert.NotNull(snapshot);
Assert.Equal(1, snapshot.SourceVersion);
Assert.NotNull(snapshot.Payload);
Assert.Equal("1", snapshot.SchemaVersion);
}
public async Task saving_should_create_snapshot()
{
var ticket = await Repository.GetById <Ticket>("Ticket_1");
ticket.Refund();
await Repository.Save(ticket, "save_snap");
var snapshot = await Snapshots.GetAsync("Ticket_1", int.MaxValue);
Assert.NotNull(snapshot);
Assert.Equal(3, snapshot.SourceVersion);
Assert.NotNull(snapshot.Payload);
Assert.Equal("1", snapshot.SchemaVersion);
}
public async Task saving_new_aggregate_should_create_snapshot()
{
var ticket = await Repository.GetById <Ticket>("new_ticket");
ticket.Sale();
await Repository.Save(ticket, "save_snap");
var snapshot = await Snapshots.Get("new_ticket", int.MaxValue);
Assert.NotNull(snapshot);
Assert.Equal(1, snapshot.AggregateVersion);
Assert.NotNull(snapshot.Data);
Assert.Equal(1, snapshot.SnapshotVersion);
}
/// <summary>
/// [Re-]initialize the simulation.
/// </summary>
private void ResetSimulation(string param)
{
Snapshots.ResetFrameNumber();
if (world != null)
{
lock ( world )
{
// ResetDataBuffers();
world.Reset(param);
ticksLast = DateTime.Now.Ticks;
timeLast = 0.0;
}
}
}
protected override YAMLMappingNode ExportYAMLRoot(IAssetsExporter exporter)
{
YAMLMappingNode node = base.ExportYAMLRoot(exporter);
//node.AddSerializedVersion(GetSerializedVersion(exporter.Version));
node.Add("m_OutputGroup", OutputGroup.ExportYAML(exporter));
node.Add("m_MasterGroup", MasterGroup.ExportYAML(exporter));
node.Add("m_Snapshots", Snapshots.ExportYAML(exporter));
node.Add("m_StartSnapshot", StartSnapshot.ExportYAML(exporter));
node.Add("m_SuspendThreshold", SuspendThreshold);
node.Add("m_EnableSuspend", EnableSuspend);
node.Add("m_UpdateMode", UpdateMode);
node.Add("m_MixerConstant", MixerConstant.ExportYAML(exporter));
return(node);
}
protected override YAMLMappingNode ExportYAMLRoot(IExportContainer container)
{
YAMLMappingNode node = base.ExportYAMLRoot(container);
//node.AddSerializedVersion(GetSerializedVersion(container.Version));
node.Add(OutputGroupName, OutputGroup.ExportYAML(container));
node.Add(MasterGroupName, MasterGroup.ExportYAML(container));
node.Add(SnapshotsName, Snapshots.ExportYAML(container));
node.Add(StartSnapshotName, StartSnapshot.ExportYAML(container));
node.Add(SuspendThresholdName, SuspendThreshold);
node.Add(EnableSuspendName, EnableSuspend);
node.Add(UpdateModeName, UpdateMode);
node.Add(MixerConstantName, MixerConstant.ExportYAML(container));
return(node);
}
public virtual void TestExistingSnapshots()
{
int numSnapshots = 3;
MockDirectoryWrapper dir = NewMockDirectory();
IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), GetDeletionPolicy(dir)));
PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
Assert.IsNull(psdp.LastSaveFile);
PrepareIndexAndSnapshots(psdp, writer, numSnapshots);
Assert.IsNotNull(psdp.LastSaveFile);
writer.Dispose();
// Make sure only 1 save file exists:
int count = 0;
foreach (string file in dir.ListAll())
{
if (file.StartsWith(PersistentSnapshotDeletionPolicy.SNAPSHOTS_PREFIX, StringComparison.Ordinal))
{
count++;
}
}
Assert.AreEqual(1, count);
// Make sure we fsync:
dir.Crash();
dir.ClearCrash();
// Re-initialize and verify snapshots were persisted
psdp = new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.APPEND);
writer = new IndexWriter(dir, GetConfig(Random(), psdp));
psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
Assert.AreEqual(numSnapshots, psdp.GetSnapshots().Count);
Assert.AreEqual(numSnapshots, psdp.SnapshotCount);
AssertSnapshotExists(dir, psdp, numSnapshots, false);
writer.AddDocument(new Document());
writer.Commit();
Snapshots.Add(psdp.Snapshot());
Assert.AreEqual(numSnapshots + 1, psdp.GetSnapshots().Count);
Assert.AreEqual(numSnapshots + 1, psdp.SnapshotCount);
AssertSnapshotExists(dir, psdp, numSnapshots + 1, false);
writer.Dispose();
dir.Dispose();
}
GetSubVolumeInfo(string SourcePath, string fullpath, CancellationToken ct)
{
var res = await Toolkit.Exec("btrfs", new[] { "sub", "show", fullpath }, ct, development);
var uuid = "";
var parentUUID = "";
var generation = 0L;
var genAtCreation = 0L;
List <string> Snapshots = null;
foreach (var y in res.output.Lines())
{
if (y.Trim().StartsWith("UUID:"))
{
uuid = y.Trim().StripBegin("UUID:").Trim();
}
else if (y.Trim().StartsWith("Parent UUID:"))
{
parentUUID = y.Trim().StripBegin("Parent UUID:").Trim();
}
else if (y.Trim().StartsWith("Generation:"))
{
generation = long.Parse(y.Trim().StripBegin("Generation:").Trim());
}
else if (y.Trim().StartsWith("Gen at creation:"))
{
genAtCreation = long.Parse(y.Trim().StripBegin("Gen at creation:").Trim());
}
else if (y.Trim().StartsWith("Snapshot(s):"))
{
Snapshots = new List <string>();
}
else if (Snapshots != null)
{
Snapshots.Add(Path.Combine(SourcePath, y.Trim()));
}
}
if (Snapshots == null)
{
Snapshots = new List <string>();
}
return(uuid, parentUUID, generation, genAtCreation, Snapshots);
}
public static void StartStopScreencast(bool start)
{
if (start == (screencast != null))
{
return;
}
if (start)
{
screencast = new Snapshots(true);
screencast.StartSaveThread();
}
else
{
screencast.StopSaveThread();
screencast = null;
}
}
public YAMLNode ExportYAML(IExportContainer container)
{
YAMLMappingNode node = new YAMLMappingNode();
//node.AddSerializedVersion(GetSerializedVersion(container.Version));
node.Add("groups", Groups.ExportYAML(container));
node.Add("groupGUIDs", GroupGUIDs.ExportYAML(container));
node.Add("effects", Effects.ExportYAML(container));
node.Add("effectGUIDs", EffectGUIDs.ExportYAML(container));
node.Add("numSideChainBuffers", NumSideChainBuffers);
node.Add("snapshots", Snapshots.ExportYAML(container));
node.Add("snapshotGUIDs", SnapshotGUIDs.ExportYAML(container));
//node.Add("groupNameBuffer", GroupNameBuffer.ExportYAML(container));
//node.Add("snapshotNameBuffer", SnapshotNameBuffer.ExportYAML(container));
//node.Add("pluginEffectNameBuffer", PluginEffectNameBuffer.ExportYAML(container));
node.Add("exposedParameterNames", ExposedParameterNames.ExportYAML(true));
node.Add("exposedParameterIndices", ExposedParameterIndices.ExportYAML(true));
return(node);
}
public FilterHandler(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter filter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, ILogWriter logWriter)
{
m_snapshot = snapshot;
m_attributeFilter = attributeFilter;
m_sourcefilter = sourcefilter;
m_emitfilter = filter;
m_symlinkPolicy = symlinkPolicy;
m_hardlinkPolicy = hardlinkPolicy;
m_logWriter = logWriter;
m_hardlinkmap = new Dictionary<string, string>();
m_mixinqueue = new Queue<string>();
bool includes;
bool excludes;
Library.Utility.FilterExpression.AnalyzeFilters(filter, out includes, out excludes);
if (includes && !excludes)
{
m_enumeratefilter = Library.Utility.FilterExpression.Combine(filter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true));
}
else
m_enumeratefilter = m_emitfilter;
}
private void RunMainOperation(Snapshots.ISnapshotService snapshot, BackendManager backend)
{
var filterhandler = new FilterHandler(snapshot, m_attributeFilter, m_sourceFilter, m_filter, m_symlinkPolicy, m_options.HardlinkPolicy, m_result);
using(new Logging.Timer("BackupMainOperation"))
{
if (m_options.ChangedFilelist != null && m_options.ChangedFilelist.Length >= 1)
{
m_result.AddVerboseMessage("Processing supplied change list instead of enumerating filesystem");
m_result.OperationProgressUpdater.UpdatefileCount(m_options.ChangedFilelist.Length, 0, true);
foreach(var p in filterhandler.Mixin(m_options.ChangedFilelist))
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
m_result.AddMessage("Stopping backup operation on request");
break;
}
try
{
this.HandleFilesystemEntry(snapshot, backend, p, snapshot.GetAttributes(p));
}
catch (Exception ex)
{
m_result.AddWarning(string.Format("Failed to process element: {0}, message: {1}", p, ex.Message), ex);
}
}
m_database.AppendFilesFromPreviousSet(m_transaction, m_options.DeletedFilelist);
}
else
{
foreach(var path in filterhandler.EnumerateFilesAndFolders())
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
m_result.AddMessage("Stopping backup operation on request");
break;
}
var fa = FileAttributes.Normal;
try { fa = snapshot.GetAttributes(path); }
catch { }
this.HandleFilesystemEntry(snapshot, backend, path, fa);
}
}
m_result.OperationProgressUpdater.UpdatefileCount(m_result.ExaminedFiles, m_result.SizeOfExaminedFiles, true);
}
}
private Dictionary<string, string> GenerateMetadata(Snapshots.ISnapshotService snapshot, string path, System.IO.FileAttributes attributes)
{
try
{
Dictionary<string, string> metadata;
if (m_options.StoreMetadata)
{
metadata = snapshot.GetMetadata(path);
if (metadata == null)
metadata = new Dictionary<string, string>();
if (!metadata.ContainsKey("CoreAttributes"))
metadata["CoreAttributes"] = attributes.ToString();
if (!metadata.ContainsKey("CoreLastWritetime"))
{
try
{
metadata["CoreLastWritetime"] = snapshot.GetLastWriteTimeUtc(path).Ticks.ToString();
}
catch (Exception ex)
{
m_result.AddWarning(string.Format("Failed to read timestamp on \"{0}\"", path), ex);
}
}
if (!metadata.ContainsKey("CoreCreatetime"))
{
try
{
metadata["CoreCreatetime"] = snapshot.GetCreationTimeUtc(path).Ticks.ToString();
}
catch (Exception ex)
{
m_result.AddWarning(string.Format("Failed to read timestamp on \"{0}\"", path), ex);
}
}
}
else
{
metadata = new Dictionary<string, string>();
}
return metadata;
}
catch(Exception ex)
{
m_result.AddWarning(string.Format("Failed to process metadata for \"{0}\", storing empty metadata", path), ex);
return new Dictionary<string, string>();
}
}
private bool HandleFilesystemEntry(Snapshots.ISnapshotService snapshot, BackendManager backend, string path, System.IO.FileAttributes attributes)
{
// If we lost the connection, there is no point in keeping on processing
if (backend.HasDied)
throw backend.LastException;
try
{
m_result.OperationProgressUpdater.StartFile(path, -1);
if (m_backendLogFlushTimer < DateTime.Now)
{
m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN);
backend.FlushDbMessages(m_database, null);
}
DateTime lastwrite = new DateTime(0, DateTimeKind.Utc);
try
{
lastwrite = snapshot.GetLastWriteTimeUtc(path);
}
catch (Exception ex)
{
m_result.AddWarning(string.Format("Failed to read timestamp on \"{0}\"", path), ex);
}
if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint)
{
if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore)
{
m_result.AddVerboseMessage("Ignoring symlink {0}", path);
return false;
}
if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store)
{
Dictionary<string, string> metadata = GenerateMetadata(snapshot, path, attributes);
if (!metadata.ContainsKey("CoreSymlinkTarget"))
{
var p = snapshot.GetSymlinkTarget(path);
if (string.IsNullOrWhiteSpace(p))
m_result.AddVerboseMessage("Ignoring empty symlink {0}", path);
else
metadata["CoreSymlinkTarget"] = p;
}
var metahash = Utility.WrapMetadata(metadata, m_options);
AddSymlinkToOutput(backend, path, DateTime.UtcNow, metahash);
m_result.AddVerboseMessage("Stored symlink {0}", path);
//Do not recurse symlinks
return false;
}
}
if ((attributes & FileAttributes.Directory) == FileAttributes.Directory)
{
IMetahash metahash;
if (m_options.StoreMetadata)
{
metahash = Utility.WrapMetadata(GenerateMetadata(snapshot, path, attributes), m_options);
}
else
{
metahash = EMPTY_METADATA;
}
m_result.AddVerboseMessage("Adding directory {0}", path);
AddFolderToOutput(backend, path, lastwrite, metahash);
return true;
}
m_result.OperationProgressUpdater.UpdatefilesProcessed(++m_result.ExaminedFiles, m_result.SizeOfExaminedFiles);
bool changed = false;
// Last scan time
DateTime oldModified;
long lastFileSize = -1;
string oldMetahash;
long oldMetasize;
var oldId = m_database.GetFileEntry(path, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize);
long filestatsize = -1;
try { filestatsize = snapshot.GetFileSize(path); }
catch { }
IMetahash metahashandsize = m_options.StoreMetadata ? Utility.WrapMetadata(GenerateMetadata(snapshot, path, attributes), m_options) : EMPTY_METADATA;
var timestampChanged = lastwrite != oldModified || lastwrite.Ticks == 0 || oldModified.Ticks == 0;
var filesizeChanged = filestatsize < 0 || lastFileSize < 0 || filestatsize != lastFileSize;
var tooLargeFile = m_options.SkipFilesLargerThan != long.MaxValue && m_options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > m_options.SkipFilesLargerThan;
var metadatachanged = !m_options.SkipMetadata && (metahashandsize.Size != oldMetasize || metahashandsize.Hash != oldMetahash);
if ((oldId < 0 || m_options.DisableFiletimeCheck || timestampChanged || filesizeChanged || metadatachanged) && !tooLargeFile)
{
m_result.AddVerboseMessage("Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", path, oldId <= 0, timestampChanged, filesizeChanged, metadatachanged, lastwrite, oldModified);
m_result.OpenedFiles++;
long filesize = 0;
var hint = m_options.GetCompressionHintFromFilename(path);
var oldHash = oldId < 0 ? null : m_database.GetFileHash(oldId);
using (var blocklisthashes = new Library.Utility.FileBackedStringList())
using (var hashcollector = new Library.Utility.FileBackedStringList())
{
using (var fs = new Blockprocessor(snapshot.OpenRead(path), m_blockbuffer))
{
try { m_result.OperationProgressUpdater.StartFile(path, fs.Length); }
catch (Exception ex) { m_result.AddWarning(string.Format("Failed to read file length for file {0}", path), ex); }
int blocklistoffset = 0;
m_filehasher.Initialize();
var offset = 0;
var remaining = fs.Readblock();
do
{
var size = Math.Min(m_blocksize, remaining);
m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset);
var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size);
if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length)
{
var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
blocklisthashes.Add(blkey);
AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
blocklistoffset = 0;
}
Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length);
blocklistoffset += blockkey.Length;
var key = Convert.ToBase64String(blockkey);
AddBlockToOutput(backend, key, m_blockbuffer, offset, size, hint, false);
hashcollector.Add(key);
filesize += size;
m_result.OperationProgressUpdater.UpdateFileProgress(filesize);
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
return false;
remaining -= size;
offset += size;
if (remaining == 0)
{
offset = 0;
remaining = fs.Readblock();
}
} while (remaining > 0);
//If all fits in a single block, don't bother with blocklists
if (hashcollector.Count > 1)
{
var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset));
blocklisthashes.Add(blkeyfinal);
AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true);
}
}
m_result.SizeOfOpenedFiles += filesize;
m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0);
var filekey = Convert.ToBase64String(m_filehasher.Hash);
if (oldHash != filekey)
{
if (oldHash == null)
m_result.AddVerboseMessage("New file {0}", path);
else
m_result.AddVerboseMessage("File has changed {0}", path);
if (oldId < 0)
{
m_result.AddedFiles++;
m_result.SizeOfAddedFiles += filesize;
if (m_options.Dryrun)
m_result.AddDryrunMessage(string.Format("Would add new file {0}, size {1}", path, Library.Utility.Utility.FormatSizeString(filesize)));
}
else
{
m_result.ModifiedFiles++;
m_result.SizeOfModifiedFiles += filesize;
if (m_options.Dryrun)
m_result.AddDryrunMessage(string.Format("Would add changed file {0}, size {1}", path, Library.Utility.Utility.FormatSizeString(filesize)));
}
AddFileToOutput(backend, path, filesize, lastwrite, metahashandsize, hashcollector, filekey, blocklisthashes);
changed = true;
}
else if (metadatachanged)
{
m_result.AddVerboseMessage("File has only metadata changes {0}", path);
AddFileToOutput(backend, path, filesize, lastwrite, metahashandsize, hashcollector, filekey, blocklisthashes);
changed = true;
}
else
{
// When we write the file to output, update the last modified time
oldModified = lastwrite;
m_result.AddVerboseMessage("File has not changed {0}", path);
}
}
}
else
{
if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || snapshot.GetFileSize(path) < m_options.SkipFilesLargerThan)
m_result.AddVerboseMessage("Skipped checking file, because timestamp was not updated {0}", path);
else
m_result.AddVerboseMessage("Skipped checking file, because the size exceeds limit {0}", path);
}
if (!changed)
AddUnmodifiedFile(oldId, lastwrite);
m_result.SizeOfExaminedFiles += filestatsize;
if (filestatsize != 0)
m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.ExaminedFiles, m_result.SizeOfExaminedFiles);
}
catch (Exception ex)
{
m_result.AddWarning(string.Format("Failed to process path: {0}", path), ex);
m_result.FilesWithError++;
}
return true;
}
public FilterHandler(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter filter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, ILogWriter logWriter)
{
m_snapshot = snapshot;
m_attributeFilter = attributeFilter;
m_sourcefilter = sourcefilter;
m_filter = filter;
m_symlinkPolicy = symlinkPolicy;
m_hardlinkPolicy = hardlinkPolicy;
m_logWriter = logWriter;
m_hardlinkmap = new Dictionary<string, string>();
}
public PathCollector(Snapshots.ISnapshotService snapshot, Options.SymlinkStrategy symlinkHandling, FileAttributes attributeMask, Utility.FilenameFilter filter, CommunicationStatistics stat)
{
m_symlinkHandling = symlinkHandling;
m_filter = filter;
m_attributeMask = attributeMask;
m_snapshot = snapshot;
m_stat = stat;
}
public IEnumerable<string> EnumerateFilesAndFolders(Snapshots.ISnapshotService snapshot)
{
foreach(var s in snapshot.EnumerateFilesAndFolders(this.AttributeFilter))
{
while (m_mixinqueue.Count > 0)
yield return m_mixinqueue.Dequeue();
yield return s;
}
while (m_mixinqueue.Count > 0)
yield return m_mixinqueue.Dequeue();
}
public IEnumerable<string> Mixin(IEnumerable<string> list, Snapshots.ISnapshotService snapshot)
{
foreach(var s in list.Where(x => {
var fa = FileAttributes.Normal;
try { fa = m_snapshot.GetAttributes(x); }
catch { }
return AttributeFilter(null, x, fa);
}))
{
while (m_mixinqueue.Count > 0)
yield return m_mixinqueue.Dequeue();
yield return s;
}
while (m_mixinqueue.Count > 0)
yield return m_mixinqueue.Dequeue();
}
