public void LoadEntries(DataFile file, IndexEntry indexEntry)
{
var list = new List <RootEntry>();
var blteEntry = new BinaryReader(DataFile.LoadBLTEEntry(indexEntry, file.readStream));
while (blteEntry.BaseStream.Position < blteEntry.BaseStream.Length)
{
var entries = new RootEntry[blteEntry.ReadInt32()];
uint contentFlags = blteEntry.ReadUInt32();
var localeFlags = (LocaleMask)blteEntry.ReadUInt32();
int fileDataIndex = 0;
for (var i = 0; i < entries.Length; i++)
{
entries[i].LocaleFlags = localeFlags;
entries[i].ContentFlags = contentFlags;
entries[i].FileDataId = fileDataIndex + blteEntry.ReadInt32();
fileDataIndex = entries[i].FileDataId + 1;
}
for (var i = 0; i < entries.Length; i++)
{
entries[i].MD5 = blteEntry.ReadBytes(16);
entries[i].Hash = blteEntry.ReadUInt64();
}
list.AddRange(entries);
}
entries = list.ToLookup(re => re.Hash);
entriesByFileDataId = list.ToLookup(re => re.FileDataId);
}
// ReSharper disable InconsistentNaming
/// <summary>
/// This does not create a full RootEntry - it does setup SetInMemoryFields().
/// This test structure.
/// Z:\
/// Z:\d2a (size 11)
/// Z:\d2b\
/// Z:\d2b\d3a\
/// Z:\d2b\d3a\d4a (size 17)
/// Z:\d2c (size 0)
/// </summary>
public static RootEntry NewTestRootEntry(out DirEntry de2a, out DirEntry de2b, out DirEntry de2c, out DirEntry de3a, out DirEntry de4a)
{
var re1 = new RootEntry {
Path = @"Z:\"
};
de2a = new DirEntry {
Path = "d2a", Size = 11
};
de2b = new DirEntry(true)
{
Path = "d2b"
};
de2c = new DirEntry {
Path = "d2c"
};
re1.Children.Add(de2c);
re1.Children.Add(de2a);
re1.Children.Add(de2b);
de3a = new DirEntry(true)
{
Path = "d3a"
};
de2b.Children.Add(de3a);
de4a = new DirEntry {
Path = "d4a", Size = 17
};
de3a.Children.Add(de4a);
return(re1);
}
public void CanFindDuplicates()
{
var duplication = new TestDuplication(_logger, _configuration, _applicationDiagnostics);
var rootEntry = new RootEntry();
rootEntry.PopulateRoot($"{FolderName}\\");
var rootEntries = new List <RootEntry> {
rootEntry
};
duplication.ApplyMd5Checksum(rootEntries);
// all 7 Files are partial hashed.
Assert.That(duplication.DuplicationStatistcs().PartialHashes, Is.EqualTo(7));
// all 7 files are full hashed because every file appears to have a duplicate at partial hash size.
Assert.That(duplication.DuplicationStatistcs().FullHashes, Is.EqualTo(7));
// need a new Duplication class() between ApplyMd5Checksum and GetDupePairs for valid operation.
duplication = new TestDuplication(_logger, _configuration, _applicationDiagnostics);
var dupes = duplication.GetDupePairs(rootEntries);
Assert.That(dupes[0].Value.Count, Is.EqualTo(2)); // testset1 has 2 dupes
Assert.That(dupes[1].Value.Count, Is.EqualTo(3)); // testset3 has 3 dupes
Console.WriteLine();
Console.WriteLine($"Dumping sets of duplicates count {dupes.Count}");
foreach (var dupe in dupes)
{
Console.WriteLine($"set of dupes to {dupe.Key.Path}");
foreach (var adupe in dupe.Value)
{
Console.WriteLine($"dupe set file {adupe.FullPath}");
}
}
}
protected override void Dispose(bool disposing)
{
if (disposing && RootEntry != null)
{
RootEntry.Dispose();
}
}
private void WriteNow()
{
try
{
using (Stream stream = File.Open(this.Filename, FileMode.Create))
using (XmlWriter writer = XmlWriter.Create(stream, new XmlWriterSettings()
{
Indent = true,
IndentChars = "\t"
}))
{
RootEntry root = new RootEntry()
{
Collections = this.data.Select(sc => new SettingCollectionEntry()
{
Name = sc.Key,
Settings = sc.Value.Select(s => new SettingEntry()
{
Name = s.Key,
Value = s.Value
}).ToList()
}).ToList(),
ValueTypeNames = this.valueTypes.Select(type => type.AssemblyQualifiedName).ToArray()
};
this.GetSerializer().Serialize(writer, root);
}
}
catch (Exception ex)
{
Debug.WriteLine("Writing XML-File \"{0}\" failed:\n{1}", this.Filename, ex);
}
}
private static void FindDupes()
{
var rootEntries = RootEntry.LoadCurrentDirCache();
var duplication = Container.Resolve <Duplication>();
duplication.FindDuplicates(rootEntries);
}
public void writeLZ4NHStream(RootEntry root, Stream s)
{
using (var xFs = new Lz4CompressionStream(s, 1 << 18, Lz4Mode.HighCompression))
{
root.Write(xFs);
}
}
public void TestParameters()
{
RootEntry root = OutputParser.parse(@"a|color=#ff0");
Assert.AreEqual(1, root.children.Count);
Assert.AreEqual("#ff0", root.children[0].color);
}
public void ApplyMd5Checksum_CheckDupesAndCompleteFullHash_DoesItEnsureAllPartialDupesAreFullHashed_Exercise()
{
var rootEntries = RootEntry.LoadCurrentDirCache();
var d = new Duplication(_logger, _configuration, _applicationDiagnostics);
d.ApplyMd5Checksum(rootEntries);
}
public void SetFullPath_OnRootDirectory_SetsAllFullPaths()
{
var re = new RootEntry {
Path = @"C:\"
};
var fe1 = new DirEntry {
Path = "fe1"
};
var de2 = new DirEntry(true)
{
Path = "de2"
};
var fe3 = new DirEntry {
Path = "fe3"
};
re.Children.Add(fe1);
re.Children.Add(de2);
de2.Children.Add(fe3);
re.SetInMemoryFields();
Assert.That(re.FullPath, Is.EqualTo(@"C:\"));
Assert.That(fe1.FullPath, Is.Null); // Is.EqualTo(@"C:\fe1")); FullPath only set on directories to save memory.
Assert.That(de2.FullPath, Is.EqualTo(@"C:\de2"));
Assert.That(fe3.FullPath, Is.Null); //Is.EqualTo(@"C:\de2\fe3"));
}
public long DoGetSearchHitsRFPairList(RootEntry root, int repeatCount, FindOptions findOptions)
{
var sw = new Stopwatch();
sw.Start();
var rootEntries = new List <RootEntry> {
root
};
var list = new List <PairDirEntry>();
for (var i = 0; i < repeatCount; i++)
{
// var totalFound = 0L;
findOptions.VisitorFunc = (p, d) =>
{
// ++totalFound;
list.Add(new PairDirEntry(p, d));
return(true);
};
findOptions.Find(rootEntries);
}
sw.Stop();
return(sw.ElapsedMilliseconds);
}
private static TreeNode BuildRootNode(RootEntry rootEntry)
{
var rootTreeNode = NewTreeNode(rootEntry);
SetDummyChildNode(rootTreeNode, rootEntry);
return(rootTreeNode);
}
protected void InitRootWithFile()
{
var nowUtc = new DateTime(2011, 12, 01, 17, 15, 13, DateTimeKind.Utc);
_rootEntry = new RootEntry {
Path = @"T:\",
VolumeName = "TestVolume",
DirEntryCount = 1,
FileEntryCount = 0,
DriveLetterHint = @"Z",
AvailSpace = 754321,
TotalSpace = 654321,
ScanStartUTC = nowUtc,
ScanEndUTC = nowUtc.AddMilliseconds(34),
DefaultFileName = "TestRootEntry.cde",
ActualFileName = @".\TestRootEntry.cde",
Description = "Test Root Entry Description",
};
_dirEntry = new DirEntry
{
Path = @"Test",
Size = 531,
Modified = new DateTime(2010, 11, 02, 18, 16, 12, DateTimeKind.Unspecified),
};
_rootEntry.Children.Add(_dirEntry);
_rootEntry.SetInMemoryFields();
_pairDirEntry = new PairDirEntry(_rootEntry, _dirEntry);
_rootList.Add(_rootEntry);
}
public static void CreateMd5OnCache()
{
var logger = Container.Resolve <ILogger>();
var diagnostics = Container.Resolve <IApplicationDiagnostics>();
logger.LogInfo("Memory pre-catload: {0}", diagnostics.GetMemoryAllocated().FormatAsBytes());
var rootEntries = RootEntry.LoadCurrentDirCache();
logger.LogInfo("Memory post-catload: {0}", diagnostics.GetMemoryAllocated().FormatAsBytes());
var duplication = Container.Resolve <Duplication>();
var sw = new Stopwatch();
sw.Start();
duplication.ApplyMd5Checksum(rootEntries);
foreach (var rootEntry in rootEntries)
{
logger.LogDebug("Saving {0}", rootEntry.DefaultFileName);
rootEntry.SaveRootEntry();
}
sw.Stop();
var ts = sw.Elapsed;
var elapsedTime = $"{ts.Hours:00}:{ts.Minutes:00}:{ts.Seconds:00}.{ts.Milliseconds/10:00}";
Console.WriteLine($"Hash took : {elapsedTime}");
}
//public static bool AreTreesSame(RootEntry re1, RootEntry re2)
//{
// LastMessage = "";
// var differenceFound = false;
// var e1 = new DirEntryEnumerator(re1);
// var e2 = new DirEntryEnumerator(re2);
// while (e1.MoveNext() && e2.MoveNext())
// {
// LastMessage = string.Format("{0} not same as {1}", e1.Current.Name, e2.Current.Name);
// if (e1.Current.Name != e2.Current.Name)
// {
// differenceFound = true;
// break;
// }
// Console.WriteLine(LastMessage);
// }
// var bothTreesComplete = e1.MoveNext() == false && e2.MoveNext() == false;
// return !differenceFound && bothTreesComplete;
//}
public static bool SameTree(this RootEntry re1, RootEntry re2)
{
LastMessage = "";
if (re1.Path != re2.Path)
{
LastMessage = $"RootPath {re1.Path} not same as {re2.Path}";
return(false);
}
var differenceFound = false;
var e1 = new DirEntryEnumerator(re1);
var e2 = new DirEntryEnumerator(re2);
while (e1.MoveNext() && e2.MoveNext())
{
LastMessage = $"{e1.Current.Path} not same as {e2.Current.Path}";
if (e1.Current.Path != e2.Current.Path)
{
differenceFound = true;
break;
}
Console.WriteLine(LastMessage);
}
var bothTreesComplete = e1.MoveNext() == false && e2.MoveNext() == false;
return(!differenceFound && bothTreesComplete);
}
protected void InitializeRootDescriptor(RootEntry rootEntry)
{
RootDescriptor rd = new RootDescriptor(rootEntry)
{
State = RootDescriptorState.AwaitingDeviceDescription
};
lock (_cpData.SyncObj)
SetRootDescriptor(rootEntry, rd);
try
{
LinkData preferredLink = rootEntry.PreferredLink;
HttpWebRequest request = CreateHttpGetRequest(new Uri(preferredLink.DescriptionLocation), preferredLink.Endpoint.EndPointIPAddress);
DescriptionRequestState state = new DescriptionRequestState(rd, request);
lock (_cpData.SyncObj)
_pendingRequests.Add(state);
IAsyncResult result = request.BeginGetResponse(OnDeviceDescriptionReceived, state);
NetworkHelper.AddTimeout(request, result, PENDING_REQUEST_TIMEOUT * 1000);
}
catch (Exception) // Don't log messages at this low protocol level
{
lock (_cpData.SyncObj)
rd.State = RootDescriptorState.Erroneous;
}
}
public void Callback_GoToDirectoryRoot_On_Different_RootEntry_Sets_New_Root()
{
var alternateRootEntry = new RootEntry {
Path = "alternate"
};
var testRootTreeNode = new TreeNode("Moo")
{
Tag = alternateRootEntry
};
_mockForm.Stub(x => x.DirectoryTreeViewNodes)
.Repeat.Times(1) // enforcing repeat makes this a fragile test ?
.Return(testRootTreeNode);
// verify does set DirectoryTreeViewNodes
_mockForm.Stub(x => x.DirectoryTreeViewNodes = Arg <TreeNode> .Is.Anything)
.Repeat.Times(1);
// ACTIVATE public method.... due to call back nature the action() call below is the actual operation.
// with CatalogListView our activated item is a RootEntry.
_sutPresenter.CatalogListViewItemActivate();
var action = GetPresenterAction(_mockCatalogListViewHelper);
action(_rootEntry);
_mockForm.VerifyAllExpectations();
}
private void LoaderForm_Shown(object sender, System.EventArgs e)
{
lblProgressMessage.Text = string.Empty;
Application.DoEvents(); // Make sure controls render before we do something.
var cacheFiles = RootEntry.GetCacheFileList(_cdeList);
var totalFiles = cacheFiles.Count();
var fileCounter = 0;
barLoading.Step = totalFiles;
barLoading.Maximum = totalFiles;
// Not doing a background worker.
_rootEntries = cacheFiles.Select(s =>
{
_timeIt.Start(s);
var re = RootEntry.LoadDirCache(s);
_timeIt.Stop();
++fileCounter;
lblProgressMessage.Text = $"Loading catalog {fileCounter} of {totalFiles}";
barLoading.Value = fileCounter;
Application.DoEvents(); // Get label to update.
return(re);
}).ToList();
Close();
}
public void writeGzipStream(RootEntry root, Stream s)
{
using (var xFs = new GZipStream(s, CompressionMode.Compress, true))
{
root.Write(xFs);
}
}
public void writeStream(RootEntry root, Stream s)
{
using (s)
{
root.Write(s);
}
}
public void writeLZ4Stream(RootEntry root, Stream s)
{
using (var xFs = new LZ4Stream(s, CompressionMode.Compress))
{
root.Write(xFs);
}
}
public void writeLZ4HStream(RootEntry root, Stream s)
{
using (var xFs = new LZ4Stream(s, LZ4StreamMode.Compress, LZ4StreamFlags.HighCompression))
{
root.Write(xFs);
}
}
public DirEntryEnumerator(RootEntry rootEntry)
{
_rootEntries = new List <RootEntry> {
rootEntry
};
Reset();
}
public void LoadEntries(DataFile file, IndexEntry indexEntry)
{
var list = new List <RootEntry>();
var blteEntry = new BinaryReader(DataFile.LoadBLTEEntry(indexEntry, file.readStream));
while (blteEntry.BaseStream.Position < blteEntry.BaseStream.Length)
{
var entries = new RootEntry[blteEntry.ReadInt32()];
blteEntry.BaseStream.Position += 4;
var locales = (Locales)blteEntry.ReadUInt32();
blteEntry.BaseStream.Position += (entries.Length << 2);
for (var i = 0; i < entries.Length; i++)
{
list.Add(new RootEntry
{
MD5 = blteEntry.ReadBytes(16),
Hash = blteEntry.ReadUInt64(),
Locales = locales
});
}
}
Entries = list.ToLookup(re => re.Hash);
}
public long DoGetSearchHitsRFCount(RootEntry root, int repeatCount, FindOptions options)
{
var sw = new Stopwatch();
sw.Start();
var rootEntries = new List <RootEntry> {
root
};
for (var i = 0; i < repeatCount; i++)
{
var totalFound = 0L;
var findOptions = new FindOptions
{
Pattern = "willxxlliw",
RegexMode = false,
IncludePath = false,
IncludeFiles = true,
IncludeFolders = true,
LimitResultCount = int.MaxValue,
VisitorFunc = (p, d) =>
{
++totalFound;
return(true);
},
};
findOptions.Find(rootEntries);
}
sw.Stop();
return(sw.ElapsedMilliseconds);
}
public void AddEntry(string path, CASResult file)
{
ulong hash = new Jenkins96().ComputeHash(path);
bool found = false;
// check to see if we're overwriting an existing entry
foreach (var root in Chunks)
{
if (!root.LocaleFlags.HasFlag(locale) && root != GlobalRoot) // skip incorrect locales and non-global roots
{
continue;
}
var index = root.Entries.FindIndex(x => x.Hash == hash);
if (index >= 0)
{
RootEntry entry = new RootEntry()
{
MD5 = file.DataHash,
Hash = hash,
Path = path,
FileDataId = root.Entries[index].FileDataId,
FileDataIdOffset = root.Entries[index].FileDataIdOffset
};
root.Entries[index] = entry; // update
found = true;
// max id check just to be safe
if (root == GlobalRoot)
{
maxId = Math.Max(entry.FileDataId, maxId); // update max id
}
CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, file.Hash));
}
}
// must be a new file, add it to the global root
if (!found)
{
RootEntry entry = new RootEntry()
{
MD5 = file.DataHash,
FileDataIdOffset = 0,
Hash = hash,
Path = path
};
var cache = CASContainer.Settings.Cache.Entries.FirstOrDefault(x => x.Path == path);
if (cache?.Path != path) // get cache id
{
entry.FileDataId = Math.Max(maxId + 1, minimumId); // calculate the Id
}
GlobalRoot.Entries.Add(entry); // add new
maxId = Math.Max(entry.FileDataId, maxId); // Update max id
CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, file.Hash));
}
}
public void TestSingleLine()
{
RootEntry root = OutputParser.parse("hello");
Assert.AreEqual(root.children.Count, 1);
Assert.AreEqual(root.menu.Count, 0);
Assert.AreEqual(root.children[0].text, "hello");
}
private IndexingTask createRemovalTask(long contentId)
{
var fileName = _util.GetFileName("file");
var rootEntry = new RootEntry <Metadata>(() => new Metadata(SequentialId.None));
var fileEntry = (FileEntry <Metadata>)rootEntry.Add(EntryType.File, fileName, new Metadata(contentId));
return(new IndexingTask(IndexingAction.RemoveContent, fileEntry, CancellationToken.None));
}
public void Scan(string name)
{
//TODO: wire up command properly
var re = new RootEntry();
re.PopulateRoot(name);
Response = String.Format("Filecount: {0}", re.FileCount);
}
private TreeNode SetNewDirectoryRoot(RootEntry newRoot)
{
var newRootNode = BuildRootNode(newRoot);
_clientForm.DirectoryTreeViewNodes = newRootNode;
_clientForm.DirectoryListViewHelper.InitSort();
return(newRootNode);
}
protected RootEntry MergeOrMoveRootEntry(RootEntry pendingRootEntry, string rootDeviceUUID)
{
lock (_cpData.SyncObj)
{
_pendingDeviceEntries.Remove(pendingRootEntry);
RootEntry targetEntry;
if (_cpData.DeviceEntries.TryGetValue(rootDeviceUUID, out targetEntry))
{
targetEntry.MergeRootEntry(pendingRootEntry);
return targetEntry;
}
targetEntry = pendingRootEntry; // From here on, the entry is not pending any more, so we use the variable targetEntry for clearness
targetEntry.RootDeviceUUID = rootDeviceUUID;
_cpData.DeviceEntries[rootDeviceUUID] = targetEntry;
return targetEntry;
}
}
private void OnSSDPRootDeviceAdded(RootEntry rootEntry)
{
InitializeRootDescriptor(rootEntry);
}
protected void InitializeRootDescriptor(RootEntry rootEntry)
{
RootDescriptor rd = new RootDescriptor(rootEntry)
{
State = RootDescriptorState.AwaitingDeviceDescription
};
lock (_cpData.SyncObj)
SetRootDescriptor(rootEntry, rd);
try
{
LinkData preferredLink = rootEntry.PreferredLink;
HttpWebRequest request = CreateHttpGetRequest(new Uri(preferredLink.DescriptionLocation), preferredLink.Endpoint.EndPointIPAddress);
DescriptionRequestState state = new DescriptionRequestState(rd, request);
lock (_cpData.SyncObj)
_pendingRequests.Add(state);
IAsyncResult result = request.BeginGetResponse(OnDeviceDescriptionReceived, state);
NetworkHelper.AddTimeout(request, result, PENDING_REQUEST_TIMEOUT * 1000);
}
catch (Exception) // Don't log messages at this low protocol level
{
lock (_cpData.SyncObj)
rd.State = RootDescriptorState.Erroneous;
}
}
private void OnSSDPRootDeviceRemoved(RootEntry rootEntry)
{
RootDescriptor rd = GetRootDescriptor(rootEntry);
if (rd == null)
return;
lock (_cpData.SyncObj)
InvalidateDescriptor(rd);
InvokeRootDeviceRemoved(rd);
}
protected void InvokeDeviceConfigurationChanged(RootEntry rootEntry)
{
try
{
DeviceConfigurationChangedDlgt dlgt = DeviceConfigurationChanged;
if (dlgt != null)
dlgt(rootEntry);
}
catch (Exception e)
{
UPnPConfiguration.LOGGER.Warn("SSDPClientController: Error invoking DeviceConfigurationChanged delegate", e);
}
}
protected void InvokeServiceAdded(RootEntry rootEntry, DeviceEntry deviceEntry, string serviceTypeVersion_URN)
{
try
{
ServiceAddedDlgt dlgt = ServiceAdded;
if (dlgt != null)
dlgt(rootEntry, deviceEntry, serviceTypeVersion_URN);
}
catch (Exception e)
{
UPnPConfiguration.LOGGER.Warn("SSDPClientController: Error invoking ServiceAdded delegate", e);
}
}
protected void RemoveRootEntry(RootEntry rootEntry)
{
lock (_cpData.SyncObj)
{
_cpData.DeviceEntries.Remove(rootEntry.RootDeviceUUID);
_pendingDeviceEntries.Remove(rootEntry);
}
}
protected RootEntry GetOrCreateRootEntry(string deviceUUID, string descriptionLocation, UPnPVersion upnpVersion, string osVersion,
string productVersion, DateTime expirationTime, EndpointConfiguration endpoint, HTTPVersion httpVersion, int searchPort, out bool wasAdded)
{
// Because the order of the UDP advertisement packets isn't guaranteed (and even not really specified by the UPnP specification),
// in the general case it is not possible to find the correct root entry for each advertisement message.
// - We cannot search by root device UUID because the upnp:rootdevice message might not be the first message, so before that message, we don't know the root device ID and
// thus we cannot use the root device id as unique key to find the root entry
// - We cannot use the device description because for multi-homed devices, more than one device description can belong to the same root device
//
// Assume the message arrive in an order so that device A over network interface N1 is announced first. Second, device B over network interface N2 is announced.
// In that case, we cannot judge if those two devices belong to the same root device or not.
//
// To face that situation, we first add all advertised devices to _pendingDeviceEntries. When a upnp:rootdevice message is received,
// we either simply move the root entry from _pendingDeviceEntries into _cpData.DeviceEntries or we merge the pending entry with an already existing
// entry in _cpData.DeviceEntries. At that time the merge is possible because we then have the root device id for both root entries.
lock (_cpData.SyncObj)
{
RootEntry result = GetRootEntryByContainedDeviceUUID(deviceUUID) ?? GetRootEntryByDescriptionLocation(descriptionLocation);
if (result != null)
{
result.ExpirationTime = expirationTime;
wasAdded = false;
}
else
{
result = new RootEntry(_cpData.SyncObj, upnpVersion, osVersion, productVersion, expirationTime);
_pendingDeviceEntries.Add(result);
wasAdded = true;
}
return result;
}
}
private void OnSSDPDeviceRebooted(RootEntry rootEntry, bool configurationChanged)
{
RootDescriptor rd = GetRootDescriptor(rootEntry);
if (rd == null)
return;
if (configurationChanged)
HandleDeviceConfigurationChanged(rd);
else
InvokeDeviceRebooted(rd);
}
private void OnSSDPDeviceConfigurationChanged(RootEntry rootEntry)
{
RootDescriptor rd = GetRootDescriptor(rootEntry);
if (rd == null)
return;
HandleDeviceConfigurationChanged(rd);
}
internal RootDescriptor(RootEntry rootEntry)
{
_rootEntry = rootEntry;
}
protected RootDescriptor GetRootDescriptor(RootEntry rootEntry)
{
object rdObj;
if (!rootEntry.ClientProperties.TryGetValue(KEY_ROOT_DESCRIPTOR, out rdObj))
return null;
return (RootDescriptor) rdObj;
}
protected void SetRootDescriptor(RootEntry rootEntry, RootDescriptor rootDescriptor)
{
rootEntry.ClientProperties[KEY_ROOT_DESCRIPTOR] = rootDescriptor;
}
