public Fat12FileSystem(Stream stream, BiosParameterBlock bpb) : base(stream, bpb)
{
ClusterMaps = new ClusterMap[bpb.NumberOfFATs];
for (int i = 0; i < bpb.NumberOfFATs; i++)
{
ClusterMaps[i] = new ClusterMap(this, i);
}
}
public Fat32FileSystem(Stream stream, BiosParameterBlock bpb) : base(stream, bpb)
{
ClusterMaps = new ClusterMap[bpb.NumberOfFATs];
for (int i = 0; i < bpb.NumberOfFATs; i++)
{
ClusterMaps[i] = new ClusterMap(this, i);
}
if (bpb is Fat32BiosParameterBlock fat32bpb)
{
using var reader = new BinaryReader(stream);
stream.Position = fat32bpb.BytesPerLogicalSector * fat32bpb.FsInfo;
_fsInfo = FsInfo.Parse(reader);
}
}
private List <UIElement> drawClusters(ClusterMap clusterMap)
{
var clusters = clusterMap.getClusterMap();
var rectangles = new List <UIElement>();
int id = 10000000;
foreach (Cluster cluster in clusters)
{
rectangles.Add(addRect(cluster.left, cluster.top, cluster.height, cluster.width, Colors.Blue, 1, id));
//Out.put(cluster.left + "," + cluster.top + "," + cluster.height + "," + cluster.width);
++id;
drawExits(cluster);
}
return(rectangles);
}
public void writeImage()
{
foreach (HeaderObject element in VolumeBootRecord)
{
element.write(imageFile);
}
clusterMap = new ClusterMap(files.ToArray());
directoryTable = new DirectoryTable(files.ToArray());
clusterMap.write(imageFile);
directoryTable.write(imageFile, 0x2600);
imageFile.Seek((2880 * 512) - 1, SeekOrigin.Begin);
imageFile.WriteByte(0x00);
foreach (FatFile file in files)
{
file.writeFileToImage(imageFile);
}
}
public static void MoveCenters()
{
var isOptimizedMap = new bool[Clusters];
for (var i = 0; i < Clusters; i++)
{
var clusterTotal = ClusterMap.Count(c => c.Value == i);
if (clusterTotal > 0)
{
DenseVector vectorSum = null;
for (var m = 0; m < MatrixContainer.VMatrix.ColumnCount; m++)
{
if (ClusterMap[m] == i)
{
if (vectorSum == null)
{
vectorSum = (DenseVector)MatrixContainer.VMatrix.Column(m);
}
else
{
vectorSum += (DenseVector)MatrixContainer.VMatrix.Column(m);
}
}
}
var newCenter = (vectorSum / clusterTotal).ToArray();
if (Centers[i] != null)
{
isOptimizedMap[i] = Distance.Cosine(Centers[i], newCenter) < OptimizationVarianceThreshold;
Console.WriteLine("Cluster #" + i + ": " + Distance.Cosine(Centers[i], newCenter));
}
Centers[i] = newCenter;
}
}
IsOptimized = isOptimizedMap.All(v => v == true);
}
public void MoveCenters()
{
var isOptimizedMap = new ConcurrentDictionary <int, bool>();
Enumerable.Range(0, Clusters).AsParallel().ForAll(i =>
{
var clusterTotal = ClusterMap.Count(c => c.Value == i);
if (clusterTotal > 0)
{
DenseVector vectorSum = null;
for (var m = 0; m < LSA.MatrixContainer.VMatrix.ColumnCount; m++)
{
if (ClusterMap[m] == i)
{
if (vectorSum == null)
{
vectorSum = (DenseVector)LSA.MatrixContainer.VMatrix.Column(m);
}
else
{
vectorSum += (DenseVector)LSA.MatrixContainer.VMatrix.Column(m);
}
}
}
var newCenter = (vectorSum / clusterTotal).ToArray();
if (Centers[i] != null)
{
isOptimizedMap[i] = Distance.Cosine(Centers[i], newCenter) < OptimizationVarianceThreshold;
}
Centers[i] = newCenter;
}
});
IsOptimized = isOptimizedMap.All(v => v.Value == true);
}
internal ClusterMapEnumerator(ClusterMap map)
{
_map = map;
}
private void button2_Click(object sender, EventArgs e)
{
DiscUtils.VolumeManager volMgr = new VolumeManager();
volMgr.AddDisk(VirtualDisk.OpenDisk(textBox1.Text, FileAccess.Read));
Stream partitionStream = null;
if (!string.IsNullOrEmpty(VolumeId))
{
partitionStream = volMgr.GetVolume(VolumeId).Open();
}
else if (Partition >= 0)
{
partitionStream = volMgr.GetPhysicalVolumes()[Partition].Open();
}
else
{
partitionStream = volMgr.GetLogicalVolumes()[0].Open();
}
SparseStream cacheStream = SparseStream.FromStream(partitionStream, Ownership.None);
cacheStream = new BlockCacheStream(cacheStream, Ownership.None);
fs = new NtfsFileSystem(cacheStream);
fs.NtfsOptions.HideHiddenFiles = false;
fs.NtfsOptions.HideSystemFiles = false;
fs.NtfsOptions.HideMetafiles = false;
ClusterMap clusterMap = null;
try
{
clusterMap = fs.BuildClusterMap();
}
catch (IOException ex)
{
// DebugConsole.LogException(ex, "Trying to build a clustermap of " + textBox1.Text);
MessageBox.Show(ex.Message, "Exception");
// return;
}
// string[] files = null;
// string[] sysfiles = fs.GetFileSystemEntries(@"\");
// string[] files = fs.GetFiles(@"\", "*.*", SearchOption.AllDirectories);
// fs.Dump(Console.Out, "");
//QueuedBackgroundWorker.QueueWorkItem(QueuedBackgroundWorker.m_Queue, null,
// (args) => { return fs.GetFiles(@"\", "*.*", SearchOption.AllDirectories); },
// (args) => { if (args.Result != null && args.Result is string[]) ProcessFiles((string[])args.Result); }
//);
BitArray ba = new BitArray((int)fs.TotalClusters);
for (int i = 0; i < fs.TotalClusters; i++)
{
if (clusterMap.ClusterToPaths(i).Length > 0)
{
ba.Set(i, true);
}
}
ClusterForm cf = new ClusterForm();
cf.Icon = Icon;
cf.Text = Text + " - Cluster Map";
//cf.clusterBitmaps1.GradientStart = Color.WhiteSmoke;
//cf.clusterBitmaps1.GradientEnd = Color.Black;
//cf.clusterBitmaps1.GradientSteps = 8;
cf.clusterBitmaps1.Clusters = ba;
cf.Show();
}
public Cluster(Random generator, int jobId, int k = 2, int maxIteration = 200)
{
Clusters = k;
MaxIterations = maxIteration;
JobId = jobId;
LSA.GetMatrixContainer(JobId);
var clusterCentersDocIndex = new HashSet <int>();
while (clusterCentersDocIndex.Count <= Clusters)
{
clusterCentersDocIndex.Add(generator.Next(LSA.MatrixContainer.VMatrix.ColumnCount));
}
for (var i = 0; i < Clusters; i++)
{
Centers[i] = LSA.MatrixContainer.VMatrix.Column(clusterCentersDocIndex.ElementAt(i)).ToArray();
}
var counter = 0;
var start = DateTime.Now;
while (!IsOptimized && ++counter < MaxIterations)
{
CalcDistancesAndAssign();
MoveCenters();
}
Debug.WriteLine($"----{Clusters}----Total Iterations: {counter}");
for (var i = 0; i < Clusters; i++)
{
DocsPerCluster[i] = ClusterMap.Count(c => c.Value == i);
}
Debug.WriteLine($"----{Clusters}----Total Cluster Calc: {DateTime.Now.Subtract(start).TotalMilliseconds} Milliseconds");
var calcSiStart = DateTime.Now;
// Silhouette (clustering)
// https://en.wikipedia.org/wiki/Silhouette_(clustering)
ClusterMap.AsParallel().ForAll(kvp =>
{
// Get aI
var currentCluster = kvp.Value;
var targetCluster = ClusterMap.Where(c => c.Value == currentCluster).ToList();
var distanceTotal = new List <float>();
for (var i = 0; i < targetCluster.Count; i++)
{
var distance = LSA.MatrixContainer.DistanceMap[kvp.Key, targetCluster[i].Key];
if (!float.IsInfinity(distance) && !float.IsNaN(distance))
{
distanceTotal.Add(distance);
}
}
var aI = distanceTotal.Sum() / DocsPerCluster[currentCluster];
// Get bI
var otherClusterDistances = new List <float>();
for (var i = 0; i < Clusters; i++)
{
if (i == currentCluster)
{
continue;
}
var otherTargetCluster = ClusterMap.Where(c => c.Value == i).ToList();
var distanceOtherTotal = new List <float>();
for (var m = 0; m < otherTargetCluster.Count; m++)
{
var distanceOther = LSA.MatrixContainer.DistanceMap[kvp.Key, otherTargetCluster[m].Key];
if (!float.IsInfinity(distanceOther) && !float.IsNaN(distanceOther))
{
distanceOtherTotal.Add(distanceOther);
}
}
var otherDistanceAverage = distanceOtherTotal.Sum() / DocsPerCluster[i];
if (!float.IsInfinity(otherDistanceAverage) && !float.IsNaN(otherDistanceAverage))
{
otherClusterDistances.Add(otherDistanceAverage);
}
}
var bI = otherClusterDistances.Min();
var sI = (bI - aI) / Math.Max(aI, bI);
if (!float.IsInfinity(sI) && !float.IsNaN(sI))
{
DocumentSi.AddOrUpdate(kvp.Key, sI, (i, f) => sI);
ClusterSi.Add(Tuple.Create(currentCluster, sI));
}
});
// Calc Cluster Si Averages
for (var m = 0; m < Clusters; m++)
{
if (ClusterSi.Any(c => c.Item1 == m))
{
ClusterSiAverages[m] = ClusterSi
.Where(c => c.Item1 == m)
.Select(c => c.Item2)
.Average();
}
}
// Calculate Si Averages
GlobalSi = DocumentSi.Average(kvp => kvp.Value);
GlobalClusterSiAverage = ClusterSiAverages.Average((kvp) => kvp.Value);
Debug.WriteLine($"****{Clusters}***** GlobalSi: {GlobalSi}");
Debug.WriteLine($"****{Clusters}***** GlobalClusterSiAverage: {GlobalClusterSiAverage}");
Debug.WriteLine($"----{Clusters}---- Total Cluster SI Calc: {DateTime.Now.Subtract(calcSiStart).TotalMilliseconds} Milliseconds");
}
public void Save(SvdEntities context, ClusterCalculation clusterCalculationEntity)
{
var binaryFormatter = new BinaryFormatter();
var jobDocs = context.JobDocuments.Where(jd => jd.JobId == JobId).ToLookup(jd => jd.OrdinalIndex);
var jobTerms = context.JobTerms.Where(jd => jd.JobId == JobId).ToLookup(jt => jt.Term.Value);
var clusterEntities = new Dictionary <int, Engine.Cluster>();
clusterCalculationEntity.ClusterCount = Clusters;
clusterCalculationEntity.GlobalSi = GlobalSi;
clusterCalculationEntity.ClusterSi = GlobalClusterSiAverage;
// Update Cluster Calculation
context.SaveChanges();
Enumerable.Range(0, Clusters).ToList().ForEach(cluster =>
{
using (var memoryStreamCenterVector = new MemoryStream())
{
binaryFormatter.Serialize(memoryStreamCenterVector, Centers[cluster]);
memoryStreamCenterVector.Position = 0;
clusterEntities.Add(cluster, new Engine.Cluster()
{
JobId = JobId,
ClusterCalculationId = clusterCalculationEntity.Id,
Si = ClusterSiAverages[cluster],
CenterVectorSerialized = memoryStreamCenterVector.ToArray()
});
}
});
// Insert Clusters
context.BulkInsert(clusterEntities.Select(kvp => kvp.Value));
var clusterJobDocumentEntities = new ConcurrentBag <ClusterJobDocument>();
var clusterJobTermEntities = new ConcurrentBag <ClusterJobTerm>();
clusterEntities.AsParallel().ForAll(clusterEntity =>
{
using (var memoryStreamCenterVector = new MemoryStream())
{
var termDistanceMap = new Dictionary <string, float>();
var centerVector = Centers[clusterEntity.Key];
foreach (var kvp in ClusterMap.Where(kvp => kvp.Value == clusterEntity.Key))
{
var docIndex = kvp.Key;
var jobDocument = jobDocs[docIndex];
if (jobDocument != null)
{
clusterJobDocumentEntities.Add(new ClusterJobDocument()
{
ClusterCalculationId = clusterCalculationEntity.Id,
ClusterId = clusterEntity.Value.Id,
JobId = JobId,
Si = DocumentSi.ContainsKey(docIndex) ? DocumentSi[docIndex] : 0,
JobDocumentId = jobDocument.First().Id
});
}
}
for (var i = 0; i < LSA.MatrixContainer.UMatrix.RowCount; i++)
{
termDistanceMap[LSA.MatrixContainer.Terms[i]] = Distance.Cosine(centerVector, LSA.MatrixContainer.UMatrix.Row(i).ToArray());
}
foreach (var term in termDistanceMap.OrderBy(t => t.Value).Take(20))
{
var jobTermLookup = jobTerms[term.Key];
if (jobTermLookup != null)
{
clusterJobTermEntities.Add(new ClusterJobTerm()
{
ClusterCalculationId = clusterCalculationEntity.Id,
ClusterId = clusterEntity.Value.Id,
JobId = JobId,
JobTermId = jobTermLookup.First().Id,
DistanceToClusterCenter = term.Value
});
}
}
}
});
// Insert Cluster Documents & Terms
context.BulkInsert(clusterJobTermEntities);
context.BulkInsert(clusterJobDocumentEntities);
SetCalculationStatus(context, clusterCalculationEntity, Contracts.ClusterCalculationStatus.Completed);
}
