/// <summary>
/// It checks all the documents in path and updates them. It is responsible for deleting document,
/// adding documents and modifying documents.
/// </summary>
/// <param name="path">The path which contains the documents.</param>
/// <param name="invt">An object of the inverter Class.</param>
public static void Crawler(String path, Inverter invt)
{
Dictionary <String, Document> files = invt.Files;
pathfiles = new List <string>();
List <String> pathfile = new List <string>();
Streamer.Invt = invt;
//run a for loop on this for all types in format
foreach (string type in invt.Formats[""])
{
pathfile = Directory.EnumerateFiles(path, "*." + type, SearchOption.AllDirectories).Select(Path.GetFullPath).ToList <string>();
pathfiles = pathfiles.Union <string>(pathfile).ToList <string>();
}
foreach (String item in files.Keys.ToArray <String>())
{
if (!pathfiles.Contains(item))
{
Streamer.RemoveFile(files[item]);
}
else if (new FileInfo(item).LastWriteTime.CompareTo(files[item].LastModified) != 0)
{
try
{
Streamer.ModifyFile(files[item]);
}
catch (Exception ex)
{
Inverter.LogMovement("!!!!!!!!!Error withdrawing from " + files[item].Address + " while modifying. ERROR MESSAGE:" + ex.Message);
}
}
}
//adding file
//creating a document object and pass into streamer.adddfile
foreach (String location in pathfiles)
{
if (!files.ContainsKey(location))
{
try
{
Document newdoc = GetDocumentFrom(location);
Streamer.AddFileFrom(newdoc);
}
catch (Exception ex)
{
Inverter.LogMovement("!!!!!!!!!Error withdrawing from " + location + ". ERROR MESSAGE:" + ex.Message);
}
}
}
}
/// <summary>
/// Adds the document to document table.
/// </summary>
/// <param name="doc">The document to be added.</param>
public void AddToDocTable(Document doc)
{
var filter = Builders <Document> .Filter.Eq("_id", doc.Address);
var answer = documents.Find(filter).FirstOrDefault();
try {
if (answer == null)
{
documents.InsertOne(doc);
}
} catch (Exception ex) {
Inverter.LogMovement("!!!!!!!!!!ERROR adding to table " + ex.Message);
}
}