public static void Index(BibtexRecord record)
{
WriteLine(5, "Indexing " + record.GetKey());
Tri index;
string doi;
StringArrayList textualDataArray;
textualDataArray = GetTextualData(record.GetURI());
index = Index(textualDataArray);
DatabaseStoreStatic.SetSearchData(record.DbId(), index.ToString());
doi = ExtractDOI(textualDataArray);
record.SetCustomDataField("indexData", index);
if (doi != null)
{
WriteLine(10, "Setting DOI: {0}", doi);
record.SetField(BibtexRecord.BibtexFieldName.DOI, doi);
WriteLine(10, "Finished setting DOI field");
}
else
{
WriteLine(10, "No DOI record found");
// Use MD5 as the BibtexKey for this record
record.SetKey((string)record.GetCustomDataField("bibliographer_last_md5"));
}
}
public Gtk.TreeIter GetIter(BibtexRecord record)
{
Gtk.TreeIter iter;
BibtexRecord rec;
if (GetIterFirst(out iter))
{
rec = GetValue(iter, 0) as BibtexRecord;
if (rec.GetHashCode() == record.GetHashCode())
{
return(iter);
}
while (IterNext(ref iter))
{
rec = GetValue(iter, 0) as BibtexRecord;
if (rec.GetHashCode() == record.GetHashCode())
{
return(iter);
}
}
}
return(iter);
}
public void UpdateTreeStore(BibtexRecord btRecord)
{
WriteLine(10, "Updating TreeStore");
UpdateAuthors(btRecord);
UpdateJournals(btRecord);
UpdateYears(btRecord);
}
public void IndexerThread()
{
WriteLine(5, "Indexer thread started");
try {
do
{
Enter(indexerQueue);
while (indexerQueue.Count > 0)
{
BibtexRecord record = (BibtexRecord)indexerQueue.Dequeue();
Exit(indexerQueue);
try {
WriteLine(10, "IndexerQueue: Indexing record - " + record.GetURI());
FileIndexer.Index(record);
} catch (Exception e) {
WriteLine(1, "Unknown exception caught with indexer");
WriteLine(1, e.Message);
WriteLine(1, e.StackTrace);
}
Enter(indexerQueue);
}
Exit(indexerQueue);
Thread.Sleep(100);
} while (true);
} catch (ThreadAbortException) {
WriteLine(5, "Indexer thread terminated");
}
}
public void ThumbGenThread()
{
WriteLine(5, "ThumbGen thread started");
try {
do
{
Enter(thumbGenQueue);
while (thumbGenQueue.Count > 0)
{
BibtexRecord record = (BibtexRecord)thumbGenQueue.Dequeue();
Exit(thumbGenQueue);
try {
WriteLine(10, "ThumbGenThread: Check if record has a thumbnail - " + record);
if (!ThumbGen.getThumbnail(record))
{
WriteLine(10, "Thumbnail does not exist");
}
} catch (Exception e) {
WriteLine(1, "Unknown exception caught with thumbGen");
WriteLine(1, e.Message);
WriteLine(1, e.StackTrace);
}
Enter(thumbGenQueue);
}
Exit(thumbGenQueue);
Thread.Sleep(100);
} while (true);
} catch (ThreadAbortException) {
WriteLine(5, "ThumbGen thread terminated");
}
}
public void DoiQueryThread()
{
WriteLine(5, "DoiQuery thread started");
try {
do
{
Enter(doiQueryQueue);
while (doiQueryQueue.Count > 0)
{
BibtexRecord record = (BibtexRecord)doiQueryQueue.Dequeue();
Exit(doiQueryQueue);
if (record != null)
{
try {
if (record.HasDOI())
{
LookupRecordData.LookupDOIData(record);
}
} catch (Exception e) {
WriteLine(1, "Unknown exception caught with doiQuery");
WriteLine(1, e.Message);
WriteLine(1, e.StackTrace);
}
}
Enter(doiQueryQueue);
}
Exit(doiQueryQueue);
Thread.Sleep(100);
} while (true);
} catch (ThreadAbortException) {
WriteLine(5, "DoiQuery thread terminated");
}
}
public void SubscribeRecordForDOILookup(BibtexRecord record)
{
Enter(doiQueryQueue);
if (!doiQueryQueue.Contains(record))
{
doiQueryQueue.Enqueue(record);
}
Exit(doiQueryQueue);
}
public void SubscribeAlteredRecord(BibtexRecord record)
{
Enter(alterationMonitorQueue);
if (!alterationMonitorQueue.Contains(record))
{
alterationMonitorQueue.Enqueue(record);
}
Exit(alterationMonitorQueue);
}
public static bool getThumbnail(BibtexRecord record)
{
//Pixbuf smallThumbnail, largeThumbnail;
Uri uri;
GLib.IFile file;
GLib.FileInfo fileInfo;
string pixbufPath;
if ((!record.HasCustomDataField("smallThumbnail")) || (!record.HasCustomDataField("largeThumbnail")))
{
// No thumbnail - so generate it and index
if (record.HasCustomDataField("bibliographer_last_uri") && record.HasCustomDataField("bibliographer_last_md5"))
{
// Check if Nautilus has generated a thumbnail for this file
uri = new Uri(record.GetURI());
file = GLib.FileFactory.NewForUri(uri);
fileInfo = file.QueryInfo("*", GLib.FileQueryInfoFlags.None, null);
pixbufPath = fileInfo.GetAttributeByteString("thumbnail::path");
if (pixbufPath != "" && pixbufPath != null)
{
using (Pixbuf lg = new Pixbuf(pixbufPath))
{
record.SetCustomDataField("largeThumbnail", lg);
using (Pixbuf sm = ((Pixbuf)lg.Clone()).ScaleSimple(20, 20, InterpType.Bilinear)) {
record.SetCustomDataField("smallThumbnail", sm);
}
}
//largeThumbnail = new Pixbuf(pixbufPath);
//smallThumbnail = ((Pixbuf)largeThumbnail.Clone()).ScaleSimple(20, 20, InterpType.Bilinear);
//record.SetCustomDataField("smallThumbnail", smallThumbnail);
//record.SetCustomDataField("largeThumbnail", largeThumbnail);
// Thumbnails exist and have now been stored in the BibtexRecord instance
return(true);
}
}
}
else
{
// Thumbnails exist and are stored in the BibtexRecord instance
return(true);
}
// Catch all other conditions. No existing thumbnail.
return(false);
}
void UpdateAuthors(BibtexRecord btRecord)
{
string author;
StringArrayList bibrecAuthors;
bibrecAuthors = btRecord.GetAuthors();
StringArrayList bibrecsAuthors;
bibrecsAuthors = bibtexRecords.GetAuthors();
// Interate through updated record's authors
for (int ii = 0; ii < bibrecAuthors.Count; ii++)
{
author = bibrecAuthors[ii];
// Insert Author
StringArrayList treeAuthors = GetAuthors();
if ((!treeAuthors.Contains(author)) && (author != null) && (author != ""))
{
WriteLine(10, "Inserting Author {0} into Filter List", author);
AppendValues(iterAuth, author);
}
}
// Iterate through tree and remove authors that are not in the bibtexRecords
Gtk.TreeIter iter;
var treeAuth = new StringArrayList();
IterChildren(out iter, iterAuth);
int n = IterNChildren(iterAuth);
for (int i = 0; i < n; i++)
{
string author1 = (string)GetValue(iter, 0);
if (bibrecsAuthors.Contains(author1) && !treeAuth.Contains(author1))
{
IterNext(ref iter);
treeAuth.Add(author1);
}
else
{
WriteLine(10, "Removing Author {0} from Side Pane", author1);
Remove(ref iter);
}
}
}
void UpdateYears(BibtexRecord btRecord)
{
string year;
year = btRecord.GetYear();
StringArrayList bibrecsYear;
bibrecsYear = bibtexRecords.GetYears();
// Insert Year
StringArrayList treeYear = GetYears();
if ((!treeYear.Contains(year)) && (year != null) && (year != ""))
{
WriteLine(10, "Inserting Year into Filter List");
AppendValues(iterYear, year);
}
// Iterate through tree and remove years that are not in the bibtexRecords
Gtk.TreeIter iter;
var treeYr = new StringArrayList();
IterChildren(out iter, iterYear);
int n = IterNChildren(iterYear);
for (int i = 0; i < n; i++)
{
string year1 = (string)GetValue(iter, 0);
if (bibrecsYear.Contains(year1) && !treeYr.Contains(year1))
{
IterNext(ref iter);
treeYr.Add(year1);
}
else
{
WriteLine(10, "Removing Year {0} from Side Pane", year1);
Remove(ref iter);
}
}
}
void UpdateJournals(BibtexRecord btRecord)
{
string journal;
journal = btRecord.GetJournal();
StringArrayList bibrecsJournals;
bibrecsJournals = bibtexRecords.GetJournals();
// Insert Journal
StringArrayList treeJournals = GetJournals();
if ((!treeJournals.Contains(journal)) && (journal != null) && (journal != ""))
{
WriteLine(10, "Inserting Journal into Filter List");
AppendValues(iterJourn, journal);
}
// Iterate through tree and remove journals that are not in the bibtexRecords
var treeJourn = new StringArrayList();
Gtk.TreeIter iter;
IterChildren(out iter, iterJourn);
int n = IterNChildren(iterJourn);
for (int i = 0; i < n; i++)
{
string journal1 = (string)GetValue(iter, 0);
if (bibrecsJournals.Contains(journal1) && !treeJourn.Contains(journal1))
{
IterNext(ref iter);
treeJourn.Add(journal1);
}
else
{
WriteLine(10, "Removing Journal {0} from Side Pane", journal1);
Remove(ref iter);
}
}
}
public static void LookupDOIData(BibtexRecord record)
{
string doi = record.GetField(BibtexRecord.BibtexFieldName.DOI);
string url = "http://api.crossref.org/works/" + doi;
WriteLine(5, "Looking up data for {0} from {1}", doi, url);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
if (request != null)
{
request.Timeout = 30000;
}
try {
System.IO.StreamReader reader;
string jsonString;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
System.IO.Stream resStream = response.GetResponseStream();
reader = new System.IO.StreamReader(resStream);
jsonString = reader.ReadToEnd();
try {
JsonDOIWork jsonObj = JsonConvert.DeserializeObject <JsonDOIWork> (jsonString);
JsonDOIWorkMessageDateTime date;
string authorString, bibtexKeyString;
authorString = "";
bibtexKeyString = "";
foreach (JsonDOIWorkMessageAuthor author in jsonObj.Message.Author)
{
if (authorString == "")
{
authorString = author.Family + ", " + author.Given;
bibtexKeyString = author.Family;
}
else
{
authorString = authorString + " and " + author.Family + ", " + author.Given;
}
}
if (jsonObj.Message.Type == "journal-article")
{
WriteLine(10, "Setting field values for article: " + doi);
record.RecordType = "article";
record.SetField(BibtexRecord.BibtexFieldName.Journal, jsonObj.Message.ContainerTitle [0]);
record.SetField(BibtexRecord.BibtexFieldName.Volume, jsonObj.Message.Volume);
record.SetField(BibtexRecord.BibtexFieldName.Number, jsonObj.Message.Issue);
record.SetField(BibtexRecord.BibtexFieldName.Pages, jsonObj.Message.Page);
}
record.SetField(BibtexRecord.BibtexFieldName.Author, authorString);
record.SetField(BibtexRecord.BibtexFieldName.Title, jsonObj.Message.Title [0]);
date = jsonObj.Message.Issued;
record.SetField(BibtexRecord.BibtexFieldName.Year, date.DateParts [0, 0].ToString());
record.SetField(BibtexRecord.BibtexFieldName.Month, date.DateParts [0, 1].ToString());
if (bibtexKeyString.Length > 0)
{
bibtexKeyString = bibtexKeyString + "_" + date.DateParts [0, 0].ToString() + "_" + (string)record.GetCustomDataField("bibliographer_last_md5");
record.SetKey(bibtexKeyString);
}
}
catch (System.Exception e) {
WriteLine(1, "Unhandled exception when parsing JSON string from {0}", doi, url);
WriteLine(1, e.Message);
}
}
catch (WebException e) {
WriteLine(1, e.Message);
}
catch (System.Exception e) {
WriteLine(1, "Unhandled exception when looking up {0} from {1}", doi, url);
WriteLine(1, e.Message);
}
}
public bool Altered(BibtexRecord record)
{
string uriString = record.GetURI();
if (record.GetFileSize() == 0 || record.GetFileMTime() == 0 || record.GetFileMD5Sum() == "")
{
lastCheck = DateTime.MinValue;
}
if (string.IsNullOrEmpty(uriString))
{
return(false);
}
Uri uri;
TimeSpan checkInterval;
uri = new Uri(uriString);
//TODO: specify checking frequencies in settings
switch (uri.Scheme)
{
case "http":
// default update interval for HTTP: 30 mins
checkInterval = new TimeSpan(0, 30, 0);
break;
case "file":
// default update interval for local files: 1 minute
checkInterval = new TimeSpan(0, 1, 0);
break;
default:
// default update interval for anything else: 5 minutes
checkInterval = new TimeSpan(0, 5, 0);
break;
}
//checkInterval = new TimeSpan (0, 0, 1);
if (DateTime.Now.Subtract(checkInterval).CompareTo(lastCheck) < 0)
{
WriteLine(10, "Not enough time has passed to check this record");
return(false);
}
if (!uri.IsFile)
{
WriteLine(10, "URI \"" + uriString + "\" does not seem to exist...");
return(false);
}
// initiating a check - recording time
lastCheck = DateTime.Now;
long size = record.GetFileSize();
ulong mtime = record.GetFileMTime();
string md5 = record.GetFileMD5Sum();
long newSize = 0;
long intSize;
GLib.IFile file;
GLib.FileInfo fileInfo;
intSize = 0;
ulong newMtime = 0;
try {
WriteLine(10, "URI \"" + uriString + "\" has the following characteristics:");
if (md5 == null)
{
md5 = "";
}
else
{
WriteLine(10, "\t* md5: " + md5);
}
WriteLine(10, "\t* Scheme: " + uri.Scheme);
file = GLib.FileFactory.NewForUri(uri);
fileInfo = file.QueryInfo("*", GLib.FileQueryInfoFlags.NofollowSymlinks, null);
if (fileInfo.Size != 0)
{
WriteLine(10, "\t* Size: " + fileInfo.Size);
newSize = fileInfo.Size;
intSize = fileInfo.Size;
}
if (fileInfo.GetAttributeULong("time::changed") != 0)
{
WriteLine(10, "\t* ctime: " + fileInfo.GetAttributeULong("time::changed"));
}
if (fileInfo.GetAttributeULong("time::modified") != 0)
{
WriteLine(10, "\t* mtime: " + fileInfo.GetAttributeULong("time::modified"));
newMtime = fileInfo.GetAttributeULong("time::modified");
}
fileInfo.Dispose();
string result;
ulong data_size;
data_size = 0;
result = GLib.ContentType.Guess(uri.ToString(), out byte data, data_size, out bool uncertain);
if (!string.IsNullOrEmpty(result))
{
WriteLine(10, "\t* Mime type: " + result);
}
} catch (Exception e) {
WriteLine(1, e.Message);
WriteLine(1, "\t*** Whoops! Caught an exception!");
}
if ((size != newSize) || (mtime != newMtime) || (md5 == ""))
{
WriteLine(10, "\t* Recalculating MD5...");
string newMd5;
using (MD5 md5er = MD5.Create()) {
using (Stream strea = File.OpenRead(uri.LocalPath)) {
byte [] hash = md5er.ComputeHash(strea);
newMd5 = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
}
}
if (size != newSize || mtime != newMtime || newMd5 != md5)
{
record.SetFileAttrs(uriString, newSize, newMtime, newMd5);
}
if (record.HasCustomDataField("indexData"))
{
record.RemoveCustomDataField("indexData");
}
if (record.HasCustomDataField("largeThumbnail"))
{
record.RemoveCustomDataField("largeThumbnail");
}
if (record.HasCustomDataField("smallThumbnail"))
{
record.RemoveCustomDataField("smallThumbnail");
}
return(true);
}
// TODO:
// This is not the correct place for this... Instead, an item with a URI,
// but without indexData should only attempt to be re-indexed on opening the file again.
//if (record.GetCustomDataField ("indexData") == null) {
// URI, but null index data. Force a re-index by returning true
// return true;
//}
return(false);
}
public void AlterationMonitorThread()
{
WriteLine(5, "Alteration monitor thread started");
try {
do
{
Enter(alterationMonitorQueue);
while (alterationMonitorQueue.Count > 0)
{
BibtexRecord record = (BibtexRecord)alterationMonitorQueue.Dequeue();
Exit(alterationMonitorQueue);
// FIXME: do the alteration monitoring stuff
// FIXME: if continuous monitoring is enabled, then
// the entry should be requeued
if (Altered(record))
{
WriteLine(10, "AlterationThread: processing - " + record.GetURI());
Enter(indexerQueue);
if (!indexerQueue.Contains(record))
{
indexerQueue.Enqueue(record);
}
Exit(indexerQueue);
Enter(thumbGenQueue);
if (!thumbGenQueue.Contains(record))
{
thumbGenQueue.Enqueue(record);
}
Exit(thumbGenQueue);
}
else if (
record.HasURI() &&
string.IsNullOrEmpty(record.GetKey()) &&
string.IsNullOrEmpty(record.RecordType) &&
!record.HasCustomDataField("largeThumbnail") &&
!record.HasCustomDataField("indexData")
)
{
Enter(indexerQueue);
if (!indexerQueue.Contains(record))
{
indexerQueue.Enqueue(record);
}
Exit(indexerQueue);
Enter(thumbGenQueue);
if (!thumbGenQueue.Contains(record))
{
thumbGenQueue.Enqueue(record);
}
Exit(thumbGenQueue);
}
Enter(alterationMonitorQueue);
// enqueue record if it has a uri
if (record.HasURI() && !alterationMonitorQueue.Contains(record))
{
alterationMonitorQueue.Enqueue(record);
}
Thread.Sleep(100);
}
Exit(alterationMonitorQueue);
Thread.Sleep(100);
} while (true);
} catch (ThreadAbortException) {
WriteLine(1, "Alteration monitor thread terminated");
}
}
