static ArrayList RemapUris(LuceneQueryingDriver driver, ArrayList uris)
{
// We only need to remap URIs in the file system backend
if (driver.IndexName != "FileSystemIndex")
{
return(uris);
}
FileAttributesStore fa_store = new FileAttributesStore(new FileAttributesStore_Mixed(Path.Combine(PathFinder.IndexDir, "FileSystemIndex"), driver.Fingerprint));
for (int i = 0; i < uris.Count; i++)
{
Uri uri = (Uri)uris [i];
string path = uri.LocalPath;
Beagrep.Daemon.FileAttributes attr = fa_store.Read(path);
if (attr == null)
{
Console.WriteLine("No file attribute info for {0}", uri);
continue;
}
Uri internal_uri = new Uri("uid:" + GuidFu.ToShortString(attr.UniqueId) + uri.Fragment);
uris [i] = internal_uri;
}
return(uris);
}
public bool HasNextIndexable()
{
if (metadata != null)
{
if (metadata.MoveNext())
{
return(true);
}
else
{
metadata = null;
FileAttributesStore.AttachLastWriteTime((string)metafiles.Current, DateTime.UtcNow);
}
}
while (metadata == null)
{
if (metafiles == null)
{
metafiles = DirectoryWalker.GetItems(nautilus_dir, is_xml_file).GetEnumerator();
}
if (!metafiles.MoveNext())
{
return(false);
}
string file = (string)metafiles.Current;
if (FileAttributesStore.IsUpToDate(file))
{
continue;
}
metadata = NautilusTools.GetMetadata((string)metafiles.Current).GetEnumerator();
if (metadata.MoveNext())
{
return(true);
}
else
{
metadata = null;
FileAttributesStore.AttachLastWriteTime(file, DateTime.UtcNow);
}
}
return(false); // Makes the compiler happy
}
public OperaIndexer(OperaQueryable queryable, FileAttributesStore store, string root_dir)
{
this.attribute_store = store;
this.queryable = queryable;
this.cache_dirs = new ArrayList ();
// Try to find all cache dirs
foreach (string dir in DirectoryWalker.GetDirectories (root_dir)) {
foreach (string file in DirectoryWalker.GetItems
(dir, new DirectoryWalker.FileFilter (IsCacheFile))) {
Inotify.Subscribe (dir, OnInotify, Inotify.EventType.MovedTo | Inotify.EventType.CloseWrite);
cache_dirs.Add (dir);
}
}
}
public OperaIndexer(OperaQueryable queryable, FileAttributesStore store, string root_dir)
{
this.attribute_store = store;
this.queryable = queryable;
this.cache_dirs = new ArrayList();
// Try to find all cache dirs
foreach (string dir in DirectoryWalker.GetDirectories(root_dir))
{
foreach (string file in DirectoryWalker.GetItems
(dir, new DirectoryWalker.FileFilter(IsCacheFile)))
{
Inotify.Subscribe(dir, OnInotify, Inotify.EventType.MovedTo | Inotify.EventType.CloseWrite);
cache_dirs.Add(dir);
}
}
}
// Remapping hack from DumpIndex
static Uri RemapUri(LuceneQueryingDriver driver, Uri uri)
{
// We only need to remap URIs in the file system backend
if (driver.IndexName != "FileSystemIndex")
{
return(uri);
}
FileAttributesStore fa_store = new FileAttributesStore(new FileAttributesStore_Mixed(Path.Combine(PathFinder.IndexDir, "FileSystemIndex"), driver.Fingerprint));
string path = uri.LocalPath;
Beagle.Daemon.FileAttributes attr = fa_store.Read(path);
if (attr == null)
{
Console.WriteLine("No file attribute info for {0}", uri);
return(uri);
}
return(new Uri("uid:" + GuidFu.ToShortString(attr.UniqueId) + uri.Fragment));
}
/////////////////////////////////////////////////
private void OnInotifyEvent(Inotify.Watch watch,
string path,
string subitem,
string srcpath,
Inotify.EventType type)
{
if (subitem == "")
{
return;
}
if (Path.GetExtension(subitem) != ".xml")
{
return;
}
// We're only handling MovedTo events here.
string file = Path.Combine(path, subitem);
DateTime last_checked = DateTime.MinValue;
FileAttributes attr;
attr = FileAttributesStore.Read(file);
if (attr != null)
{
last_checked = attr.LastWriteTime;
}
foreach (NautilusTools.NautilusMetadata nm in NautilusTools.GetMetadata(file, last_checked))
{
Indexable indexable = GetIndexable(nm);
Scheduler.Task task;
task = this.target_queryable.NewAddTask(indexable);
task.Priority = Scheduler.Priority.Immediate;
ThisScheduler.Add(task);
}
}
private void StartWorker()
{
if (!Directory.Exists(konq_dir))
{
GLib.Timeout.Add(60000, new GLib.TimeoutHandler(CheckForExistence));
return;
}
if (Inotify.Enabled)
{
Inotify.EventType mask = Inotify.EventType.CloseWrite
| Inotify.EventType.MovedTo;
Inotify.Subscribe(konq_dir, OnInotifyEvent, mask);
}
else
{
FileSystemWatcher fsw = new FileSystemWatcher();
fsw.Path = konq_dir;
fsw.Filter = bookmark_file;
fsw.Changed += new FileSystemEventHandler(OnChanged);
fsw.Created += new FileSystemEventHandler(OnChanged);
fsw.Renamed += new RenamedEventHandler(OnChanged);
fsw.EnableRaisingEvents = true;
}
if (File.Exists(bookmark_file))
{
if (!FileAttributesStore.IsUpToDate(bookmark_file))
{
Index();
}
else
{
ScanBookmarkInitial();
}
}
}
/////////////////////////////////////////////////
private Indexable FileToIndexable(string path, bool crawling)
{
//Logger.Log.Debug ("KonqQ: Trying to index " + path);
Stream stream;
try {
stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
} catch (FileNotFoundException) {
// that was fast - lost the file
return(null);
}
bool gzip_cached = false;
if (KonqHistoryUtil.IsGZipCache(stream))
{
gzip_cached = true;
}
if (gzip_cached)
{
stream = new GZipInputStream(stream);
}
using (StreamReader reader = new StreamReader(stream, latin_encoding)) {
string url = null;
string creation_date = null;
string mimetype = null;
string charset = null;
string revision = null;
bool is_ok = KonqHistoryUtil.ShouldIndex(reader,
out url,
out creation_date,
out mimetype,
out charset,
out revision);
if (!is_ok || url == String.Empty)
{
//Logger.Log.Debug ("KonqQ: Skipping non-html file " + path + " of type=" + mimetype);
// finding out if a cache file should be indexed is expensive
// so, soon after we run the test, write lastwritetime attribute
FileAttributesStore.AttachLastWriteTime(path, DateTime.UtcNow);
return(null); // we wont index bad files and non-html files
}
//Logger.Log.Debug ("KonqQ: Indexing {0} = {1},{2},{3},{4},{5}", path, url, creation_date, mimetype, charset, revision);
Uri uri = new Uri(url, true);
if (uri.Scheme == Uri.UriSchemeHttps)
{
Logger.Log.Error("Indexing secure https:// URIs is not secure!");
return(null);
}
Indexable indexable = new Indexable(uri);
indexable.HitType = "WebHistory";
indexable.MimeType = KonqHistoryUtil.KonqCacheMimeType;
// store www.beaglewiki.org as www beagle org, till inpath: query is implemented
indexable.AddProperty(Property.NewUnstored("fixme:urltoken", StringFu.UrlFuzzyDivide(url)));
// hint for the filter about the charset
indexable.AddProperty(Property.NewUnsearched(StringFu.UnindexedNamespace + "charset", charset));
indexable.AddProperty(Property.NewUnsearched(StringFu.UnindexedNamespace + "gzipped", gzip_cached));
indexable.AddProperty(Property.NewUnsearched(StringFu.UnindexedNamespace + "revision", revision));
indexable.FlushBufferCache = crawling;
DateTime date = DateTimeUtil.UnixToDateTimeUtc(0);
date = date.AddSeconds(Int64.Parse(creation_date));
indexable.Timestamp = date;
indexable.ContentUri = UriFu.PathToFileUri(path);
return(indexable);
}
}
public UidManager(FileAttributesStore fa_store,
LuceneQueryingDriver driver)
{
this.fa_store = fa_store;
this.name_resolver = (LuceneNameResolver)driver;
}