static ArrayList RemapUris(LuceneQueryingDriver driver, ArrayList uris)
{
// We only need to remap URIs in the file system backend
if (driver.IndexName != "FileSystemIndex")
{
return(uris);
}
FileAttributesStore fa_store = new FileAttributesStore(new FileAttributesStore_Mixed(Path.Combine(PathFinder.IndexDir, "FileSystemIndex"), driver.Fingerprint));
for (int i = 0; i < uris.Count; i++)
{
Uri uri = (Uri)uris [i];
string path = uri.LocalPath;
Beagrep.Daemon.FileAttributes attr = fa_store.Read(path);
if (attr == null)
{
Console.WriteLine("No file attribute info for {0}", uri);
continue;
}
Uri internal_uri = new Uri("uid:" + GuidFu.ToShortString(attr.UniqueId) + uri.Fragment);
uris [i] = internal_uri;
}
return(uris);
}
// We assume that uri is uid: URI
public void Remove(Uri uri)
{
driver.Remove(uri);
Guid uid = GuidFu.FromUri(uri);
name_index.Remove(uid);
}
// We assume that
// (a) old_uri is a uid: URI
// (b) new_uri is a file: URI
// (c) new_uri's filename is meaningful
public void Rename(Uri old_uri, Uri new_uri)
{
Guid uid = GuidFu.FromUri(old_uri);
string name = FileSystem.GetFileName(new_uri.LocalPath);
name_index.Add(uid, name);
renamed_uris.Add(old_uri);
renamed_uris.Add(new_uri);
}
// We assume that
// (a) indexable.Uri is a uid: URI
// (b) indexable.ContentUri is a file: URI
// (c) indexable.ContentUri's filename is meaningful (i.e. isn't a temporary file)
public void Add(Indexable indexable)
{
driver.Add(indexable);
Guid uid = GuidFu.FromUri(indexable.Uri);
string name = FileSystem.GetFileName(indexable.ContentUri.LocalPath);
name_index.Add(uid, name);
}
private Indexable GetNewXmpIndexable(ref Indexable indexable,
string path,
Guid id,
DirectoryModel parent)
{
// In non-crawl mode, check if a corresponding xmp file is present and not already scheduled and index it.
// If file.xmp and file are rapidly written/updated (in that order), this does the right thing.
// If file and file.xmp are rapidly written/updated (in that order), either
// - file.xmp is present during FileToIndexable(file): in which case xmp properties are
// added to file; and when file.xmp is indexed, it will replace the xmp properties
// - file.xmp is not present during FileToIndexable(file): when the xmp file is later indexed
// it will add the xmp properties
// since the uid file will still be in the uid-cache, correct uid will be used for xmp prop-change indexable
string possible_xmp_file_path = string.Concat(path, ".xmp");
if (!File.Exists(possible_xmp_file_path))
{
return(null);
}
Guid xmp_id = queryable.RegisterFile(parent, (Path.GetFileName(possible_xmp_file_path)));
if (xmp_id == Guid.Empty)
{
return(null);
}
XmpFile xmp_file = null;
try {
xmp_file = new XmpFile(possible_xmp_file_path);
} catch {
uid_manager.ForgetNewId(possible_xmp_file_path);
return(null);
}
// FIXME: Should also delete previous xmp properties!
foreach (Property p in xmp_file.Properties)
{
p.IsMutable = true;
indexable.AddProperty(p);
}
xmp_file.Close();
// Also need to save some local states for PostAddHook,
// namely, path to the xmp file, path to basefile and generated uid
indexable.LocalState ["XmpFilePath"] = possible_xmp_file_path;
indexable.LocalState ["BaseFilePath"] = path;
indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString(xmp_id);
if (Debug)
{
Log.Debug("Adding properties from {0}({2}) to {1}({3})", possible_xmp_file_path, path, GuidFu.ToShortString(xmp_id), GuidFu.ToShortString(id));
}
return(null);
}
private NameInfo DocumentToNameInfo(Document doc)
{
NameInfo info;
info = new NameInfo();
string str;
str = doc.Get("Uri");
info.Id = GuidFu.FromUriString(str);
bool have_name = false;
bool have_parent_id = false;
bool have_is_dir = false;
foreach (Field f in doc.Fields())
{
Property prop;
prop = GetPropertyFromDocument(f, doc, false);
if (prop == null)
{
continue;
}
switch (prop.Key)
{
case Property.ExactFilenamePropKey:
info.Name = prop.Value;
have_name = true;
break;
case Property.ParentDirUriPropKey:
info.ParentId = GuidFu.FromUriString(prop.Value);
have_parent_id = true;
break;
case Property.IsDirectoryPropKey:
info.IsDirectory = (prop.Value == "true");
have_is_dir = true;
break;
}
if (have_name && have_parent_id && have_is_dir)
{
break;
}
}
return(info);
}
protected void InitFileAttributesStore(string name, string external_fingerprint)
{
string storage_path = Path.Combine(PathFinder.IndexDir, name);
string fingerprint_file = Path.Combine(storage_path, "fingerprint");
string internal_fingerprint;
if (!Directory.Exists(storage_path))
{
Directory.CreateDirectory(storage_path);
internal_fingerprint = GuidFu.ToShortString(Guid.NewGuid());
StreamWriter writer = new StreamWriter(fingerprint_file);
writer.WriteLine(internal_fingerprint);
writer.Close();
}
else
{
StreamReader reader = new StreamReader(fingerprint_file);
internal_fingerprint = reader.ReadLine();
reader.Close();
}
string fingerprint;
if (external_fingerprint != null)
{
fingerprint = internal_fingerprint + "-" + external_fingerprint;
}
else
{
fingerprint = internal_fingerprint;
}
IFileAttributesStore ifa_store;
if (ExtendedAttribute.Supported)
{
ifa_store = new FileAttributesStore_ExtendedAttribute(fingerprint);
}
else
{
ifa_store = new FileAttributesStore_Sqlite(storage_path, fingerprint);
}
fa_store = new FileAttributesStore(ifa_store);
}
private FileAttributes GetFromReader(SqliteDataReader reader)
{
FileAttributes attr = new FileAttributes();
attr.UniqueId = GuidFu.FromShortString(reader.GetString(0));
attr.Path = System.IO.Path.Combine(reader.GetString(1), reader.GetString(2));
attr.LastWriteTime = StringFu.StringToDateTime(reader.GetString(3));
attr.LastAttrTime = StringFu.StringToDateTime(reader.GetString(4));
attr.FilterName = reader.GetString(5);
attr.FilterVersion = int.Parse(reader.GetString(6));
if (attr.FilterName == "")
{
attr.FilterName = null;
}
return(attr);
}
public bool Write(FileAttributes attr)
{
if (Disable)
{
return(false);
}
try {
if (ExtendedAttribute.OldExists(attr.Path, "Fingerprint"))
{
DropObsoleteAttributes(attr.Path);
}
string fingerprint = String.Format("{0:00} {1}", EA_VERSION, index_fingerprint);
string uid = GuidFu.ToShortString(attr.UniqueId);
string mtime = StringFu.DateTimeToString(attr.LastWriteTime);
string filter = String.Empty;
if (attr.HasFilterInfo)
{
filter = String.Format("{0:000} {1}", attr.FilterVersion, attr.FilterName);
}
attr.LastAttrTime = DateTime.UtcNow;
string attrtime = StringFu.DateTimeToString(attr.LastAttrTime);
string [] csv = { fingerprint, uid, mtime, attrtime, filter };
ExtendedAttribute.Set(attr.Path, String.Join(",", csv));
return(true);
} catch (IOException e) {
// An IOException here probably means that we don't have the right
// permissions to set the EAs. We just fail silently and return false rather
// than spewing a bunch of scary exceptions.
//Logger.Log.Debug (e);
return(false);
} catch (Exception e) {
//Logger.Log.Debug (e, "Caught exception writing EAs to {0}", attr.Path);
// FIXME: Do something smarter with the exception.
return(false);
}
}
public FileAttributes Read(string path)
{
if (Disable)
{
return(null);
}
try {
string tmp = ExtendedAttribute.Get(path);
if (tmp == null)
{
return(null);
}
string[] csv = tmp.Split(',');
if (int.Parse(csv [0].Substring(0, 2)) != EA_VERSION ||
(index_fingerprint != null && csv [0].Substring(3) != index_fingerprint))
{
return(null);
}
FileAttributes attr = new FileAttributes();
attr.UniqueId = GuidFu.FromShortString(csv [1]);
attr.Path = path;
attr.LastWriteTime = StringFu.StringToDateTime(csv [2]);
attr.LastAttrTime = StringFu.StringToDateTime(csv [3]);
if (!String.IsNullOrEmpty(csv [4]))
{
attr.FilterVersion = int.Parse(csv [4].Substring(0, 3));
attr.FilterName = csv [4].Substring(4);
}
return(attr);
} catch (Exception e) {
//Logger.Log.Debug ("Caught exception reading EAs from {0}", path);
//Logger.Log.Debug (e);
// FIXME: Do something smarter with the exception.
return(null);
}
}
// Remapping hack from DumpIndex
static Uri RemapUri(LuceneQueryingDriver driver, Uri uri)
{
// We only need to remap URIs in the file system backend
if (driver.IndexName != "FileSystemIndex")
{
return(uri);
}
FileAttributesStore fa_store = new FileAttributesStore(new FileAttributesStore_Mixed(Path.Combine(PathFinder.IndexDir, "FileSystemIndex"), driver.Fingerprint));
string path = uri.LocalPath;
Beagle.Daemon.FileAttributes attr = fa_store.Read(path);
if (attr == null)
{
Console.WriteLine("No file attribute info for {0}", uri);
return(uri);
}
return(new Uri("uid:" + GuidFu.ToShortString(attr.UniqueId) + uri.Fragment));
}
////////////////////////////////////////////////////////////////
public NameInfo GetNameInfoById(Guid id)
{
Uri uri;
uri = GuidFu.ToUri(id);
IndexReader reader;
reader = LuceneCommon.GetReader(SecondaryStore);
TermDocs term_docs;
term_docs = reader.TermDocs();
Term term = new Term("Uri", UriFu.UriToEscapedString(uri));
term_docs.Seek(term);
int match_id = -1;
if (term_docs.Next())
{
match_id = term_docs.Doc();
}
term_docs.Close();
NameInfo info = null;
if (match_id != -1)
{
Document doc;
doc = reader.Document(match_id, fields_nameinfo);
info = DocumentToNameInfo(doc);
}
LuceneCommon.ReleaseReader(reader);
return(info);
}
public bool Write(FileAttributes fa)
{
SetPathFlag(fa.Path);
int ret = 0;
string filter_name;
// We need to quote any 's that appear in the strings
// (in particular, in the path)
lock (connection) {
// If a transaction has been requested, start it now.
MaybeStartTransaction();
filter_name = fa.FilterName;
if (filter_name == null)
{
filter_name = "";
}
filter_name = filter_name.Replace("'", "''");
string[] param = new string [] { "@unique_id", "@directory", "@filename", "@last_mtime", "@last_attrtime", "@filter_name", "@filter_version" };
object[] vals = new object [] {
GuidFu.ToShortString(fa.UniqueId),
fa.Directory.Replace("'", "''"), fa.Filename.Replace("'", "''"),
StringFu.DateTimeToString(fa.LastWriteTime),
StringFu.DateTimeToString(fa.LastAttrTime),
filter_name,
fa.FilterVersion
};
for (int i = 0; i < param.Length; i++)
{
InsertCommand.Parameters.AddWithValue(param[i], vals[i]);
}
ret = SqliteUtils.DoNonQuery(InsertCommand);
}
return(ret != 0);
}
public Indexable GetXmpQueryable(string path, Guid id, DirectoryModel parent)
{
Log.Debug("Asked to create xmp indexable for ({0}) {1}", GuidFu.ToShortString(id), path);
// Should be at least 6 characters /<...>.xmp
if (path.Length < 6)
{
return(null);
}
string basefile_path = Path.ChangeExtension(path, null);
// Ignore xmp files by itself
// FIXME: To support indexing independent xmp files will require even greater trouble
if (!File.Exists(basefile_path))
{
return(null);
}
XmpFile xmp_file = null;
try {
xmp_file = new XmpFile(path);
} catch {
Log.Warn("Cannot create xmpfile from {0}", path);
return(null);
}
// Try to get the correct uid for the basefile
// First we need to see if basefile is already scheduled (yet to be dispatched)
Uri basefile_uri = null;
Indexable base_indexable;
if (uid_manager.HasNewId(basefile_path))
{
// Since uid_manager has a new id for this basefile, so basefile is already scheduled
// Get basefile uid from there
Guid basefile_id = uid_manager.GetNewId(basefile_path);
basefile_uri = GuidFu.ToUri(basefile_id);
Log.Debug("{0} is already scheduled with uri {1}", basefile_path, basefile_uri);
}
else
{
// Basefile is not scheduled in the current batch
string basefile_name = Path.GetFileName(basefile_path);
// Try to schedule it for addition
base_indexable = queryable.GetCrawlingFileIndexable(parent, basefile_name);
if (base_indexable == null)
{
// GetCrawlingFileIndexable returns null if file does not need to be indexed
// So basefile is up-to-date
// Need to figure out id from uid manager
Guid basefile_id = uid_manager.GetIdByNameAndParentId(basefile_name, parent.UniqueId);
basefile_uri = GuidFu.ToUri(basefile_id);
Log.Debug("{0} is not scheduled and need not be, uri is {1}", basefile_path, basefile_uri);
}
else
{
Log.Debug("Need to index {0}", basefile_path);
// basefile needs to be indexed
// FIXME: Move the task business out of handler and into FSQ.cs
Scheduler.Task task;
task = queryable.NewAddTask(base_indexable);
// FIXME: What is the correct priority ?
// If should have similar priority to the one that this xmp-indexable will be a part of
task.Priority = Scheduler.Priority.Immediate;
queryable.ThisScheduler.Add(task);
// Get the basefile uri from the indexable
basefile_uri = base_indexable.Uri;
}
}
Log.Debug("Adding xmp-indexable for {0} (basefile uri {1}) with uid {2}",
path,
basefile_uri,
GuidFu.ToShortString(id));
Indexable indexable = new Indexable(IndexableType.PropertyChange, basefile_uri);
// Set the timestamp of the indexable as the timestamp of the basefile
// It could have also been skipped, the original Indexable.Add would anyway have it
indexable.Timestamp = File.GetLastWriteTimeUtc(basefile_path);
indexable.DisplayUri = UriFu.PathToFileUri(path);
// If the file was somehow deleted before this point, bail out.
if (!FileSystem.ExistsByDateTime(indexable.Timestamp))
{
xmp_file.Close();
return(null);
}
// Save some local states for PostAddHook, namely, path to the xmp file, path to basefile and generated uid
indexable.LocalState ["XmpFilePath"] = path;
indexable.LocalState ["BaseFilePath"] = basefile_path;
indexable.LocalState ["XmpGuid"] = GuidFu.ToShortString(id);
// FIXME: Should also delete previous xmp properties!
foreach (Property p in xmp_file.Properties)
{
p.IsMutable = true;
indexable.AddProperty(p);
}
xmp_file.Close();
return(indexable);
}
////////////////////////////////////////////////////////////////
public Guid GetIdByNameAndParentId(string name, Guid parent_id)
{
string parent_uri_str;
parent_uri_str = GuidFu.ToUriString(parent_id);
string key1, key2;
key1 = PropertyToFieldName(PropertyType.Keyword, Property.ParentDirUriPropKey);
key2 = PropertyToFieldName(PropertyType.Keyword, Property.ExactFilenamePropKey);
Term term1, term2;
term1 = new Term(key1, parent_uri_str);
term2 = new Term(key2, name.ToLower());
// Lets walk the exact file name terms first (term2)
// since there are probably fewer than parent directory
// Uri terms.
List <int> term2_doc_ids = new List <int> ();
IndexReader reader = LuceneCommon.GetReader(SecondaryStore);
TermDocs term_docs = reader.TermDocs();
term_docs.Seek(term2);
while (term_docs.Next())
{
term2_doc_ids.Add(term_docs.Doc());
}
term_docs.Seek(term1);
int match_id = -1;
while (term_docs.Next())
{
int doc_id = term_docs.Doc();
if (term2_doc_ids.BinarySearch(doc_id) >= 0)
{
match_id = doc_id;
break;
}
}
term_docs.Close();
Guid id;
if (match_id != -1)
{
Document doc;
doc = reader.Document(match_id);
id = GuidFu.FromUriString(doc.Get("Uri"));
}
else
{
id = Guid.Empty;
}
LuceneCommon.ReleaseReader(reader);
return(id);
}
