private void WriteObjectToTar (TarOutputStream tar_out,
FileSystemObject fso,
EventTracker tracker)
{
MemoryStream memory = null;
TarHeader header;
header = new TarHeader ();
StringBuilder name_builder;
name_builder = new StringBuilder (fso.FullName);
name_builder.Remove (0, this.FullName.Length+1);
header.Name = name_builder.ToString ();
header.ModTime = fso.Timestamp;
if (fso is DirectoryObject) {
header.Mode = 511; // 0777
header.TypeFlag = TarHeader.LF_DIR;
header.Size = 0;
} else {
header.Mode = 438; // 0666
header.TypeFlag = TarHeader.LF_NORMAL;
memory = new MemoryStream ();
((FileObject) fso).AddToStream (memory, tracker);
header.Size = memory.Length;
}
TarEntry entry;
entry = new TarEntry (header);
tar_out.PutNextEntry (entry);
if (memory != null) {
tar_out.Write (memory.ToArray (), 0, (int) memory.Length);
memory.Close ();
}
tar_out.CloseEntry ();
// If this is a directory, write out the children
if (fso is DirectoryObject)
foreach (FileSystemObject child in fso.Children)
WriteObjectToTar (tar_out, child, tracker);
}
private void WriteObjectToZip (ZipOutputStream zip_out,
FileSystemObject fso,
EventTracker tracker)
{
MemoryStream memory = null;
string name;
name = fso.FullName.Substring (this.FullName.Length + 1);
if (fso is DirectoryObject)
name += "/";
ZipEntry entry;
entry = new ZipEntry (name);
entry.DateTime = fso.Timestamp;
if (fso is DirectoryObject)
entry.Size = 0;
else {
memory = new MemoryStream ();
((FileObject) fso).AddToStream (memory, tracker);
entry.Size = memory.Length;
}
zip_out.PutNextEntry (entry);
if (memory != null) {
zip_out.Write (memory.ToArray (), 0, (int) memory.Length);
memory.Close ();
}
// If this is a directory, write out the children
if (fso is DirectoryObject)
foreach (FileSystemObject child in fso.Children)
WriteObjectToZip (zip_out, child, tracker);
}
static public bool CheckQuery (Query query, FileSystemObject root)
{
// Find the set of objects that we expect to match the query,
// based on our knowledge of the current state of the tree.
ICollection matching_fsos;
matching_fsos = root.RecursiveQuery (query);
// Query the daemon and get the actual list of hits.
Hashtable matching_hits;
matching_hits = QueryFu.GetHits (query);
bool success;
success = true;
foreach (FileSystemObject fso in matching_fsos) {
string uri = UriFu.UriToEscapedString (fso.Uri);
if (matching_hits.Contains (uri))
matching_hits.Remove (uri);
else {
Log.Failure ("Hit missing from beagled query results: {0}", uri);
success = false;
}
}
foreach (Hit hit in matching_hits.Values) {
Log.Failure ("Unexpected extra hit in beagled query results: {0}", hit.Uri);
Log.Failure (" Properties:");
foreach (Property prop in hit.Properties)
Log.Failure (" {0} = {1}", prop.Key, prop.Value);
success = false;
}
return success;
}
static public bool VerifyIndex (FileSystemObject root)
{
bool success;
success = true;
Log.Info ("Verifying index for root {0}", root.FullName);
for (int i = 0; i < Token.Count; ++i) {
Query query;
query = QueryFu.NewTokenQuery (i);
if (! CheckQuery (query, root)) {
Log.Spew ("Failed query is:");
QueryFu.SpewQuery (query);
success = false;
}
}
if (success)
Log.Info ("Index successfully verified");
else
Log.Info ("Verification failed");
return success;
}
///////////////////////////////////////////////////////////////////////
static private void GetCountDeltas (FileSystemObject fso, out int d_files, out int d_dirs)
{
if (fso is DirectoryObject) {
DirectoryObject dir = (DirectoryObject) fso;
d_files = dir.n_files_below;
d_dirs = dir.n_dirs_below + 1; // add one for ourself
} else if (fso is FileObject) {
d_files = 1; // just ourself
d_dirs = 0;
} else {
throw new Exception ("Unknown type " + fso);
}
}
private void WriteObjectToZip(ZipOutputStream zip_out,
FileSystemObject fso,
EventTracker tracker)
{
MemoryStream memory = null;
string name;
name = fso.FullName.Substring(this.FullName.Length + 1);
if (fso is DirectoryObject)
{
name += "/";
}
ZipEntry entry;
entry = new ZipEntry(name);
entry.DateTime = fso.Timestamp;
if (fso is DirectoryObject)
{
entry.Size = 0;
}
else
{
memory = new MemoryStream();
((FileObject)fso).AddToStream(memory, tracker);
entry.Size = memory.Length;
}
zip_out.PutNextEntry(entry);
if (memory != null)
{
zip_out.Write(memory.ToArray(), 0, (int)memory.Length);
memory.Close();
}
// If this is a directory, write out the children
if (fso is DirectoryObject)
{
foreach (FileSystemObject child in fso.Children)
{
WriteObjectToZip(zip_out, child, tracker);
}
}
}
///////////////////////////////////////////////////////////////////////
static private void GetCountDeltas(FileSystemObject fso, out int d_files, out int d_dirs)
{
if (fso is DirectoryObject)
{
DirectoryObject dir = (DirectoryObject)fso;
d_files = dir.n_files_below;
d_dirs = dir.n_dirs_below + 1; // add one for ourself
}
else if (fso is FileObject)
{
d_files = 1; // just ourself
d_dirs = 0;
}
else
{
throw new Exception("Unknown type " + fso);
}
}
static public bool CheckQuery(Query query, FileSystemObject root)
{
// Find the set of objects that we expect to match the query,
// based on our knowledge of the current state of the tree.
ICollection matching_fsos;
matching_fsos = root.RecursiveQuery(query);
// Query the daemon and get the actual list of hits.
Hashtable matching_hits;
matching_hits = QueryFu.GetHits(query);
bool success;
success = true;
foreach (FileSystemObject fso in matching_fsos)
{
string uri = UriFu.UriToEscapedString(fso.Uri);
if (matching_hits.Contains(uri))
{
matching_hits.Remove(uri);
}
else
{
Log.Failure("Hit missing from beagled query results: {0}", uri);
success = false;
}
}
foreach (Hit hit in matching_hits.Values)
{
Log.Failure("Unexpected extra hit in beagled query results: {0}", hit.Uri);
Log.Failure(" Properties:");
foreach (Property prop in hit.Properties)
{
Log.Failure(" {0} = {1}", prop.Key, prop.Value);
}
success = false;
}
return(success);
}
static public bool TestRandomQueries (double minutes_to_run, FileSystemObject root)
{
if (minutes_to_run < 0)
minutes_to_run = default_minutes_to_run;
Log.Info ("Running random queries for {0:0.0} minutes", minutes_to_run);
bool success;
success = true;
Stopwatch sw;
sw = new Stopwatch ();
sw.Start ();
int count = 0;
while (true) {
if ((count % 100 == 0) && sw.ElapsedTime > minutes_to_run * 60)
break;
++count;
if (count % 1000 == 0)
Log.Spew ("{0} queries run", count);
Beagle.Query query;
query = QueryFu.NewRandomQuery ();
if (! CheckQuery (query, root)) {
Log.Spew ("Failed query is:");
QueryFu.SpewQuery (query);
success = false;
break;
}
}
// In case we ended early
minutes_to_run = sw.ElapsedTime / 60;
Log.Spew ("Ran {0} queries in {1:0.0} minutes ({2:0.0} queries/s)",
count, minutes_to_run, count / (minutes_to_run * 60));
return success;
}
public override void RemoveChild(FileSystemObject child, EventTracker tracker)
{
// Likewise, we have to walk up the tree and adjust
// the n_*_below counts when we remove a child.
int d_files, d_dirs;
GetCountDeltas(child, out d_files, out d_dirs);
DirectoryObject curr;
curr = this;
while (curr != null)
{
curr.n_files_below -= d_files;
curr.n_dirs_below -= d_dirs;
curr = (DirectoryObject)curr.Parent;
}
base.RemoveChild(child, tracker);
}
public virtual void AddChild(FileSystemObject child, EventTracker tracker)
{
if (children == null)
{
throw new Exception("Can't add a child to " + Uri.ToString());
}
if (child.parent != null)
{
throw new Exception("Can't add parented child " + child.Uri.ToString() + " to " + Uri.ToString());
}
// FIXME: Need to handle the case of the added child
// clobbering another w/ the same name.
child.parent = this;
children [child.Name] = child;
if (IsRooted)
{
child.AddOnDisk(tracker);
}
}
public override void AddChild(FileSystemObject child, EventTracker tracker)
{
// Every time we add a child, we walk up the tree
// and adjust the n_*_below counts for every node
// above us.
int d_files, d_dirs;
GetCountDeltas(child, out d_files, out d_dirs);
DirectoryObject curr;
curr = this;
while (curr != null)
{
curr.n_files_below += d_files;
curr.n_dirs_below += d_dirs;
curr = (DirectoryObject)curr.Parent;
}
base.AddChild(child, tracker);
}
public virtual void MoveChild(FileSystemObject child, FileSystemObject new_parent, EventTracker tracker)
{
if (child.parent != this)
{
throw new Exception(child.Uri.ToString() + " is not a child of " + Uri.ToString());
}
if (new_parent == null || new_parent == child.parent)
{
return;
}
// We can't move child into new_parent if child is
// already above new_parent in the tree.
if (child.IsAncestorOf(new_parent))
{
throw new Exception("Can't move " + child.Uri.ToString() + " to " + new_parent.Uri.ToString());
}
string old_full_name;
old_full_name = child.FullName;
// FIXME: We need to handle the case of the moved
// child clobbering another w/ the same name.
child.parent = new_parent;
this.children.Remove(child.Name);
new_parent.children [child.Name] = child;
// FIXME: What if this is not rooted, but new_parent is?
if (new_parent.IsRooted)
{
child.MoveOnDisk(old_full_name, tracker);
}
}
public DirectoryObject()
{
timestamp = FileSystemObject.PickTimestamp();
AllowChildren();
}
static private Query NewRandomQuery(int length,
bool allow_inexpensive,
bool inside_an_or)
{
Query query;
query = new Query();
// One in four queries will contain some OR terms.
if (!inside_an_or && random.Next(4) == 0)
{
int N = random.Next(3) + 1;
for (int i = 0; i < N; ++i)
{
QueryPart_Or part;
part = new QueryPart_Or();
int sub_length;
sub_length = random.Next(length) + 1;
if (sub_length < 2)
{
sub_length = 2;
}
// We generate a new query at random, and stuff its QueryParts
// into our Or QueryPart.
Query or_query;
or_query = NewRandomQuery(sub_length, allow_inexpensive, true);
foreach (QueryPart sub_part in or_query.Parts)
{
part.Add(sub_part);
}
query.AddPart(part);
}
}
if (allow_inexpensive && !inside_an_or)
{
int mime_type;
mime_type = random.Next(3);
QueryPart_Or mime_type_part = new QueryPart_Or();
QueryPart_Property part;
part = new QueryPart_Property();
part.Type = PropertyType.Keyword;
part.Key = "beagle:MimeType";
if (mime_type == 0)
{
part.Value = "inode/directory";
mime_type_part.Add(part);
query.AddPart(mime_type_part);
}
else if (mime_type == 1)
{
part.Value = "text/plain";
mime_type_part.Add(part);
query.AddPart(mime_type_part);
}
}
// Every query must contain at least
// one required part.
bool contains_required;
contains_required = false;
for (int i = 0; i < length; ++i)
{
QueryPart_Text part;
part = new QueryPart_Text();
part.Text = Token.GetRandom();
// Prohibited parts are not allowed inside an or
if (contains_required && !inside_an_or)
{
if (random.Next(2) == 0)
{
part.Logic = QueryPartLogic.Prohibited;
}
}
else
{
// This part will be required.
contains_required = true;
}
if (random.Next(2) == 0)
{
part.SearchTextProperties = false;
}
else if (allow_inexpensive && random.Next(2) == 0)
{
part.SearchFullText = false;
}
query.AddPart(part);
}
// Note the ! inside_an_or; date range queries don't
// work right inside OR queries when being searched
// within the resolution of one day. See the FIXME
// about hit filters in LuceneCommon.cs
if (allow_inexpensive && !inside_an_or && random.Next(3) == 0)
{
DateTime a, b;
FileSystemObject.PickTimestampRange(out a, out b);
QueryPart_DateRange part;
part = new QueryPart_DateRange();
part.StartDate = a;
part.EndDate = b;
query.AddPart(part);
}
return(query);
}
public virtual void MoveChild (FileSystemObject child, FileSystemObject new_parent, EventTracker tracker)
{
if (child.parent != this)
throw new Exception (child.Uri.ToString () + " is not a child of " + Uri.ToString ());
if (new_parent == null || new_parent == child.parent)
return;
// We can't move child into new_parent if child is
// already above new_parent in the tree.
if (child.IsAncestorOf (new_parent))
throw new Exception ("Can't move " + child.Uri.ToString () + " to " + new_parent.Uri.ToString ());
string old_full_name;
old_full_name = child.FullName;
// FIXME: We need to handle the case of the moved
// child clobbering another w/ the same name.
child.parent = new_parent;
this.children.Remove (child.Name);
new_parent.children [child.Name] = child;
// FIXME: What if this is not rooted, but new_parent is?
if (new_parent.IsRooted)
child.MoveOnDisk (old_full_name, tracker);
}
override protected Uri GetChildUri (FileSystemObject child)
{
string s = Path.Combine (UriFu.UriToEscapedString (this.Uri), child.Name);
//Console.WriteLine ("Asked to combine {0} and {1} = {2}", UriFu.UriToEscapedString (this.Uri), child.Name, s);
return UriFu.EscapedStringToUri (s);
}
override protected Uri GetChildUri(FileSystemObject children)
{
// FIXME: What is the uri scheme for bzip2 files?
return(new Uri(this.Uri.ToString() + "#" + children.Name));
}
// This should return null for objects that are not directly
// represented on the file system (i.e. files inside an archive)
virtual protected string GetChildFullName(FileSystemObject child)
{
return(Path.Combine(this.FullName, child.Name));
}
// By definition, an object is an ancestor of itself.
public bool IsAncestorOf (FileSystemObject fso)
{
if (! this.HasChildren)
return false;
while (fso != null) {
if (this == fso)
return true;
fso = fso.Parent;
}
return false;
}
public WaitUntilVerifiedClosure (FileSystemObject root,
VerifiedHandler verified)
{
this.root = root;
this.verified = verified;
}
override protected Uri GetChildUri (FileSystemObject children)
{
// FIXME: What is the uri scheme for bzip2 files?
return new Uri (this.Uri.ToString () + "#" + children.Name);
}
public WaitUntilVerifiedClosure(FileSystemObject root,
VerifiedHandler verified)
{
this.root = root;
this.verified = verified;
}
public FileObject()
{
timestamp = FileSystemObject.PickTimestamp();
}
public override void AddChild (FileSystemObject child, EventTracker tracker)
{
// Every time we add a child, we walk up the tree
// and adjust the n_*_below counts for every node
// above us.
int d_files, d_dirs;
GetCountDeltas (child, out d_files, out d_dirs);
DirectoryObject curr;
curr = this;
while (curr != null) {
curr.n_files_below += d_files;
curr.n_dirs_below += d_dirs;
curr = (DirectoryObject) curr.Parent;
}
base.AddChild (child, tracker);
}
public override void RemoveChild (FileSystemObject child, EventTracker tracker)
{
// Likewise, we have to walk up the tree and adjust
// the n_*_below counts when we remove a child.
int d_files, d_dirs;
GetCountDeltas (child, out d_files, out d_dirs);
DirectoryObject curr;
curr = this;
while (curr != null) {
curr.n_files_below -= d_files;
curr.n_dirs_below -= d_dirs;
curr = (DirectoryObject) curr.Parent;
}
base.RemoveChild (child, tracker);
}
virtual protected Uri GetChildUri(FileSystemObject child)
{
throw new Exception("Invalid GetChildUri call");
}
public virtual void AddChild (FileSystemObject child, EventTracker tracker)
{
if (children == null)
throw new Exception ("Can't add a child to " + Uri.ToString ());
if (child.parent != null)
throw new Exception ("Can't add parented child " + child.Uri.ToString () + " to " + Uri.ToString ());
// FIXME: Need to handle the case of the added child
// clobbering another w/ the same name.
child.parent = this;
children [child.Name] = child;
if (IsRooted)
child.AddOnDisk (tracker);
}
public virtual void ClobberingAddChild (FileSystemObject child, FileSystemObject victim, EventTracker tracker)
{
if (children == null)
throw new Exception ("Can't add a child to " + Uri.ToString ());
if (child.parent != null)
throw new Exception ("Can't add parented child " + child.Uri.ToString () + " to " + Uri.ToString ());
if (victim.parent != this)
throw new Exception ("Victim " + victim.Uri.ToString () + " is not a child of " + Uri.ToString ());
if (child.Extension != victim.Extension)
throw new Exception ("Extension mismatch: " + child.Extension + " vs. " + victim.Extension);
victim.parent = null;
child.parent = this;
child.id = victim.id;
child.base_name = victim.base_name;
child.name = null;
children [child.Name] = child;
if (IsRooted)
child.AddOnDisk (tracker);
}
public virtual void RemoveChild (FileSystemObject child, EventTracker tracker)
{
if (child.parent != this)
throw new Exception (child.Uri.ToString () + " is not a child of " + Uri.ToString ());
if (IsRooted)
child.DeleteOnDisk (tracker);
child.parent = null;
children.Remove (child.Name);
}
virtual protected Uri GetChildUri (FileSystemObject child)
{
throw new Exception ("Invalid GetChildUri call");
}
public override void MoveChild (FileSystemObject child, FileSystemObject new_parent, EventTracker tracker)
{
int d_files, d_dirs;
GetCountDeltas (child, out d_files, out d_dirs);
DirectoryObject curr;
curr = this;
while (curr != null) {
curr.n_files_below -= d_files;
curr.n_dirs_below -= d_dirs;
curr = (DirectoryObject) curr.Parent;
}
curr = (DirectoryObject) new_parent;
while (curr != null) {
curr.n_files_below += d_files;
curr.n_dirs_below += d_dirs;
curr = (DirectoryObject) curr.Parent;
}
base.MoveChild (child, new_parent, tracker);
}
static public void WaitUntilVerified (FileSystemObject root, VerifiedHandler verified)
{
WaitUntilVerifiedClosure closure;
closure = new WaitUntilVerifiedClosure (root, verified);
closure.Start ();
}
// This should return null for objects that are not directly
// represented on the file system (i.e. files inside an archive)
virtual protected string GetChildFullName (FileSystemObject child)
{
return Path.Combine (this.FullName, child.Name);
}
