public void TestConsistencyOnExceptions()
{
// so the handler's index isn't empty
replicator.Publish(CreateRevision(1));
client.UpdateNow();
client.Dispose();
callback.Dispose();
// Replicator violates write-once policy. It may be that the
// handler copies files to the index dir, then fails to copy a
// file and reverts the copy operation. On the next attempt, it
// will copy the same file again. There is nothing wrong with this
// in a real system, but it does violate write-once, and MDW
// doesn't like it. Disabling it means that we won't catch cases
// where the handler overwrites an existing index file, but
// there's nothing currently we can do about it, unless we don't
// use MDW.
handlerIndexDir.PreventDoubleWrite = (false);
handlerTaxoDir.PreventDoubleWrite = (false);
// wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
ISourceDirectoryFactory @in = sourceDirFactory;
AtomicInt32 failures = new AtomicInt32(AtLeast(10));
sourceDirFactory = new SourceDirectoryFactoryAnonymousInnerClass(this, @in, failures);
handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, () =>
{
if (Random.NextDouble() < 0.2 && failures > 0)
{
throw new Exception("random exception from callback");
}
return(null);
});
client = new ReplicationClientAnonymousInnerClass(this, replicator, handler, @in, failures);
client.StartUpdateThread(10, "indexAndTaxo");
Directory baseHandlerIndexDir = handlerIndexDir.Delegate;
int numRevisions = AtLeast(20) + 2;
for (int i = 2; i < numRevisions; i++)
{
replicator.Publish(CreateRevision(i));
AssertHandlerRevision(i, baseHandlerIndexDir);
}
// disable errors -- maybe randomness didn't exhaust all allowed failures,
// and we don't want e.g. CheckIndex to hit false errors.
handlerIndexDir.MaxSizeInBytes = (0);
handlerIndexDir.RandomIOExceptionRate = (0.0);
handlerIndexDir.RandomIOExceptionRateOnOpen = (0.0);
handlerTaxoDir.MaxSizeInBytes = (0);
handlerTaxoDir.RandomIOExceptionRate = (0.0);
handlerTaxoDir.RandomIOExceptionRateOnOpen = (0.0);
}
public override void SetUp()
{
base.SetUp();
publishDir = NewMockDirectory();
handlerDir = NewMockDirectory();
sourceDirFactory = new PerSessionDirectoryFactory(CreateTempDir("replicationClientTest").FullName);
replicator = new LocalReplicator();
callback = new IndexReadyCallback(handlerDir);
handler = new IndexReplicationHandler(handlerDir, callback.Call);
client = new ReplicationClient(replicator, handler, sourceDirFactory);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.IndexDeletionPolicy = new SnapshotDeletionPolicy(conf.IndexDeletionPolicy);
publishWriter = new IndexWriter(publishDir, conf);
}
public override void SetUp()
{
base.SetUp();
publishIndexDir = NewDirectory();
publishTaxoDir = NewDirectory();
handlerIndexDir = NewMockDirectory();
handlerTaxoDir = NewMockDirectory();
clientWorkDir = CreateTempDir("replicationClientTest");
sourceDirFactory = new PerSessionDirectoryFactory(clientWorkDir.FullName);
replicator = new LocalReplicator();
callback = new IndexAndTaxonomyReadyCallback(handlerIndexDir, handlerTaxoDir);
handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, callback.Call);
client = new ReplicationClient(replicator, handler, sourceDirFactory);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.IndexDeletionPolicy = new SnapshotDeletionPolicy(conf.IndexDeletionPolicy);
publishIndexWriter = new IndexWriter(publishIndexDir, conf);
publishTaxoWriter = new IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter(publishTaxoDir);
config = new FacetsConfig();
config.SetHierarchical("A", true);
}
public ReplicationClientAnonymousInnerClass(IndexReplicationClientTest test, IReplicator replicator, IReplicationHandler handler, ISourceDirectoryFactory factory, AtomicInt32 failures)
: base(replicator, handler, factory)
{
this.test = test;
this.failures = failures;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="replicator">The <see cref="IReplicator"/> used for checking for updates</param>
/// <param name="handler">The <see cref="IReplicationHandler"/> notified when new revisions are ready</param>
/// <param name="factory">The <see cref="ISourceDirectoryFactory"/> for returning a <see cref="Directory"/> for a given source and session</param>
public ReplicationClient(IReplicator replicator, IReplicationHandler handler, ISourceDirectoryFactory factory)
{
this.replicator = replicator;
this.handler = handler;
this.factory = factory;
}
public XdcrModule(IReplicationHandler handler)
{
this.RequiresAuthentication();
Get["/pools"] = x =>
{
var output = new
{
pools = new object[]
{
new { name = "default", uri = "/pools/default?uuid=" + UUID_POOL }
},
uuid = UUID_POOL
};
return(Response.AsJson(output));
};
Get["pools/default"] = x =>
{
var output = new
{
buckets = new { uri = "/pools/default/buckets?uuid=" + UUID_POOL },
nodes = new object[]
{
new
{
ports = new { direct = XDCR_PORT },
couchApiBase = string.Concat("http://", XDCR_RECEIVER, ":", XDCR_PORT, "/"),
hostname = string.Concat(XDCR_RECEIVER, ":", XDCR_PORT)
}
}
};
return(Response.AsJson(output));
};
Get["pools/default/buckets"] = x =>
{
var output = new object[]
{
new
{
bucketCapabilities = new string[] { "couchapi" },
bucketType = "membase",
nodes = new object[]
{
new
{
ports = new { direct = XDCR_PORT },
couchApiBase = string.Concat("http://", XDCR_RECEIVER, ":", XDCR_PORT, "/default"),
hostname = string.Concat(XDCR_RECEIVER, ":", XDCR_PORT)
}
},
name = XDCR_BUCKET,
vBucketServerMap = new
{
serverList = new string[] { string.Concat(XDCR_RECEIVER, ":", XDCR_PORT) },
vBucketMap = VBucketMap
},
uuid = UUID_BUCKET,
uri = string.Concat("/pools/default/buckets/", XDCR_BUCKET, "?bucket_uuid=", UUID_BUCKET)
}
};
return(Response.AsJson(output));
};
Get["pools/default/buckets/{bucket}"] = x =>
{
var output = new object[]
{
new
{
bucketCapabilities = new string[] { "couchapi" },
bucketType = "membase",
nodes = new object[]
{
new
{
ports = new { direct = XDCR_PORT },
couchApiBase = string.Concat("http://", XDCR_RECEIVER, ":", XDCR_PORT, "/default"),
hostname = string.Concat(XDCR_RECEIVER, ":", XDCR_PORT)
}
},
name = XDCR_BUCKET,
vBucketServerMap = new
{
serverList = new string[] { string.Concat(XDCR_RECEIVER, ":", XDCR_PORT) },
vBucketMap = VBucketMap
},
uuid = UUID_BUCKET,
uri = string.Concat("/pools/default/buckets/", XDCR_BUCKET, "?bucket_uuid=", UUID_BUCKET)
}
};
return(Response.AsJson(output));
};
Get[REGEX_VBUCKET] = x =>
{
var status = getBucketExistsStatusCode(x.bucket);
if (Request.Method == "HEAD")
{
return(status);
}
var result = new { db_name = XDCR_BUCKET };
return(status == HttpStatusCode.OK ? Response.AsJson(result) : null);
};
Get[REGEX_MASTER_VBUCKET] = x =>
{
var status = getBucketExistsStatusCode(x.bucket);
if (Request.Method == "HEAD")
{
return(status);
}
var result = new { db_name = XDCR_BUCKET };
return(status == HttpStatusCode.OK ? Response.AsJson(result) : null);
};
//TODO: figure out a regex for both this pattern and master_vbucket - Sinatra version works like that
Get[REGEX_MASTER_VBUCKET_LOCAL] = x =>
{
var status = getBucketExistsStatusCode(x.bucket);
if (Request.Method == "HEAD")
{
return(status);
}
var result = new { db_name = XDCR_BUCKET };
return(status == HttpStatusCode.OK ? Response.AsJson(result) : null);
};
Post[REGEX_REVS_DIFF] = x =>
{
var body = "";
Context.Request.Body.Position = 0;
using (var sr = new StreamReader(Context.Request.Body))
{
body = sr.ReadToEnd();
}
var jobj = JObject.Parse(body);
var outDict = new Dictionary <string, object>();
foreach (var item in jobj)
{
var key = item.Key;
var rev = item.Value.ToString();
if (handler.IsMissing(key, rev))
{
outDict[key] = new { missing = rev };
}
}
return(Response.AsJson(outDict));
};
Post[REGEX_FULL_COMMIT] = x =>
{
return(Response.AsJson(new { ok = true }, HttpStatusCode.Created));
};
Post[REGEX_BULK_DOCS] = x =>
{
var body = "";
Context.Request.Body.Position = 0;
using (var sr = new StreamReader(Context.Request.Body))
{
body = sr.ReadToEnd();
}
var jobj = JObject.Parse(body);
var newEdits = jobj.Value <bool>("new_edits");
var docs = jobj.Value <JArray>("docs");
foreach (var doc in docs)
{
var originalDoc = Encoding.UTF8.GetString(Convert.FromBase64String(doc.Value <string>("base64")));
var meta = doc["meta"] as JObject;
var document = new Document
{
Id = meta.Value <string>("id"),
Revision = meta.Value <string>("rev"),
Expiration = meta.Value <int>("expiration"),
Flags = meta.Value <int>("flags"),
Value = originalDoc
};
handler.CreateDocument(document);
}
return(HttpStatusCode.Created);
};
}
public XdcrModule(IReplicationHandler handler)
{
this.RequiresAuthentication();
Get["/pools"] = x =>
{
var output = new
{
pools = new object[]
{
new { name = "default", uri = "/pools/default?uuid=" + UUID_POOL }
},
uuid = UUID_POOL
};
return Response.AsJson(output);
};
Get["pools/default"] = x =>
{
var output = new
{
buckets = new { uri = "/pools/default/buckets?uuid=" + UUID_POOL },
nodes = new object[]
{
new
{
ports = new { direct = XDCR_PORT } ,
couchApiBase = string.Concat("http://", XDCR_RECEIVER, ":", XDCR_PORT, "/"),
hostname = string.Concat(XDCR_RECEIVER, ":", XDCR_PORT)
}
}
};
return Response.AsJson(output);
};
Get["pools/default/buckets"] = x =>
{
var output = new object[]
{
new
{
bucketCapabilities = new string[] { "couchapi" },
bucketType = "membase",
nodes = new object[]
{
new
{
ports = new { direct = XDCR_PORT } ,
couchApiBase = string.Concat("http://", XDCR_RECEIVER, ":", XDCR_PORT, "/default"),
hostname = string.Concat(XDCR_RECEIVER, ":", XDCR_PORT)
}
},
name = XDCR_BUCKET,
vBucketServerMap = new
{
serverList = new string[] { string.Concat(XDCR_RECEIVER, ":", XDCR_PORT) },
vBucketMap = VBucketMap
},
uuid = UUID_BUCKET,
uri = string.Concat("/pools/default/buckets/", XDCR_BUCKET, "?bucket_uuid=", UUID_BUCKET)
}
};
return Response.AsJson(output);
};
Get["pools/default/buckets/{bucket}"] = x =>
{
var output = new object[]
{
new
{
bucketCapabilities = new string[] { "couchapi" },
bucketType = "membase",
nodes = new object[]
{
new
{
ports = new { direct = XDCR_PORT } ,
couchApiBase = string.Concat("http://", XDCR_RECEIVER, ":", XDCR_PORT, "/default"),
hostname = string.Concat(XDCR_RECEIVER, ":", XDCR_PORT)
}
},
name = XDCR_BUCKET,
vBucketServerMap = new
{
serverList = new string[] { string.Concat(XDCR_RECEIVER, ":", XDCR_PORT) },
vBucketMap = VBucketMap
},
uuid = UUID_BUCKET,
uri = string.Concat("/pools/default/buckets/", XDCR_BUCKET, "?bucket_uuid=", UUID_BUCKET)
}
};
return Response.AsJson(output);
};
Get[REGEX_VBUCKET] = x =>
{
var status = getBucketExistsStatusCode(x.bucket);
if (Request.Method == "HEAD")
{
return status;
}
var result = new { db_name = XDCR_BUCKET };
return status == HttpStatusCode.OK ? Response.AsJson(result) : null;
};
Get[REGEX_MASTER_VBUCKET] = x =>
{
var status = getBucketExistsStatusCode(x.bucket);
if (Request.Method == "HEAD")
{
return status;
}
var result = new { db_name = XDCR_BUCKET };
return status == HttpStatusCode.OK ? Response.AsJson(result) : null;
};
//TODO: figure out a regex for both this pattern and master_vbucket - Sinatra version works like that
Get[REGEX_MASTER_VBUCKET_LOCAL] = x =>
{
var status = getBucketExistsStatusCode(x.bucket);
if (Request.Method == "HEAD")
{
return status;
}
var result = new { db_name = XDCR_BUCKET };
return status == HttpStatusCode.OK ? Response.AsJson(result) : null;
};
Post[REGEX_REVS_DIFF] = x =>
{
var body = "";
Context.Request.Body.Position = 0;
using (var sr = new StreamReader(Context.Request.Body))
{
body = sr.ReadToEnd();
}
var jobj = JObject.Parse(body);
var outDict = new Dictionary<string, object>();
foreach (var item in jobj)
{
var key = item.Key;
var rev = item.Value.ToString();
if (handler.IsMissing(key, rev))
{
outDict[key] = new { missing = rev };
}
}
return Response.AsJson(outDict);
};
Post[REGEX_FULL_COMMIT] = x =>
{
return Response.AsJson(new { ok = true }, HttpStatusCode.Created);
};
Post[REGEX_BULK_DOCS] = x =>
{
var body = "";
Context.Request.Body.Position = 0;
using (var sr = new StreamReader(Context.Request.Body))
{
body = sr.ReadToEnd();
}
var jobj = JObject.Parse(body);
var newEdits = jobj.Value<bool>("new_edits");
var docs = jobj.Value<JArray>("docs");
foreach (var doc in docs)
{
var originalDoc = Encoding.UTF8.GetString(Convert.FromBase64String(doc.Value<string>("base64")));
var meta = doc["meta"] as JObject;
var document = new Document
{
Id = meta.Value<string>("id"),
Revision = meta.Value<string>("rev"),
Expiration = meta.Value<int>("expiration"),
Flags = meta.Value<int>("flags"),
Value = originalDoc
};
handler.CreateDocument(document);
}
return HttpStatusCode.Created;
};
}
