public void OnRequest(HttpRequestHead head, IDataProducer body, IHttpResponseDelegate response)
{
var env = new Environment();
if (context != null)
foreach (var kv in context)
env[kv.Key] = kv.Value;
env.Headers = head.Headers ?? new Dictionary<string, string>();
env.Method = head.Method ?? "";
env.Path = head.Path ?? "";
env.PathBase = "";
env.QueryString = head.QueryString ?? "";
env.Scheme = "http"; // XXX
env.Version = "1.0";
if (body == null)
env.Body = null;
else
env.Body = (onData, onError, onEnd) =>
{
var d = body.Connect(new DataConsumer(onData, onError, onEnd));
return () => { if (d != null) d.Dispose(); };
};
appDelegate(env, HandleResponse(response), HandleError(response));
}
protected override object ProcessData(IDataProducer sender, object data)
{
Utils.ThrowException(!(data is string[]) ? new ArgumentTypeException("data") : null);
DateTime timeStart = DateTime.Now;
DocumentCorpus corpus = new DocumentCorpus();
foreach (string line in (string[])data)
{
int splitIdx = line.IndexOfAny(new char[] { ' ', '\t', '\n' });
Document doc;
if (!mIsNamedDoc || splitIdx < 0)
{
doc = new Document("", line.Trim());
}
else
{
doc = new Document(line.Substring(0, splitIdx).Trim(), line.Substring(splitIdx).Trim());
}
doc.Features.SetFeatureValue("_time", DateTime.Now.ToString(Utils.DATE_TIME_SIMPLE));
corpus.AddDocument(doc);
}
corpus.Features.SetFeatureValue("_provider", GetType().ToString());
corpus.Features.SetFeatureValue("_isNamedDoc", mIsNamedDoc.ToString());
corpus.Features.SetFeatureValue("_timeStart", timeStart.ToString(Utils.DATE_TIME_SIMPLE));
corpus.Features.SetFeatureValue("_timeEnd", DateTime.Now.ToString(Utils.DATE_TIME_SIMPLE));
return corpus;
}
public void OnRequest(HttpRequestHead head,
IDataProducer body,
IHttpResponseDelegate response)
{
if (head.Method.ToUpperInvariant() == "POST")
{
ProcessPOSTRequest(body, response);
}
else if (head.Method.ToUpperInvariant() == "GET" && head.Uri == "/crossdomain.xml")
{
ProcessCrossDomainRequest(body, response);
}
else if (head.Method.ToUpperInvariant() == "GET" && head.QueryString.Contains("metrics"))
{
ProcessGETRequest(body, head, response);
}
else if (head.Method.ToUpperInvariant() == "GET" && head.Uri == "/")
{
ProcessLoadBalancerRequest(body, response);
}
else
{
ProcessFileNotFound(body, response);
}
}
public void OnRequest(HttpRequestHead head, IDataProducer body, IHttpResponseDelegate response)
{
var env = new Dictionary<string, object>();
var request = new RequestEnvironment(env);
if (context != null)
foreach (var kv in context)
env[kv.Key] = kv.Value;
if (head.Headers == null)
request.Headers = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
else
request.Headers = head.Headers.ToDictionary(kv => kv.Key, kv => new[] { kv.Value }, StringComparer.OrdinalIgnoreCase);
request.Method = head.Method ?? "";
request.Path = head.Path ?? "";
request.PathBase = "";
request.QueryString = head.QueryString ?? "";
request.Scheme = "http"; // XXX
request.Version = "1.0";
if (body == null)
request.BodyDelegate = null;
else
request.BodyDelegate = (write, end, cancellationToken) =>
{
var d = body.Connect(new DataConsumer(
write,
end,
() => end(null)));
cancellationToken.Register(d.Dispose);
};
appDelegate(env, HandleResponse(response), HandleError(response));
}
public void OnRequest(HttpRequestHead request, IDataProducer requestBody, IHttpResponseDelegate response)
{
if (request.Uri.StartsWith("/feed.xml"))
{
var body = new FeedBuilder(Port).Generate(FeedTitle, FilePaths, ImagePath);
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new SimpleProducer(body));
return;
}
// deal with request for file content
string uri = request.Uri.Replace("%20", " ").Replace("/", "\\");
string filePath = FilePaths.Where(d => d.Contains(uri)).FirstOrDefault();
if (filePath != null)
{
FileInfo fi = new FileInfo(filePath);
string mimeType = GetMimeType(filePath);
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", mimeType },
{ "Content-Length", fi.Length.ToString() },
}
};
response.OnResponse(headers, new FileProducer(filePath));
return;
}
else
{
var responseBody = "The resource you requested ('" + request.Uri + "') could not be found.";
var headers = new HttpResponseHead()
{
Status = "404 Not Found",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", responseBody.Length.ToString() }
}
};
var body = new SimpleProducer(responseBody);
response.OnResponse(headers, body);
return;
}
}
protected override void ConsumeData(IDataProducer sender, object data)
{
Utils.ThrowException(!(data is DocumentCorpus) ? new ArgumentTypeException("data") : null);
DocumentCorpus corpus = (DocumentCorpus)data;
foreach (Document document in corpus.Documents)
{
ConsumeDocument(document);
}
}
protected override void ConsumeData(IDataProducer sender, object data)
{
Utils.ThrowException(!(data is DocumentCorpus) ? new ArgumentTypeException("data") : null);
DocumentCorpus corpus = (DocumentCorpus)data;
if (mCsrftoken == null || !SendDocumentCorpusInfo(corpus))
{
GetDjangoCookie();
SendDocumentCorpusInfo(corpus);
}
}
/*protected*/
public override object ProcessData(IDataProducer sender, object data)
{
Utils.ThrowException(!(data is DocumentCorpus) ? new ArgumentTypeException("data") : null);
DocumentCorpus corpus = (DocumentCorpus)data;
foreach (Document document in corpus.Documents)
{
ProcessDocument(document);
}
return corpus;
}
protected override void ConsumeData(IDataProducer sender, object data)
{
Utils.ThrowException(!(data is DocumentCorpus) ? new ArgumentTypeException("data") : null);
DocumentCorpus corpus = (DocumentCorpus)data;
if (mCsrftoken == null || !SendDocumentCorpusInfo(corpus))
{
GetDjangoCookie();
SendDocumentCorpusInfo(corpus);
}
}
public override void ProcessPost(HttpRequestHead request, IDataProducer requestBody, IHttpResponseDelegate response)
{
//Figure out how to get the post params from within Kayak so some of these operations that
//should be a post can be. Right now I am limited to just post based on verbs on the url.
if (request.Uri.StartsWith("/api/play/stopSequence"))
{
StopSequence(request, response);
return;
}
UnsupportedOperation(request, response);
}
/// <summary>
/// Returns the number of elements in a sequence, as a future value.
/// </summary>
/// <typeparam name="TSource">The type of the elements of source.</typeparam>
/// <param name="source">A sequence that contains elements to be counted.</param>
/// <returns>The number of elements in the input sequence, as a future value.
/// The actual count can only be retrieved after the source has indicated the end
/// of its data.
/// </returns>
public static IFuture <int> Count <TSource>(this IDataProducer <TSource> source)
{
source.ThrowIfNull("source");
Future <int> ret = new Future <int>();
int count = 0;
source.DataProduced += t => count++;
source.EndOfData += () => ret.Value = count;
return(ret);
}
public override void ProcessPost(HttpRequestHead request, IDataProducer requestBody, IHttpResponseDelegate response)
{
//Figure out how to get the post params from within Kayak so some of these operations that
//should be a post can be. Right now I am limited to just post based on verbs on the url.
if (request.Uri.StartsWith("/api/play/stopSequence"))
{
StopSequence(request, response);
return;
}
UnsupportedOperation(request,response);
}
/// <summary>
/// Converts an IDataProducer into a future array.
/// </summary>
/// <remarks>This will force all values to be buffered</remarks>
public static IFuture <TSource[]> ToFutureArray <TSource>(this IDataProducer <TSource> source)
{
source.ThrowIfNull("source");
Future <TSource[]> ret = new Future <TSource[]>();
List <TSource> list = source.ToList();
source.EndOfData += () => ret.Value = list.ToArray();
return(ret);
}
/// <summary>
/// Groups the elements of a sequence according to a specified key selector function
/// and creates a result value from each group and its key.
/// </summary>
/// <typeparam name="TKey">The return-type of the transform used to group the sequence</typeparam>
/// <typeparam name="TResult">The final values to be yielded after processing</typeparam>
/// <typeparam name="TSource">The values to be yielded by the original data-source</typeparam>
/// <param name="comparer">Used to compare grouping keys</param>
/// <param name="keySelector">A function to extract the key for each element in hte original sequence.</param>
/// <param name="resultSelector">A function to create a result value from each group.</param>
/// <param name="source">The data-source to be grouped</param>
/// <remarks>This will force each unique grouping key to
/// be buffered, but not the data itself</remarks>
public static IDataProducer <TResult> GroupBy <TSource, TKey, TResult>
(this IDataProducer <TSource> source,
Func <TSource, TKey> keySelector,
Func <TKey, IDataProducer <TSource>, TResult> resultSelector,
IEqualityComparer <TKey> comparer)
{
return(source.GroupBy(keySelector,
elt => elt,
resultSelector,
comparer));
}
public /*protected*/ override object ProcessData(IDataProducer sender, object data)
{
Utils.ThrowException(!(data is DocumentCorpus) ? new ArgumentTypeException("data") : null);
DocumentCorpus corpus = (DocumentCorpus)data;
foreach (Document document in corpus.Documents)
{
ProcessDocument(document);
}
return(corpus);
}
public void OnRequest(HttpRequestHead request, IDataProducer requestBody,
IHttpResponseDelegate response)
{
var verror = "404 Not Found";
var errorString = "Not Found";
try
{
var path = request.Uri;
if (path == "/api")
{
var method = request.Method;
var stream = new MemoryStream();
if (method == "POST")
{
requestBody.Connect(new BufferedConsumer(bufferedBody =>
{
var data = Encoding.UTF8.GetBytes(Encoding.UTF8.GetString(bufferedBody).Split(new[] { "\r\n\r\n" }, 2, StringSplitOptions.RemoveEmptyEntries)[1]);
stream.Write(data, 0, data.Length);
stream.Position = 0;
HandleAPIRequest(stream, method, response);
}, error =>
{
Log.Error(error.ToString());
}));
}
else
{
HandleAPIRequest(stream, method, response);
}
return;
}
}
catch (Exception ex)
{
verror = "500 Internal Server Error";
Log.Error(ex.ToString());
errorString = ex.ToString();
}
response.OnResponse(new HttpResponseHead()
{
Status = verror,
Headers = new Dictionary <string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", errorString.Length.ToString() }
}
}, new BufferedProducer(errorString));
}
/// <summary>
/// Groups the elements of a sequence according to a specified key selector function
/// and creates a result value from each group and its key. The elements of each
/// group are projected by using a specified function.
/// </summary>
/// <typeparam name="TElement">The return-type of the transform used to process the
/// values within each grouping.</typeparam>
/// <typeparam name="TKey">The return-type of the transform used to group the sequence</typeparam>
/// <typeparam name="TSource">The values to be yielded by the original data-source</typeparam>
/// <param name="comparer">Used to compare grouping keys</param>
/// <param name="elementSelector">A function to map each source element to an element in the appropriate group</param>
/// <param name="keySelector">A function to extract the key for each element in hte original sequence.</param>
/// <param name="source">The data-source to be grouped</param>
/// <remarks>This will force each unique grouping key to
/// be buffered, but not the data itself</remarks>
public static IDataProducer <IProducerGrouping <TKey, TElement> > GroupBy <TSource, TKey, TElement>
(this IDataProducer <TSource> source,
Func <TSource, TKey> keySelector,
Func <TSource, TElement> elementSelector,
IEqualityComparer <TKey> comparer)
{
return(source.GroupBy(keySelector,
elementSelector,
(key, elements) => (IProducerGrouping <TKey, TElement>) new ProducerGrouping <TKey, TElement>(key, elements),
comparer));
}
/// <summary>
/// Create a new OrderedDataProducer
/// </summary>
/// <param name="baseProducer">The base source which will supply data</param>
/// <param name="comparer">The comparer to use when sorting the data (once complete)</param>
public OrderedDataProducer(
IDataProducer <T> baseProducer,
IComparer <T> comparer)
{
baseProducer.ThrowIfNull("baseProducer");
this.baseProducer = baseProducer;
this.comparer = comparer ?? Comparer <T> .Default;
baseProducer.DataProduced += OriginalDataProduced;
baseProducer.EndOfData += EndOfOriginalData;
}
/// <summary>
/// Returns a projection on the data-producer, using a transformation
/// (involving the elements's index in the sequence) to
/// map each element into a new form.
/// </summary>
/// <typeparam name="TSource">The source type</typeparam>
/// <typeparam name="TResult">The projected type</typeparam>
/// <param name="source">The source data-producer</param>
/// <param name="projection">The transformation to apply to each element.</param>
public static IDataProducer <TResult> Select <TSource, TResult>(this IDataProducer <TSource> source, DotNet20.Func <TSource, int, TResult> projection)
{
source.ThrowIfNull("source");
projection.ThrowIfNull("projection");
DataProducer <TResult> ret = new DataProducer <TResult>();
int index = 0;
source.DataProduced += value => ret.Produce(projection(value, index++));
source.EndOfData += () => ret.End();
return(ret);
}
/// <summary>
/// Create a new OrderedDataProducer
/// </summary>
/// <param name="baseProducer">The base source which will supply data</param>
/// <param name="comparer">The comparer to use when sorting the data (once complete)</param>
public OrderedDataProducer(
IDataProducer <T> baseProducer,
IComparer <T> comparer)
{
baseProducer.ThrowIfNull("baseProducer");
BaseProducer = baseProducer;
Comparer = comparer ?? Comparer <T> .Default;
baseProducer.DataProduced += new Action <T>(OriginalDataProduced);
baseProducer.EndOfData += new Action(EndOfOriginalData);
}
public Merge(IDataProducer <VirtualSensorData> idealsensor, IDataProducer <InertialSensorData> actualsensor)
{
this.ActualSensor = actualsensor;
IdealSensor = idealsensor;
ActualSensor.NewTData += MyFunctionToDealWithGettingData;
IdealSensor.NewTData += OnIdealSensorNewTData;
virtualAcc = new CircularBuffer <AccelerationTime>(KinectFPS * CalibrationLookBackTimeInSec); //kinect
actualAcc = new CircularBuffer <AccelerationTime>(InertialMPS * CalibrationLookBackTimeInSec); //sensor
calibrators = new CircularBuffer <SensorCalibrator>(KinectFPS * CalibrationLookBackTimeInSec - 4); //calibrators
finalCalibrator = new SensorCalibrator();
}
public void PumpWithEarlyStop()
{
DataProducer <int> producer = new DataProducer <int>();
IDataProducer <int> take3 = producer.Take(3);
// This will show us everything produced
List <int> allResults = producer.ToList();
var result = producer.PumpProduceAndEnd(new int[] { 1, 2, 3, 4, 5 }, take3);
Assert.IsTrue(result.SequenceEqual(new int[] { 1, 2, 3 }));
Assert.IsTrue(allResults.SequenceEqual(new int[] { 1, 2, 3 }));
}
public void OnRequest(HttpRequestHead head, IDataProducer body, IHttpResponseDelegate response)
{
var env = new Dictionary <string, object>();
var request = new RequestEnvironment(env);
if (context != null)
{
foreach (var kv in context)
{
env[kv.Key] = kv.Value;
}
}
if (head.Headers == null)
{
env[OwinConstants.RequestHeaders] = new Dictionary <string, string[]>(StringComparer.OrdinalIgnoreCase);
}
else
{
env[OwinConstants.RequestHeaders] = head.Headers.ToDictionary(kv => kv.Key, kv => new[] { kv.Value }, StringComparer.OrdinalIgnoreCase);
}
env[OwinConstants.ResponseHeaders] = new Dictionary <string, string[]>(StringComparer.OrdinalIgnoreCase);
request.Method = head.Method ?? "";
request.Path = head.Path ?? "";
request.PathBase = "";
request.QueryString = head.QueryString ?? "";
request.Scheme = "http"; // XXX
request.Version = "1.0";
if (body == null)
{
env[OwinConstants.RequestBody] = Stream.Null;
}
/*
* else
* request.Body = (write, end, cancellationToken) =>
* {
* var d = body.Connect(new DataConsumer(
* write,
* end,
* () => end(null)));
* cancellationToken.Register(d.Dispose);
* };
*/// TODO: Request body stream
appFunc(env)
.Then(() => HandleResponse(response, env))
.Catch(errorInfo => HandleError(response, errorInfo));
}
public void WhereNoResults()
{
DataProducer <string> subject = new DataProducer <string>();
IDataProducer <string> result = subject.Where(x => x.Length >= 3);
List <string> output = new List <string>();
bool gotEnd = false;
result.DataProduced += value => output.Add(value);
result.EndOfData += () => gotEnd = true;
subject.ProduceAndEnd("a", "b", "c", "d");
Assert.IsTrue(gotEnd);
Assert.AreEqual(0, output.Count);
}
public void SetUp()
{
_processor = new RequestProcessor(_ruleThatReturnsFirstHandler, new RequestHandlerList());
_requestHandlerFactory = new RequestHandlerFactory(_processor);
_dataProducer = MockRepository.GenerateStub <IDataProducer>();
_httpResponseDelegate = MockRepository.GenerateStub <IHttpResponseDelegate>();
_ruleThatReturnsFirstHandler = MockRepository.GenerateStub <IMatchingRule>();
_ruleThatReturnsFirstHandler.Stub(x => x.IsEndpointMatch(null, new HttpRequestHead())).IgnoreArguments().Return(true).Repeat.Once();
_ruleThatReturnsNoHandlers = MockRepository.GenerateStub <IMatchingRule>();
_ruleThatReturnsNoHandlers.Stub(x => x.IsEndpointMatch(null, new HttpRequestHead())).IgnoreArguments().Return(false);
}
public void OnResponse(HttpResponseHead head, IDataProducer body)
{
var writeResponse = false;
var hasBody = body != null;
state.OnResponse(hasBody, out writeResponse);
this.head = head;
this.body = body;
if (writeResponse)
BeginResponse();
}
public void OnResponse(HttpResponseHead head, IDataProducer body)
{
var begin = false;
var hasBody = body != null;
state.OnResponse(hasBody, out begin);
this.head = head;
this.body = body;
if (begin)
Begin();
}
/// <summary>
/// Converts an IDataProducer into an IFuture[IEnumerable]. The results
/// are buffered in memory (as a list), so be warned that this loses the "streaming"
/// nature of most of the IDataProducer extension methods. The "future" nature of
/// the result ensures that all results are produced before the enumeration can take
/// place.
/// </summary>
/// <remarks>This will force all values to be buffered</remarks>
public static IFuture <IEnumerable <TSource> > AsFutureEnumerable <TSource>(this IDataProducer <TSource> source)
{
source.ThrowIfNull("source");
Future <IEnumerable <TSource> > ret = new Future <IEnumerable <TSource> >();
List <TSource> list = new List <TSource>();
source.DataProduced += value => list.Add(value);
source.EndOfData += () => ret.Value = list;
return(ret);
}
protected override void ConsumeData(IDataProducer sender, object data)
{
Utils.ThrowException(!(data is DocumentCorpus) ? new ArgumentTypeException("data") : null);
StringWriter stringWriter;
XmlWriterSettings xmlSettings = new XmlWriterSettings();
xmlSettings.Indent = true;
xmlSettings.NewLineOnAttributes = true;
xmlSettings.CheckCharacters = false;
XmlWriter writer = XmlWriter.Create(stringWriter = new StringWriter(), xmlSettings);
((DocumentCorpus)data).WriteXml(writer, /*writeTopElement=*/true);
writer.Close();
// send message
mMessenger.sendMessage(stringWriter.ToString());
}
/// <summary>
/// Converts an IDataProducer into a dictionary.
/// </summary>
/// <param name="elementSelector">A transform function to produce a result element value from each element.</param>
/// <param name="keySelector">A function to extract a key from each element.</param>
/// <param name="keyComparer">Used to compare keys.</param>
/// <param name="source">The data source.</param>
/// <remarks>This will force all values to be buffered</remarks>
public static IDictionary <TKey, TElement> ToDictionary <TSource, TKey, TElement>(
this IDataProducer <TSource> source,
Func <TSource, TKey> keySelector, Func <TSource, TElement> elementSelector, IEqualityComparer <TKey> keyComparer)
{
source.ThrowIfNull("source");
keySelector.ThrowIfNull("keySelector");
elementSelector.ThrowIfNull("elementSelector");
keyComparer.ThrowIfNull("keyComparer");
Dictionary <TKey, TElement> dict = new Dictionary <TKey, TElement>(keyComparer);
source.DataProduced += t => dict.Add(keySelector(t), elementSelector(t));
return(dict);
}
public FileLogger(IDataProducer datareciever, string filelocation, bool startloggingnow = false)
{
DataReciever = datareciever;
FileLocation = filelocation;
if (FileLocation == null)
{
throw new NullReferenceException("Must set filelocation");
}
OStream = new StreamWriter(filelocation, false);
if (startloggingnow)
{
((ILogger)this).StartLogging();
}
}
/// <summary>
/// Returns a future to the sum of a sequence of values that are
/// obtained by taking a transform of the input sequence
/// </summary>
/// <remarks>Null values are removed from the sum</remarks>
public static IFuture <TResult> Sum <TSource, TResult>(this IDataProducer <TSource> source, Func <TSource, TResult> selector)
{
source.ThrowIfNull("source");
selector.ThrowIfNull("selector");
Future <TResult> ret = new Future <TResult>();
TResult sum = Operator <TResult> .Zero;
source.DataProduced += item => {
Operator.AddIfNotNull(ref sum, selector(item));
};
source.EndOfData += () => ret.Value = sum;
return(ret);
}
private void ProcessClientAccessPolicyRequest(IDataProducer body, IHttpResponseDelegate response)
{
var responseHead = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary <string, string>
{
{ "Content-Type", "text/xml" },
{ "Content-Length", Encoding.UTF8.GetByteCount(SILVERLIGHT_CROSSDOMAIN).ToString() }
}
};
response.OnResponse(responseHead, new BufferedProducer(SILVERLIGHT_CROSSDOMAIN));
}
public void Handle(string appname, string[] components, HttpRequestHead head, IDataProducer body, IHttpResponseDelegate response)
{
// Look inside the cache for the specified file.
Cache c = new Cache(false);
string path = HttpUtility.UrlDecode(components.Where((value, row) => row >= 2).Aggregate((a, b) => a + "/" + b));
if (!c.Exists("server/" + appname + "/store/" + path))
{
response.OnResponse(HttpErrorResponseHead.Get(), new HttpErrorDataProducer());
return;
}
// Calculate patch path from source to destination.
string result = "";
Hash source = Hash.FromString(components[0]);
Hash destination = Hash.FromString(components[1]);
while (source != destination)
{
// Find the patch in the patches that will turn source
// into the next patch.
IEnumerable<string> patches = c.List("server/" + appname + "/patches/" + path).Where(v => v.StartsWith(source.ToString() + "-"));
if (patches.Count() != 1)
{
response.OnResponse(HttpErrorResponseHead.Get(), new HttpErrorDataProducer());
return;
}
string next = patches.First();
source = Hash.FromString(next.Substring((source.ToString() + "-").Length));
using (StreamReader reader = new StreamReader(c.GetFilePath("server/" + appname + "/patches/" + path + "/" + next)))
{
result += "--- NEXT PATCH (" + reader.BaseStream.Length + ") ---\r\n";
result += reader.ReadToEnd();
result += "\r\n";
}
}
result += "--- END OF PATCHES ---\r\n";
// Return data.
response.OnResponse(new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>
{
{ "Content-Type", "text/plain" },
{ "Content-Length", result.Length.ToString() },
{ "Connection", "close" }
}
}, new BufferedProducer(result));
}
//private int CalibrationCount = 0; //locally stored count on the number of times calibrator has been called.
public Merge(IDataProducer <VirtualSensorData> idealsensor, IDataProducer <InertialSensorData> actualsensor, int count)
{
//CalibrationCount = count;
this.ActualSensor = actualsensor;
IdealSensor = idealsensor;
ActualSensor.NewTData += OnActualSensorNewTData;
IdealSensor.NewTData += OnIdealSensorNewTData;
virtualAcc = new CircularBuffer <AccelerationTime>((int)(KinectFPS * CalibrationLookBackTimeInSec)); //kinect
actualAcc = new CircularBuffer <AccelerationTime>((int)(InertialMPS * (CalibrationLookBackTimeInSec + 1))); //sensor
calibrators = new CircularBuffer <SensorCalibratorData>((int)(KinectFPS * CalibrationLookBackTimeInSec - 4)); //calibrators
finalCalibrator = new SensorCalibratorData();
//DataTracker.CurrentSection = 999;
}
public void OnResponse(HttpResponseHead head, IDataProducer body)
{
var writeResponse = false;
var hasBody = body != null;
state.OnResponse(hasBody, out writeResponse);
this.head = head;
this.body = body;
if (writeResponse)
{
BeginResponse();
}
}
/// <summary>
/// Converts an IDataProducer into a lookup.
/// </summary>
/// <param name="elementSelector">A transform function to produce a result element value from each element.</param>
/// <param name="keySelector">A function to extract a key from each element.</param>
/// <param name="keyComparer">Used to compare keys.</param>
/// <param name="source">The data source.</param>
/// <remarks>This will force all values to be buffered</remarks>
public static ILookup <TKey, TElement> ToLookup <TSource, TKey, TElement>(
this IDataProducer <TSource> source,
Func <TSource, TKey> keySelector, Func <TSource, TElement> elementSelector, IEqualityComparer <TKey> keyComparer)
{
source.ThrowIfNull("source");
keySelector.ThrowIfNull("keySelector");
elementSelector.ThrowIfNull("elementSelector");
keyComparer.ThrowIfNull("keyComparer");
EditableLookup <TKey, TElement> lookup = new EditableLookup <TKey, TElement>(keyComparer);
source.DataProduced += t => lookup.Add(keySelector(t), elementSelector(t));
source.EndOfData += () => lookup.TrimExcess();
return(lookup);
}
private void ProcessCrossDomainRequest(IDataProducer body, IHttpResponseDelegate response)
{
var responseHead = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary <string, string>
{
{ "Content-Type", "application-xml" },
{ "Content-Length", Encoding.UTF8.GetByteCount(FLASH_CROSSDOMAIN).ToString() },
{ "Access-Control-Allow-Origin", "*" }
}
};
response.OnResponse(responseHead, new BufferedProducer(FLASH_CROSSDOMAIN));
}
public static void DispatchData(IDataProducer producer, object data, bool cloneDataOnFork, DispatchPolicy dispatchPolicy,
Set<IDataConsumer> dataConsumers, Logger logger)
{
if (data == null) { return; }
if (dataConsumers.Count == 0)
{
if (logger != null) { logger.Warn("DispatchData", "Data ready but nobody is listening."); }
return;
}
if (dispatchPolicy == DispatchPolicy.BalanceLoadSum || dispatchPolicy == DispatchPolicy.BalanceLoadMax)
{
if (logger != null) { logger.Trace("DispatchData", "Dispatching data of type {0} (load balancing) ...", data.GetType()); }
int minLoad = int.MaxValue;
IDataConsumer target = null;
foreach (IDataConsumer consumer in dataConsumers)
{
int load = (dispatchPolicy == DispatchPolicy.BalanceLoadSum) ? GetBranchLoadSum(consumer) : GetBranchLoadMax(consumer);
if (load < minLoad) { minLoad = load; target = consumer; }
}
target.ReceiveData(producer, data);
}
else if (dispatchPolicy == DispatchPolicy.Random)
{
if (logger != null) { logger.Trace("DispatchData", "Dispatching data of type {0} (random policy) ...", data.GetType()); }
ArrayList<IDataConsumer> tmp = new ArrayList<IDataConsumer>(dataConsumers.Count);
foreach (IDataConsumer dataConsumer in dataConsumers) { tmp.Add(dataConsumer); }
tmp[mRandom.Next(0, tmp.Count)].ReceiveData(producer, data);
}
else
{
if (logger != null) { logger.Trace("DispatchData", "Dispatching data of type {0} (to-all policy) ...", data.GetType()); }
if (dataConsumers.Count > 1 && cloneDataOnFork)
{
foreach (IDataConsumer dataConsumer in dataConsumers)
{
dataConsumer.ReceiveData(producer, Utils.Clone(data, /*deepClone=*/true));
}
}
else
{
foreach (IDataConsumer dataConsumer in dataConsumers)
{
dataConsumer.ReceiveData(producer, data);
}
}
}
if (logger != null) { logger.Trace("DispatchData", "Data dispatched."); }
}
private void ProcessOPTIONSRequest(HttpRequestHead head, IDataProducer body, IHttpResponseDelegate response)
{
var responseHead = new HttpResponseHead()
{
Status = "200 OK",
Headers = _corsValidator.AppendCorsHeaderDictionary(
head,
new Dictionary <string, string>
{
{ "Content-Type", "text/plain" },
{ "Content-Length", "0" }
})
};
response.OnResponse(responseHead, new EmptyResponse());
}
public void WriteResponse(HttpResponseHead head, IDataProducer body)
{
if (gotResponse)
{
throw new InvalidOperationException("WriteResponse was previously called.");
}
gotResponse = true;
this.head = head;
this.body = body;
if (transaction != null)
{
DoWriteResponse();
}
}
private void ProcessFileNotFound(IDataProducer body, IHttpResponseDelegate response)
{
_parent._systemMetrics.LogCount("listeners.http.404");
var headers = new HttpResponseHead()
{
Status = "404 Not Found",
Headers = new Dictionary <string, string>
{
{ "Content-Type", "text/plain" },
{ "Content-Length", Encoding.UTF8.GetByteCount("not found").ToString() },
{ "Access-Control-Allow-Origin", "*" }
}
};
response.OnResponse(headers, new BufferedProducer("not found"));
}
public void OnRequest(HttpRequestHead request, IDataProducer body, IHttpResponseDelegate response)
{
_log.DebugFormat("Start Processing request for : {0}:{1}", request.Method, request.Uri);
if (GetHandlerCount() < 1) {
ReturnHttpMockNotFound(response);
return;
}
var handler = _requestMatcher.Match(request, _handlers);
if (handler == null) {
_log.DebugFormat("No Handlers matched");
ReturnHttpMockNotFound(response);
return;
}
HandleRequest(request, body, response, handler);
}
private void ProcessOPTIONSRequest(IDataProducer body, IHttpResponseDelegate response)
{
var responseHead = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary <string, string>
{
{ "Content-Type", "text/plain" },
{ "Content-Length", "0" },
{ "Access-Control-Allow-Origin", "*" },
{ "Access-Control-Allow-Methods", "GET, POST, OPTIONS" },
{ "Access-Control-Allow-Headers", "X-Requested-With,Content-Type" }
}
};
response.OnResponse(responseHead, new EmptyResponse());
}
private static void ApiPost(HttpRequestHead request, IDataProducer requestBody, IHttpResponseDelegate response)
{
const string pattern = @"^/api/([A-Za-z]+)/.*$";
Match match = Regex.Match(request.Uri, pattern);
if (match.Success)
{
IController controller = ControllerFactory.Get(match.Groups[1].Value);
if (controller != null)
{
controller.ProcessPost(request, requestBody, response);
return;
}
}
NotFoundResponse(request, response);
}
public void Handle(string appname, string[] components, HttpRequestHead head, IDataProducer body, IHttpResponseDelegate response)
{
// Look inside the cache for a list of files.
Cache c = new Cache(false);
string s = "";
foreach (string key in c.ListRecursive("server/" + appname + "/hashes"))
s += c.Get<Hash>("server/" + appname + "/hashes/" + key) + " " + key + "\r\n";
response.OnResponse(new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>
{
{ "Content-Type", "text/plain" },
{ "Content-Length", s.Length.ToString() },
{ "Connection", "close" }
}
}, new BufferedProducer(s));
}
public void OnRequest(HttpRequestHead request, IDataProducer requestBody, IHttpResponseDelegate response)
{
if (request.Method.ToUpperInvariant() == "POST" && request.Uri.ToLower().StartsWith("/api"))
{
ApiPost(request, requestBody, response);
} else if (request.Uri.StartsWith("/api"))
{
ApiGet(request, response);
} else if (request.Uri.StartsWith("/resx"))
{
GetResource(request, response);
} else if (request.Uri.StartsWith("/"))
{
GetResourceByName("vixen.htm", response);
} else
{
NotFoundResponse(request, response);
}
}
public void Handle(string appname, string[] components, HttpRequestHead head, IDataProducer body, IHttpResponseDelegate response)
{
// Look inside the cache for the specified file.
Cache c = new Cache(false);
string path = HttpUtility.UrlDecode(components.Aggregate((a, b) => a + "/" + b));
if (!c.Exists("server/" + appname + "/store/" + path))
{
response.OnResponse(HttpErrorResponseHead.Get(), new HttpErrorDataProducer());
return;
}
response.OnResponse(new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>
{
{ "Connection", "close" }
}
}, new StreamReaderProducer(c.GetFilePath("server/" + appname + "/store/" + path)));
}
public void OnRequest(HttpRequestHead head, IDataProducer body, IHttpResponseDelegate response)
{
var env = new Dictionary<string, object>();
var request = new RequestEnvironment(env);
if (context != null)
foreach (var kv in context)
env[kv.Key] = kv.Value;
if (head.Headers == null)
env[OwinConstants.RequestHeaders] = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
else
env[OwinConstants.RequestHeaders] = head.Headers.ToDictionary(kv => kv.Key, kv => new[] { kv.Value }, StringComparer.OrdinalIgnoreCase);
env[OwinConstants.ResponseHeaders] = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
request.Method = head.Method ?? "";
request.Path = head.Path ?? "";
request.PathBase = "";
request.QueryString = head.QueryString ?? "";
request.Scheme = "http"; // XXX
request.Version = "1.0";
if (body == null)
env[OwinConstants.RequestBody] = Stream.Null;
/*
else
request.Body = (write, end, cancellationToken) =>
{
var d = body.Connect(new DataConsumer(
write,
end,
() => end(null)));
cancellationToken.Register(d.Dispose);
};
*/ // TODO: Request body stream
appFunc(env)
.Then(() => HandleResponse(response, env))
.Catch(errorInfo => HandleError(response, errorInfo));
}
public void OnRequest(HttpRequestHead request, IDataProducer requestBody, IHttpResponseDelegate response)
{
var ea = new RequestReceivedEventArgs();
ea.RequestHead = request;
ea.ResponseHead = ResponseMessageHelper.GetHttpResponseHead();
string contentType = string.Empty;
if(ea.RequestHead.Headers.ContainsKey("Content-Type"))
contentType=ea.RequestHead.Headers["Content-Type"];
int contentSize = 0;
if(ea.RequestHead.Headers.ContainsKey("Content-Length"))
{
int.TryParse(ea.RequestHead.Headers["Content-Length"],out contentSize);
}
BufferedConsumer bc=new BufferedConsumer(bodyContents =>
{
try
{
ea.RequestBody = bodyContents;
//Called when request body is read to end
if (RequestReceived != null)
{
RequestReceived(this, ea);
}
var bp = ea.ResponseBodyProducer as BufferedProducer;
if (bp != null)
{
ea.ResponseHead.Headers["Content-Length"] = bp.GetContentLength().ToString();
}
}
finally
{
response.OnResponse(ea.ResponseHead, ea.ResponseBodyProducer);
}
}, error =>
{
}, contentType,contentSize);
//Gets complete HTTP Request and runs code defined over
requestBody.Connect(bc);
}
public void OnRequest(HttpRequestHead request, IDataProducer body, IHttpResponseDelegate response)
{
_log.DebugFormat("Start Processing request for : {0}:{1}", request.Method, request.Uri);
if (GetHandlerCount() < 1) {
ReturnHttpMockNotFound(response);
return;
}
RequestHandler handler = MatchHandler(request);
if (handler == null) {
_log.DebugFormat("No Handlers matched");
ReturnHttpMockNotFound(response);
return;
}
_log.DebugFormat("Matched a handler {0},{1}, {2}", handler.Method, handler.Path, DumpQueryParams(handler.QueryParams));
IDataProducer dataProducer = GetDataProducer(request, handler);
if (request.HasBody())
{
body.Connect(new BufferedConsumer(
bufferedBody =>
{
handler.RecordRequest(request, bufferedBody);
_log.DebugFormat("Body: {0}", bufferedBody);
response.OnResponse(handler.ResponseBuilder.BuildHeaders(), dataProducer);
},
error =>
{
_log.DebugFormat("Error while reading body {0}", error.Message);
response.OnResponse(handler.ResponseBuilder.BuildHeaders(), dataProducer);
}
));
} else {
response.OnResponse(handler.ResponseBuilder.BuildHeaders(), dataProducer);
handler.RecordRequest(request, null);
}
_log.DebugFormat("End Processing request for : {0}:{1}", request.Method, request.Uri);
}
public MessageConsumer(IOutputSegment previous, IDataProducer message) : base(previous)
{
this.message = message;
}
public void OnRequest(HttpRequestHead request, IDataProducer requestBody,
IHttpResponseDelegate response)
{
if (request.Method.ToUpperInvariant() == "POST" && request.Uri.StartsWith("/bufferedecho"))
{
// when you subecribe to the request body before calling OnResponse,
// the server will automatically send 100-continue if the client is
// expecting it.
requestBody.Connect(new BufferedConsumer(bufferedBody =>
{
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", request.Headers["Content-Length"] },
{ "Connection", "close" }
}
};
response.OnResponse(headers, new BufferedProducer(bufferedBody));
}, error =>
{
// XXX
// uh oh, what happens?
}));
}
else if (request.Method.ToUpperInvariant() == "POST" && request.Uri.StartsWith("/echo"))
{
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Connection", "close" }
}
};
if (request.Headers.ContainsKey("Content-Length"))
headers.Headers["Content-Length"] = request.Headers["Content-Length"];
// if you call OnResponse before subscribing to the request body,
// 100-continue will not be sent before the response is sent.
// per rfc2616 this response must have a 'final' status code,
// but the server does not enforce it.
response.OnResponse(headers, requestBody);
}
else if (request.Uri.StartsWith("/"))
{
var body = string.Format(
"Hello world.\r\nHello.\r\n\r\nUri: {0}\r\nPath: {1}\r\nQuery:{2}\r\nFragment: {3}\r\n",
request.Uri,
request.Path,
request.QueryString,
request.Fragment);
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else
{
var responseBody = "The resource you requested ('" + request.Uri + "') could not be found.";
var headers = new HttpResponseHead()
{
Status = "404 Not Found",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", responseBody.Length.ToString() }
}
};
var body = new BufferedProducer(responseBody);
response.OnResponse(headers, body);
}
}
public void OnResponse(HttpResponseHead head, IDataProducer body)
{
Head = head;
Body = body;
}
public void OnResponse(HttpResponseHead head, IDataProducer body)
{
// XXX still need to better account for Connection: close.
// this should cause the queue to drop pending responses
// perhaps segment.Abort which disposes transation
if (!shouldKeepAlive)
{
if (head.Headers == null)
head.Headers = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase);
head.Headers["Connection"] = "close";
}
Segment.WriteResponse(head, ignoreResponseBody ? null : body);
}
public void OnRequest(HttpRequestHead request, IDataProducer requestBody, IHttpResponseDelegate response)
{
if (request.Method.ToUpperInvariant() == "GET" && request.Uri.StartsWith("/next"))
{
// when you subscribe to the request body before calling OnResponse,
// the server will automatically send 100-continue if the client is
// expecting it.
bool ret = MainWindow.Next();
var body = ret ? "Successfully skipped." : "You have to wait for 20 seconds to skip again.";
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else if (request.Method.ToUpperInvariant() == "GET" && request.Uri.StartsWith("/pause"))
{
MainWindow.Pause();
var body = "Paused.";
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else if (request.Method.ToUpperInvariant() == "GET" && request.Uri.StartsWith("/play"))
{
MainWindow.Play();
var body = "Playing.";
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else if (request.Method.ToUpperInvariant() == "GET" && request.Uri.StartsWith("/toggleplaypause"))
{
var body = "";
if (MainWindow._player.Playing)
{
body = "Paused.";
}
else
{
body = "Playing.";
}
MainWindow.PlayPauseToggle();
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else if (request.Method.ToUpperInvariant() == "GET" && request.Uri.StartsWith("/like"))
{
MainWindow.Like();
var body = "Like";
if (MainWindow.GetCurrentSong().Loved)
body = "Liked";
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else if (request.Method.ToUpperInvariant() == "GET" && request.Uri.StartsWith("/dislike"))
{
MainWindow.Dislike();
var body = "Disliked.";
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else if (request.Method.ToUpperInvariant() == "GET" && request.Uri.StartsWith("/currentsong"))
{
Song s = MainWindow.GetCurrentSong();
var body = new JavaScriptSerializer().Serialize(s);
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else if (request.Method.ToUpperInvariant() == "GET" && request.Uri.StartsWith("/connect"))
{
var body = "true";
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else if (request.Uri.StartsWith("/"))
{
var body = string.Format(
"Hello world.\r\nHello.\r\n\r\nUri: {0}\r\nPath: {1}\r\nQuery:{2}\r\nFragment: {3}\r\n",
request.Uri,
request.Path,
request.QueryString,
request.Fragment);
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", body.Length.ToString() },
}
};
response.OnResponse(headers, new BufferedProducer(body));
}
else
{
var responseBody = "The resource you requested ('" + request.Uri + "') could not be found.";
var headers = new HttpResponseHead()
{
Status = "404 Not Found",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", responseBody.Length.ToString() }
}
};
var body = new BufferedProducer(responseBody);
response.OnResponse(headers, body);
}
}
public void SetUp()
{
_processor = new RequestProcessor(_ruleThatReturnsFirstHandler, new RequestHandlerList());
_requestHandlerFactory = new RequestHandlerFactory(_processor);
_dataProducer = MockRepository.GenerateStub<IDataProducer>();
_httpResponseDelegate = MockRepository.GenerateStub<IHttpResponseDelegate>();
_ruleThatReturnsFirstHandler = MockRepository.GenerateStub<IMatchingRule>();
_ruleThatReturnsFirstHandler.Stub(x => x.IsEndpointMatch(null, new HttpRequestHead())).IgnoreArguments().Return(true).Repeat.Once();
_ruleThatReturnsNoHandlers = MockRepository.GenerateStub<IMatchingRule>();
_ruleThatReturnsNoHandlers.Stub(x => x.IsEndpointMatch(null, new HttpRequestHead())).IgnoreArguments().Return(false);
}
public void OnRequest(HttpRequestHead request, IDataProducer requestBody,
IHttpResponseDelegate response)
{
if (request.Uri == "/")
{
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", "20" },
}
};
var body = new BufferedProducer("Hello world.\r\nHello.");
response.OnResponse(headers, body);
}
else if (request.Uri == "/bufferedecho")
{
// when you subecribe to the request body before calling OnResponse,
// the server will automatically send 100-continue if the client is
// expecting it.
requestBody.Connect(new BufferedConsumer(bufferedBody =>
{
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", request.Headers["Content-Length"] },
{ "Connection", "close" }
}
};
response.OnResponse(headers, new BufferedProducer(bufferedBody));
}, error =>
{
// XXX
// uh oh, what happens?
}));
}
else if (request.Uri == "/echo")
{
var headers = new HttpResponseHead()
{
Status = "200 OK",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", request.Headers["Content-Length"] },
{ "Connection", "close" }
}
};
// if you call OnResponse before subscribing to the request body,
// 100-continue will not be sent before the response is sent.
// per rfc2616 this response must have a 'final' status code,
// but the server does not enforce it.
response.OnResponse(headers, requestBody);
}
else
{
var responseBody = "The resource you requested ('" + request.Uri + "') could not be found.";
var headers = new HttpResponseHead()
{
Status = "404 Not Found",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "text/plain" },
{ "Content-Length", responseBody.Length.ToString() }
}
};
var body = new BufferedProducer(responseBody);
response.OnResponse(headers, body);
}
}
protected override void ConsumeData(IDataProducer sender, object data)
{
Utils.ThrowException(!(data is DocumentCorpus) ? new ArgumentTypeException("data") : null);
DocumentCorpus corpus = (DocumentCorpus)data;
string corpusId = corpus.Features.GetFeatureValue("guid").Replace("-", "");
StringWriter stringWriter;
XmlWriterSettings xmlSettings = new XmlWriterSettings();
xmlSettings.Indent = true;
xmlSettings.NewLineOnAttributes = true;
xmlSettings.CheckCharacters = false;
XmlWriter writer = XmlWriter.Create(stringWriter = new StringWriter(), xmlSettings);
corpus.WriteXml(writer, /*writeTopElement=*/true);
writer.Close();
DateTime timeEnd = DateTime.Parse(corpus.Features.GetFeatureValue("timeEnd"));
string recordId = timeEnd.ToString("HH_mm_ss_") + corpusId;
// write to file
if (mXmlDataRoot != null)
{
string path = string.Format(@"{3}\{0}\{1}\{2}\", timeEnd.Year, timeEnd.Month, timeEnd.Day, mXmlDataRoot.TrimEnd('\\'));
if (!Directory.Exists(path))
{
lock (mLock)
{
if (!Directory.Exists(path)) { Directory.CreateDirectory(path); }
}
}
StreamWriter w = new StreamWriter(path + recordId + ".xml", /*append=*/false, Encoding.UTF8);
w.Write(stringWriter.ToString().Replace("<?xml version=\"1.0\" encoding=\"utf-16\"?>", "<?xml version=\"1.0\" encoding=\"utf-8\"?>"));
w.Close();
}
// write to database
if (mWriteToDatabase)
{
bool success = mConnection.ExecuteNonQuery("insert into Corpora (id, title, language, sourceUrl, timeStart, timeEnd, siteId, rejected) values (?, ?, ?, ?, ?, ?, ?, ?)",
corpusId,
Utils.Truncate(corpus.Features.GetFeatureValue("title"), 400),
Utils.Truncate(corpus.Features.GetFeatureValue("language"), 100),
Utils.Truncate(corpus.Features.GetFeatureValue("sourceUrl"), 400),
Utils.Truncate(corpus.Features.GetFeatureValue("timeStart"), 26),
Utils.Truncate(corpus.Features.GetFeatureValue("timeEnd"), 26),
Utils.Truncate(corpus.Features.GetFeatureValue("siteId"), 100),
mIsDumpWriter
);
if (!success) { mLogger.Warn("ConsumeData", "Unable to write to database."); }
foreach (Document document in corpus.Documents)
{
string documentId = new Guid(document.Features.GetFeatureValue("guid")).ToString("N");
string bpCharCountStr = document.Features.GetFeatureValue("bprBoilerplateCharCount");
string contentCharCountStr = document.Features.GetFeatureValue("bprContentCharCount");
string unseenContentCharCountStr = document.Features.GetFeatureValue("unseenContentCharCount");
string unseenContent = document.Features.GetFeatureValue("unseenContent");
success = mConnection.ExecuteNonQuery("insert into Documents (id, corpusId, name, description, category, link, responseUrl, urlKey, time, pubDate, mimeType, contentType, charSet, contentLength, detectedLanguage, detectedCharRange, domain, bpCharCount, contentCharCount, rejected, unseenContent, unseenContentCharCount, rev) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
documentId,
corpusId,
Utils.Truncate(document.Name, 400),
Utils.Truncate(document.Features.GetFeatureValue("description"), 400),
Utils.Truncate(document.Features.GetFeatureValue("category"), 400),
Utils.Truncate(document.Features.GetFeatureValue("link"), 400),
Utils.Truncate(document.Features.GetFeatureValue("responseUrl"), 400),
Utils.Truncate(document.Features.GetFeatureValue("urlKey"), 400),
Utils.Truncate(document.Features.GetFeatureValue("time"), 26),
Utils.Truncate(document.Features.GetFeatureValue("pubDate"), 26),
Utils.Truncate(document.Features.GetFeatureValue("mimeType"), 80),
Utils.Truncate(document.Features.GetFeatureValue("contentType"), 40),
Utils.Truncate(document.Features.GetFeatureValue("charSet"), 40),
Convert.ToInt32(document.Features.GetFeatureValue("contentLength")),
Utils.Truncate(document.Features.GetFeatureValue("detectedLanguage"), 100),
Utils.Truncate(document.Features.GetFeatureValue("detectedCharRange"), 100),
Utils.Truncate(document.Features.GetFeatureValue("domainName"), 100),
bpCharCountStr == null ? null : (object)Convert.ToInt32(bpCharCountStr),
contentCharCountStr == null ? null : (object)Convert.ToInt32(contentCharCountStr),
mIsDumpWriter,
Utils.Truncate(unseenContent, 20),
unseenContentCharCountStr == null ? null : (object)Convert.ToInt32(unseenContentCharCountStr),
Convert.ToInt32(document.Features.GetFeatureValue("rev"))
);
if (!success) { mLogger.Warn("ConsumeData", "Unable to write to database."); }
}
}
}
void DoConnect()
{
if (message != null)
{
var m = message;
message = null;
abortMessage = m.Connect(this);
}
}
