protected void OnDataReceived(DynamicActivityBase activity)
{
if (DataReceived != null)
{
DataReceived(this, activity);
}
}
public override void BeginStreaming()
{
HttpWebRequest request = GetStreamRequest();
Stream stream = null;
StreamReader reader = null;
try
{
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
stream = (Connection.UseEncoding) ? new GZipStream(response.GetResponseStream(), CompressionMode.Decompress) : response.GetResponseStream();
byte[] buffer = new byte[8192];
List <byte> output = new List <byte>();
while (stream.CanRead)
{
Thread.Sleep(1000);
if (response.StatusCode != HttpStatusCode.OK)
{
throw new StreamingException(response.StatusCode, response.StatusDescription);
}
int readCount = stream.Read(buffer, 0, buffer.Length);
output.AddRange(buffer.Take(readCount));
string outputString = Encoding.UTF8.GetString(output.ToArray());
if (!Formatter.HasDelimiter(outputString))
{
continue;
}
int delimiterIndex = Formatter.FindLastDelimiter(outputString);
outputString = outputString.Substring(0, delimiterIndex);
output.RemoveRange(0, Encoding.UTF8.GetByteCount(outputString));
if (String.IsNullOrEmpty(outputString))
{
continue;
}
outputString = Formatter.Normalize(outputString);
OnHeartbeatReceived(new HeartbeatEventArgs()
{
Message = outputString
});
string[] lines = outputString.Split(new[] { Environment.NewLine }, new StringSplitOptions());
for (int i = 0; i < lines.Length; i++)
{
string heartBeatCheck = lines[i];
if (heartBeatCheck.Trim().Length > 0)
{
if (!Formatter.IsValid(heartBeatCheck))
{
continue;
}
Type contentType = GnipStreamProcessorBase.GetObjectTypeByService(Connection.DataSource);
try
{
DynamicActivityBase activity = null;
if (Connection.DataFormat == GnipDataFormat.XML)
{
MemoryStream memStr = new MemoryStream(Encoding.UTF8.GetBytes(heartBeatCheck));
XmlTextReader xmlReader = new XmlTextReader(memStr);
DynamicXMLObjectConverter converter = new DynamicXMLObjectConverter();
dynamic obj = converter.Deserialize(xmlReader, typeof(DynamicXMLObject));
activity = ReflectionHelper.CreateAndCastInstance(contentType, obj);
}
else
{
JavaScriptSerializer serializer = new JavaScriptSerializer();
serializer.RegisterConverters(new[] { new DynamicJsonObjectConverter() });
dynamic obj = serializer.Deserialize(heartBeatCheck, typeof(DynamicJsonObject));
activity = ReflectionHelper.CreateAndCastInstance(contentType, obj);
}
OnDataReceived(activity);
}
catch (Exception ex)
{
OnErrorHappened(ex);
}
}
}
}
}
catch (WebException ex)
{
string status = "";
string message = "";
if (ex.Status == WebExceptionStatus.ProtocolError)
{
HttpWebResponse response = (HttpWebResponse)ex.Response;
using (response)
{
status = ((int)response.StatusCode).ToString();
message = response.StatusDescription;
stream = response.GetResponseStream();
if (stream != null && stream.CanRead)
{
reader = new StreamReader(stream);
message = reader.ReadToEnd();
OnErrorHappened(ex);
}
}
}
else
{
OnErrorHappened(ex);
}
}
catch (Exception ex)
{
OnErrorHappened(ex);
}
finally
{
if (stream != null)
{
stream.Close();
}
if (reader != null)
{
reader.Close();
reader = null;
}
}
}
private void processor_DataReceived(object sender, DynamicActivityBase activity)
{
GnipStreamProcessorAsync processor = sender as GnipStreamProcessorAsync;
if (processor == null)
{
return;
}
switch (processor.Connection.DataSource)
{
case GnipSources.Twitter:
{
List <string> row = new List <string>(_headers.Count);
foreach (KeyValuePair <string, string> title in _headers)
{
object value = ReflectionHelper.GetNestedPropertyValue(title.Key, activity);
if (value == null)
{
row.Add(string.Empty);
}
else
{
if (ReflectionHelper.IsEnumerable(value) && !ReflectionHelper.IsValidCast(typeof(string), value))
{
IEnumerable enumerable = value as IEnumerable;
string res = string.Empty;
foreach (object item in enumerable)
{
res = string.Format("{0} {1}", res, item);
}
res = res.Trim();
row.Add(res);
}
else
{
row.Add(value.ToString());
}
}
}
_results.Add(row);
WriteLineToTrace("{0} record(s) have been received.", _results.Count);
int index = ((_results.Count - (_settings.append + 1)) < 0) ? 0 : _results.Count - (_settings.append + 1);
int count = ((index + _settings.append) > _results.Count) ? _results.Count - index : _settings.append;
if (_results.Count % _settings.append == 0)
{
GenerateCsv(_results.GetRange(index, count), true);
}
int progress = GetControlPropertyThreadSafe <int>(pbProgress, "Value");
SetControlPropertyThreadSafe(pbProgress, "Value", progress + 1);
if (_results.Count == _settings.total)
{
processor.EndStreaming();
SetControlPropertyThreadSafe(pbProgress, "Value", 0);
SetControlPropertyThreadSafe(bStart, "Text", "Start");
EnableControls(true);
_isRun = false;
}
}
break;
}
}
