/// <summary>
/// extracts the given zip file and loads its contents into the database
/// </summary>
/// <param name="zipStream"></param>
/// <param name="fileType"></param>
/// <returns></returns>
void extractAndLoadZipIntoDatabase(MemoryStream zipStream, string fileType, string downloadURL)
{
Stream unzippedEntryStream; //Unzipped data from a file in the archive
ZipArchive archive = new ZipArchive(zipStream);
bool lookingForFirstBook = true;
//adds txt files to list of streams
foreach (ZipArchiveEntry entry in archive.Entries)
{
try
{
if (entry.FullName.EndsWith(fileType, StringComparison.OrdinalIgnoreCase) && lookingForFirstBook)
{
unzippedEntryStream = entry.Open(); // .Open will return a stream
byte[] byteArray = ReadFully(unzippedEntryStream); //converts stream to byte array
string bookName = getTitle(byteArray);
DateTime sqlDate = DateTime.Now;
ScraperUtilities.addCorpusContent(bookName, "text",
this.m_guid, this.GetType().FullName, sqlDate, downloadURL,
byteArray, m_context, m_corpusId);
unzippedEntryStream.Dispose();
lookingForFirstBook = false;
}
}
catch { } //ignore invalid files
}
}
/// <summary>
/// sets up conditions for twitter stream and runs the stream
/// </summary>
void StartTwitterStream()
{
List <ITweet> tweetList = new List <ITweet>();
// Enable Automatic RateLimit handling
RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackAndAwait;
var stream = Stream.CreateSampleStream();
stream.StallWarnings = true;
stream.AddTweetLanguageFilter(LanguageFilter.English);
stream.FilterLevel = Tweetinvi.Streaming.Parameters.StreamFilterLevel.None;
bool auth = Authorized;
m_downloadCount = 0;
m_timer.Reset();
//bool downloadLimitReached = downloadStop();
//bool timeLimitReached = timeStop();
m_timer.Start();
//have to add this before startign the stream, tells us what to do
//each time we recieve a tweet
stream.TweetReceived += (sender, args) =>
{
ITweet tweet = args.Tweet;
//Debug.Assert(false);
try
{
Debug.WriteLine(tweet);
Console.WriteLine(tweet);
string title = getTweetName(tweet);
ScraperUtilities.addCorpusContent(tweet.Text, "tweet", this.Guid,
this.GetType().FullName, tweet, this.m_context, m_corpusId);
m_downloadCount++;
m_progress = (float)m_downloadCount / m_downloadLimit;
if (timeStop() || downloadStop())
{
StopTwitterStream(stream);
}
}
catch
{
StopTwitterStream(stream);
}
};
//start the stream, now that we know what to do with it
if (m_authorized)
{
stream.StartStream();
}
else
{
m_status = "No Twitter Authorization";
//string consumerKey = "GzWUY0oTfH4AMZdnMqrm0wcde";
//string consumerSecret = "QfuQ7YgmLTmvQguuw3siKrwzPCiQ9EW7NleCvhxdRrjSKhfZww";
//UserAuthentication(consumerKey, consumerSecret);
//StartTwitterStream();
}
}
