public void TestInsertMany()
{
string conStr = "Server=www.forrestireland.com;Port=5432;User Id=CharlieBrown;Password=AcademicCoffeeBean59943;Database=pipeline;";
using (NpgsqlConnection con = new NpgsqlConnection(conStr))
{
IPipelineRepository repo = new PipelineRepository(con);
IEnumerable <WordHashtagPair> pairs = new List <WordHashtagPair>
{
new WordHashtagPair
{
Word = "banana",
HashTag = "yellow"
},
new WordHashtagPair
{
Word = "banana",
HashTag = "curvy"
},
new WordHashtagPair
{
Word = "banana",
HashTag = "yellow"
}
};
repo.InsertMany(pairs);
}
}
public void PipelineRepository_Get_NameDoesntExist_Throws()
{
const string pipelineName = "Pipeline2";
var pipelineRepository = new PipelineRepository();
pipelineRepository.Get(pipelineName);
}
public ActionResult Imports(SessionViewModel sessionViewModel)
{
SessionViewModel viewShow = new SessionViewModel();
viewShow.MessageType = new Dictionary <string, string>();
SapDataContext _sapDataContext = new SapDataContext();
viewShow.Projections = new List <ProjectionViewModel>();
PipelineRepository pipeline = new PipelineRepository();
Check check = new Check(pipeline.GetProjectionContext, _sapDataContext);
try
{
IEnumerable <ProjectionViewModel> Data = check.NewBudgets(sessionViewModel);
ushort?lastDocument = pipeline.Read().Select(p => p.IdDoc).Max() == null ? (ushort)0 :
(ushort)pipeline.Read().Select(p => p.IdDoc).Max();
check.InsertNewBudgets(Data, (ushort)lastDocument, sessionViewModel);
viewShow.Projections = Data.ToList();
viewShow.MessageType.Add("Success", "Se cargaron " + Data.Count() + " presupuestos");
viewShow.DocumentNumber = (ushort)lastDocument;
}
catch (Exception e)
{
viewShow.MessageType.Add("Error", (!String.IsNullOrEmpty(e.InnerException.Message)
? e.Message : e.InnerException.Message));
}
finally
{
check.Dispose();
pipeline.Dispose();
}
return(View(viewShow));
}
public void PipelineRepository_Get_NameExists_ReturnsPipeline()
{
const string pipelineName = "Pipeline3";
var pipelineRepository = new PipelineRepository();
var newPipeline = pipelineRepository.CreateNew(pipelineName);
var getPipeline = pipelineRepository.Get(pipelineName);
getPipeline.Should().Be(newPipeline);
}
public void PipelineRepository_CreateNew_CreatesPipelineWithName()
{
const string pipelineName = "Pipeline1";
var pipelineRepository = new PipelineRepository();
var pipeline = pipelineRepository.CreateNew(pipelineName);
pipeline.Should().NotBeNull();
pipeline.Name.Should().Be(pipelineName);
}
public void PipelineRepository_Get_FromOtherRepository_ReturnsPipeline()
{
const string pipelineName = "Pipeline4";
var pipelineRepository = new PipelineRepository();
var otherPipelineRepository = new PipelineRepository();
var newPipeline = pipelineRepository.CreateNew(pipelineName);
var getPipeline = otherPipelineRepository.Get(pipelineName);
getPipeline.Should().Be(newPipeline);
}
public void TestSingleInsert()
{
string conStr = "Server=www.forrestireland.com;Port=5432;User Id=CharlieBrown;Password=AcademicCoffeeBean59943;Database=pipeline;";
using (NpgsqlConnection con = new NpgsqlConnection(conStr))
{
IPipelineRepository repo = new PipelineRepository(con);
WordHashtagPair pair = new WordHashtagPair
{
Word = "banana",
HashTag = "yellow"
};
repo.Insert(pair);
}
// Manual verification that insert appeared in database;
// Made by Forrest
}
public IHttpActionResult GetPipelinesByUser([FromBody] PipelineByUserDTO pipelineByUser)
{
try
{
PipelineRepository repo = new PipelineRepository();
var result = repo.GetPipelinesByUser(pipelineByUser).ToList();
if (result.Count > 0)
{
return(Ok(result));
}
else
{
return(StatusCode(System.Net.HttpStatusCode.NoContent));
}
}
catch (Exception ex)
{
Logger.AppLogManager(ex.Source, "PipelineController", ex.Message);
return(StatusCode(System.Net.HttpStatusCode.NotFound));
}
}
public ActionResult Imports()
{
SessionViewModel viewModel = new SessionViewModel();
PipelineRepository pipeline = new PipelineRepository();
try
{
int? lastDocument = pipeline.Read().Select(p => p.IdDoc).Max();
DateTime?lastUpdate = pipeline.Read().Select(p => p.FechaHora).Max();
viewModel.DocumentNumber = (lastDocument == null ? (ushort)0 : (ushort)lastDocument);
viewModel.LastUpdate = lastUpdate;
}
catch (Exception)
{
viewModel.DocumentNumber = 0;
}
finally
{
pipeline.Dispose();
}
return(View(viewModel));
}
static void Main()
{
Configuration config = ConfigurationManager.OpenExeConfiguration(ConfigurationUserLevel.None);
string consumerKey = config.AppSettings.Settings["ConsumerKey"].Value;
string consumerSecret = config.AppSettings.Settings["ConsumerSecret"].Value;
string apiToken = config.AppSettings.Settings["ApiToken"].Value;
string apiTokenSecret = config.AppSettings.Settings["ApiTokenSecret"].Value;
string stopWordFileName = config.AppSettings.Settings["StopWordFile"].Value;
string connectionStringPipeline = config.AppSettings.Settings["connectionStringPipeline"].Value;
string connectionStringPostGre = config.AppSettings.Settings["connectionStringPostGre"].Value;
string performanceLog = config.AppSettings.Settings["performanceLog"].Value;
Credentials = Auth.CreateCredentials(consumerKey, consumerSecret, apiToken, apiTokenSecret);
TwitterStreamProcessor streamProcessor = null;
bool stopMonitoring = false;
var st = new Thread(() =>
{
while (!stopMonitoring)
{
try
{
using (NpgsqlConnection conPipeline = new NpgsqlConnection(connectionStringPipeline))
{
ITweetFilter tweetFilter = new TweetFilter();
ITweetTrim tweetTrimmer = new TweetTrim.TweetTrim(stopWordFileName);
IHashPairGenerator pairGenerator = new HashPairGenerator();
IPipelineRepository repo = new PipelineRepository(conPipeline);
if (conPipeline.State == System.Data.ConnectionState.Open)
{
streamProcessor = new TwitterStreamProcessor(Credentials, tweetFilter, tweetTrimmer, pairGenerator, repo);
streamProcessor.Start();
}
}
Thread.Sleep(5000);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
});
st.Start();
var t = new Thread(() =>
{
int lastTweetsReceived = 0;
int lastTweetsAccepted = 0;
int lastPairsStored = 0;
Dictionary <int, int> lastWordSetsStored = new Dictionary <int, int> {
{ 1, 0 }, { 2, 0 }, { 3, 0 }, { 4, 0 }, { 5, 0 }
};
DateTime origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
while (!stopMonitoring)
{
try
{
int tweetsReceived = (int)streamProcessor.TweetsReceived;
int newTweets = tweetsReceived - lastTweetsReceived;
lastTweetsReceived = tweetsReceived;
int tweetsAccepted = (int)streamProcessor.TweetsAccepted;
int newTweetsAccepted = tweetsAccepted - lastTweetsAccepted;
lastTweetsAccepted = tweetsAccepted;
int pairsStored = (int)streamProcessor.WordHashtagPairsStored;
int newPairsStored = pairsStored - lastPairsStored;
lastPairsStored = pairsStored;
int[] newWordSetsStored = new int[5];
for (int i = 1; i <= 5; i++)
{
int wordSetsStored = (int)streamProcessor.WordSetsStored[i];
newWordSetsStored[i - 1] = wordSetsStored - lastWordSetsStored[i];
lastWordSetsStored[i] = wordSetsStored;
}
double timestamp = (DateTime.Now.ToUniversalTime() - origin).TotalSeconds;
using (StreamWriter sw = new FileInfo(performanceLog).AppendText())
{
sw.WriteLine($"{DateTime.Now.ToLongTimeString()},{newTweets},{newTweetsAccepted},{newPairsStored},{newWordSetsStored[0]},{newWordSetsStored[1]},{newWordSetsStored[2]},{newWordSetsStored[3]},{newWordSetsStored[4]}, {streamProcessor.State}");
}
}
catch (Exception)
{
/* Do nothing */
}
Thread.Sleep(5000);
}
});
t.Start();
while (true)
{
string s = Console.ReadLine();
if (s == "q" || s == "quit")
{
stopMonitoring = true;
break;
}
}
Console.WriteLine(streamProcessor.Stop());
st.Join();
Console.WriteLine($"Average Tweet throughput {streamProcessor.TweetThroughput} tweets/s.");
Console.WriteLine($"Received {streamProcessor.TweetsReceived} tweets.");
Console.WriteLine($"Accepted {streamProcessor.TweetsAccepted} tweets.");
Console.WriteLine($"Stored {streamProcessor.WordHashtagPairsStored} word-hashtag pairs.");
Console.WriteLine($"Completed in {streamProcessor.MilliSecondsRunning} ms");
}
