//Stage created for Petro
private void SetupStage2()
{
twitter = new Twitter();
IList <string> words = new List <String>()
{
"tibio",
};
configuration = new QueryConfiguration()
{
Keywords = words,
Location = Locations.Colombia,
Language = Languages.Spanish,
Filter = Filters.None,
SearchType = SearchTypes.Mixed,
SinceDate = new DateTime(2018, 01, 01),
UntilDate = DateTime.Now.AddDays(1),
MaxPublicationCount = 100
};
IList <IPublication> found = twitter.Search(configuration);
if (found.Count > 0)
{
publication = found[0];
}
}
//CREDENTIAL
//Stage created for Fajardo
//Stage created for Fajardo
private void SetupStage1()
{
facebook = new Facebook();
IList <string> words = new List <String>()
{
"Fajardo",
"Sergio Fajardo"
};
configuration = new QueryConfiguration()
{
Keywords = words,
Location = Locations.Colombia,
Language = Languages.Spanish,
Filter = Filters.None,
SearchType = SearchTypes.Mixed,
SinceDate = new DateTime(2018, 01, 01),
UntilDate = DateTime.Now.AddDays(1),
MaxPublicationCount = 100
};
IList <IPublication> found = facebook.Search(configuration);
if (found.Count > 0)
{
publication = found[0];
}
}
//for project facebook's group
public void SetupStage3()
{
facebook = new Facebook();
IList <string> words = new List <String>()
{
"Uribe",
"Uribe Velez"
};
//[ ]+Petro[ ,.]{}
//
configuration = new QueryConfiguration()
{
Keywords = words,
Location = Locations.Colombia,
Language = Languages.Spanish,
Filter = Filters.None,
SearchType = SearchTypes.Mixed,
SinceDate = new DateTime(2018, 01, 01),
UntilDate = DateTime.Now.AddDays(1),
MaxPublicationCount = 200
};
IList <IPublication> found = facebook.Search(configuration);
if (found.Count > 0)
{
publication = found[0];
}
}
private IPublication ParsePublicationOfJsonResponse(dynamic item, IQueryConfiguration queryConfiguration)
{
string id = item.id ?? "Not found";
DateTime createDate = item.created_time ?? QueryConfiguration.NONE_DATE;
string message = item.message ?? "Not message";
string decodeMessage = DecodeHtmlText(message);
if (decodeMessage != null)
{
message = decodeMessage;
}
string wroteBy = "Not found";
if (item.from != null)
{
wroteBy = (item.from.name + item.from.id);
}
IPublication publication = new Publication()
{
Id = "Facebook:" + id,
CreateDate = createDate,
Message = message,
WroteBy = wroteBy,
ConfigurationName = queryConfiguration.Name
};
return(publication);
}
internal void ExportQueryConfiguration(IQueryConfiguration currentConfiguration)
{
Thread thread = new Thread(ShowExportDialog(currentConfiguration));
thread.SetApartmentState(ApartmentState.STA);
thread.Start();
}
private void ParseFilter(ISearchTweetsParameters parameters, IQueryConfiguration queryConfiguration)
{
switch (queryConfiguration.Filter)
{
case Filters.Hashtag:
{
parameters.Filters = TweetSearchFilters.Hashtags;
break;
}
case Filters.Image:
{
parameters.Filters = TweetSearchFilters.Images;
break;
}
case Filters.News:
{
parameters.Filters = TweetSearchFilters.News;
break;
}
case Filters.None:
{
parameters.Filters = TweetSearchFilters.None;
break;
}
case Filters.Video:
{
parameters.Filters = TweetSearchFilters.Videos;
break;
}
}
}
internal IList <IPublication> InternalSearch(IQueryConfiguration queryConfiguration)
{
if (queryConfiguration != null)
{
IList <IPublication> publications = new List <IPublication>();
IDictionary <string, string> fields = new Dictionary <string, string>();
int totalPublications = queryConfiguration.MaxPublicationCount;
if (totalPublications > 100)
{
queryConfiguration.MaxPublicationCount = 100;
}
SetQueryFields(queryConfiguration, fields);
int roundSearchesByPage = totalPublications;
foreach (string page in pages)
{
IList <IPublication> partialPublication = RequestFeedToGraph(page, fields, roundSearchesByPage, queryConfiguration);
((List <IPublication>)publications).AddRange(partialPublication);
if (publications.Count > (totalPublications * 2))
{
break;
}
}
publications = FilterPublications(publications, queryConfiguration.Keywords);
publications = ReorganizeSearches(publications, totalPublications);
queryConfiguration.MaxPublicationCount = totalPublications; // Restore default settings in queryConfiguration
return(publications);
}
return(null);
}
public static IPublication ParseTweetToPublication(ITweet tweet, IQueryConfiguration queryConfiguration)
{
string message = "";
message = tweet.FullText ?? (tweet.Text + tweet.Suffix);
IPublication publication = null;
long id = tweet.Id;
if (String.IsNullOrEmpty(message))
{
if (String.IsNullOrEmpty(tweet.FullText))
{
message = tweet.Prefix + tweet.Text + tweet.Suffix;
}
else
{
message = tweet.FullText;
}
}
publication = new Publication()
{
Id = "Twitter:" + id,
Message = message,
WroteBy = tweet.CreatedBy.Name,
CreateDate = tweet.TweetLocalCreationDate,
ConfigurationName = queryConfiguration.Name
};
return(publication);
}
public QueryManager(IQueryConfiguration config, EntityDefinition[] foreignEntityDefinitions)
{
var instructions = new QueryInstructions <TEntity>(foreignEntityDefinitions);
Querier = instructions.Query;
ExpandMembers = config.ExpandMembers?.ToArray();
}
public void SetupStage3()
{
twitter = new Twitter();
string consumerKey = System.Configuration.ConfigurationManager.AppSettings["consumerKey"];
string consumerSecret = System.Configuration.ConfigurationManager.AppSettings["consumerSecret"];;
string accessToken = System.Configuration.ConfigurationManager.AppSettings["accessToken"];
string secretToken = System.Configuration.ConfigurationManager.AppSettings["secretToken"];
Tweetinvi.Auth.SetUserCredentials(consumerKey, consumerSecret, accessToken, secretToken);
IList <string> words = new List <String>()
{
"Petro",
};
configuration = new QueryConfiguration()
{
Keywords = words,
Location = Locations.Colombia,
Language = Languages.Spanish,
Filter = Filters.None,
SearchType = SearchTypes.Mixed,
SinceDate = new DateTime(2018, 01, 01),
UntilDate = DateTime.Now.AddDays(1),
MaxPublicationCount = 10
};
IList <IPublication> found = twitter.Search(configuration);
if (found.Count > 0)
{
publication = found[0];
}
}
// Test facebook page RealidadPoliticaColombiana
private void SetupStage4()
{
facebook = new Facebook();
IList <string> words = new List <String>()
{
"gallup",
};
configuration = new QueryConfiguration()
{
Keywords = words,
Location = Locations.USA,
Language = Languages.English,
Filter = Filters.None,
SearchType = SearchTypes.Mixed,
SinceDate = new DateTime(2018, 03, 18),
UntilDate = DateTime.Now.AddDays(1),
MaxPublicationCount = 40
};
IList <IPublication> found = facebook.Search(configuration);
if (found.Count > 0)
{
publication = found[0];
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TestSpecificationReportByQueryModel" /> class for the console extension.
/// </summary>
/// <param name="tfsService">Team foundation service.</param>
/// <param name="documentModel">Model of word document to work with.</param>
/// <param name="testAdapter">The test adapter.</param>
/// <param name="workitems">The work items.</param>
public TestSpecificationReportByQueryModel(ITfsService tfsService, ISyncServiceDocumentModel documentModel,
ITfsTestAdapter testAdapter, List <int> workitems)
{
if (tfsService == null)
{
throw new ArgumentNullException("tfsService");
}
if (documentModel == null)
{
throw new ArgumentNullException("documentModel");
}
TfsService = tfsService;
_testAdapter = testAdapter;
WordDocument = documentModel.WordDocument as Document;
DocumentModel = documentModel;
DocumentModel.PropertyChanged += DocumentModelOnPropertyChanged;
_queryConfiguration = new QueryConfiguration
{
UseLinkedWorkItems = false
};
_queryConfiguration.ImportOption = QueryImportOption.IDs;
_queryConfiguration.ByIDs = workitems;
LinkTypes = TfsService.GetLinkTypes().Select(
x => new DataItemModel <ITFSWorkItemLinkType>(x)
{
IsChecked = QueryConfiguration.LinkTypes.Contains(x.ReferenceName)
}).ToList();
}
public IQueryCommand WithDefaultConfigs()
{
QueryConfiguration = new DefaultQueryConfiguration();
FingerprintConfiguration = new DefaultFingerprintConfiguration();
createFingerprintMethod = () => fingerprintingMethodFromSelector().WithFingerprintConfig(FingerprintConfiguration);
return(this);
}
private void Export(ISearchDataSet dataset, int quantity)
{
DialogResult result = MessageBox.Show("¿Desea exportar las publicaciones (publicaciones de twitter) con el contenido completo?", "Export content", MessageBoxButtons.YesNoCancel, MessageBoxIcon.Question);
if (result == DialogResult.Yes)
{
IList <IPublication> publications = dataset.GetPublications();
ReadHtmlContents(publications);
dataset.AddOrReplacePublications(publications);
}
IQueryConfiguration queryConfiguration = queriesControl.GetCurrentQueryConfiguration();
if (queryConfiguration != null)
{
dataset.BaseName = queryConfiguration.SinceDate.ToShortDateString().Replace("/", "-") + "_" + queryConfiguration.UntilDate.AddDays(-1).ToShortDateString().Replace("/", "-");
}
else
{
dataset.BaseName = DateTime.Now.AddDays(-1).ToShortDateString().Replace("/", "-") + "_" + DateTime.Now.ToShortDateString().Replace("/", "-");
}
InvokeHandlers invokeHandlers = new InvokeHandlers(InvokeScrapperHandlers);
this.Invoke(invokeHandlers);
}
public IQueryCommand WithConfigs <T1, T2>() where T1 : IFingerprintConfiguration, new() where T2 : IQueryConfiguration, new()
{
QueryConfiguration = new T2();
FingerprintConfiguration = new T1();
createFingerprintMethod = () => fingerprintingMethodFromSelector().WithFingerprintConfig(FingerprintConfiguration);
return(this);
}
public IQueryCommand WithConfigs(IFingerprintConfiguration fingerprintConfiguration, IQueryConfiguration configuration)
{
QueryConfiguration = configuration;
FingerprintConfiguration = fingerprintConfiguration;
createFingerprintMethod = () => fingerprintingMethodFromSelector().WithFingerprintConfig(fingerprintConfiguration);
return(this);
}
public IUsingQueryServices WithConfigs(Action <CustomFingerprintConfiguration> fingerprintConfig, Action <CustomQueryConfiguration> queryConfig)
{
QueryConfiguration = new CustomQueryConfiguration();
queryConfig((CustomQueryConfiguration)QueryConfiguration);
FingerprintConfiguration = new CustomFingerprintConfiguration();
fingerprintConfig((CustomFingerprintConfiguration)FingerprintConfiguration);
return(this);
}
public override async Task <IReadOnlyList <TOut> > ListAsync <TOut>(IQueryConfiguration <TEntity, TOut> queryConfiguration, CancellationToken cancellationToken = default)
{
var queryPipe = QueryBuilder.Build(queryConfiguration);
var query = queryPipe.Query(EntitySet);
var results = await query.ToListAsync(cancellationToken).ConfigureAwait(false);
return(results);
}
public IQueryCommand WithConfigs(Action <CustomFingerprintConfiguration> fingerprintConfig, Action <CustomQueryConfiguration> queryConfig)
{
QueryConfiguration = new CustomQueryConfiguration();
queryConfig((CustomQueryConfiguration)QueryConfiguration);
FingerprintConfiguration = new CustomFingerprintConfiguration();
fingerprintConfig((CustomFingerprintConfiguration)FingerprintConfiguration);
createFingerprintMethod = () => fingerprintingMethodFromSelector().WithFingerprintConfig(FingerprintConfiguration);
return(this);
}
public virtual async Task RemoveAsync(IQueryConfiguration <TEntity> queryConfiguration, CancellationToken cancellationToken = default)
{
var results = await ListAsync(queryConfiguration, cancellationToken).ConfigureAwait(false);
foreach (var entity in results)
{
await RemoveAsync(entity, cancellationToken).ConfigureAwait(false);
}
}
private IEnumerable<SubFingerprintData> GetSubFingerprints(IModelService modelService, HashData hash, IQueryConfiguration queryConfiguration)
{
if (!string.IsNullOrEmpty(queryConfiguration.TrackGroupId))
{
return modelService.ReadSubFingerprintDataByHashBucketsThresholdWithGroupId(hash.HashBins, queryConfiguration.ThresholdVotes, queryConfiguration.TrackGroupId);
}
return modelService.ReadSubFingerprintDataByHashBucketsWithThreshold(hash.HashBins, queryConfiguration.ThresholdVotes);
}
public override async Task <int> CountAsync(IQueryConfiguration <TEntity> queryConfiguration, CancellationToken cancellationToken = default)
{
var queryPipe = QueryBuilder.Build(queryConfiguration);
var query = queryPipe.Query(EntitySet);
var count = await query.CountAsync(cancellationToken).ConfigureAwait(false);
return(count);
}
public override async Task <TOut> FindAsync <TOut>(IQueryConfiguration <TEntity, TOut> queryConfiguration, CancellationToken cancellationToken = default)
{
var queryPipe = QueryBuilder.Build(queryConfiguration);
var query = queryPipe.Query(EntitySet);
var result = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
return(result);
}
public virtual IQueryPipe <TEntity> Build <TEntity>(IQueryConfiguration <TEntity> configuration)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
return(BuildPipe(configuration.Specifications));
}
private void SetQueryFields(IQueryConfiguration queryConfiguration, IDictionary <string, string> args)
{
string max = (queryConfiguration.MaxPublicationCount > 0 ? queryConfiguration.MaxPublicationCount : 500) + "";
args.Add("limit", max);
string fieldsRequest = GetFieldsRequest(queryConfiguration);
args.Add("fields", fieldsRequest);
SetDatesRangeInFields(args, queryConfiguration);
}
private bool Classify(IPublication publication, IQueryConfiguration queryConfiguration)
{
bool clasified = false;
if (publication != null)
{
clasified = ClassifyByDate(publication, queryConfiguration.SinceDate, queryConfiguration.UntilDate);
}
return(clasified);
}
private void SetDatesRangeInFields(IDictionary <string, string> args, IQueryConfiguration queryConfiguration)
{
DateTime until = queryConfiguration.UntilDate;
DateTime since = queryConfiguration.SinceDate;
string untilRequest = until.ToShortDateString().Replace("/", "-");
string sinceRequest = since.ToShortDateString().Replace("/", "-");
args["since"] = sinceRequest;
args["until"] = untilRequest;
}
public IQueryPipe <TEntity> Build <TEntity>(IQueryConfiguration <TEntity> queryConfiguration)
where TEntity : class
{
if (queryConfiguration == null)
{
throw new ArgumentNullException(nameof(queryConfiguration));
}
return(CreateCompositePipe(queryConfiguration.OutputSpecifications));
}
private void ParseLocation(ISearchTweetsParameters parameters, IQueryConfiguration queryConfiguration)
{
//if (queryConfiguration.Location == Locations.Colombia)
//{
// parameters.Locale = "Colombia";
//}
//else if(queryConfiguration.Location == Locations.USA)
//{
// parameters.Locale = "USA";
//}
}
private void AddLanguajes(IQueryConfiguration queryConfiguration)
{
if (rdbSpanish.Checked)
{
queryConfiguration.Language = Languages.Spanish;
}
else if (rdbEnglish.Checked)
{
queryConfiguration.Language = Languages.English;
}
}
private void AddLocation(IQueryConfiguration queryConfiguration)
{
if (rdbColombia.Checked)
{
queryConfiguration.Location = Locations.Colombia;
}
else if (rdbUsa.Checked)
{
queryConfiguration.Location = Locations.USA;
}
}
public QueryResult Query(IModelService modelService, IEnumerable<HashData> hashes, IQueryConfiguration queryConfiguration)
{
var hammingSimilarities = new Dictionary<IModelReference, int>();
foreach (var hash in hashes)
{
var subFingerprints = GetSubFingerprints(modelService, hash, queryConfiguration);
foreach (var subFingerprint in subFingerprints)
{
int similarity = SimilarityUtility.CalculateHammingSimilarity(hash.SubFingerprint, subFingerprint.Signature);
if (hammingSimilarities.ContainsKey(subFingerprint.TrackReference))
{
hammingSimilarities[subFingerprint.TrackReference] += similarity;
}
else
{
hammingSimilarities.Add(subFingerprint.TrackReference, similarity);
}
}
}
if (hammingSimilarities.Any())
{
var topMatches = hammingSimilarities.OrderByDescending(pair => pair.Value).Take(queryConfiguration.MaximumNumberOfTracksToReturnAsResult);
var resultSet = topMatches.Select(match => new ResultEntry { Track = modelService.ReadTrackByReference(match.Key), Similarity = match.Value }).ToList();
return new QueryResult
{
ResultEntries = resultSet,
IsSuccessful = true,
AnalyzedCandidatesCount = hammingSimilarities.Count
};
}
return new QueryResult
{
ResultEntries = Enumerable.Empty<ResultEntry>().ToList(),
IsSuccessful = false,
AnalyzedCandidatesCount = 0
};
}
public QueryResult Query(IEnumerable<bool[]> fingerprints, IQueryConfiguration queryConfiguration)
{
Dictionary<int, int> hammingSimilarities = new Dictionary<int, int>();
foreach (var fingerprint in fingerprints)
{
var tuple = hashingAlgorithm.Hash(fingerprint, queryConfiguration.NumberOfLSHTables, queryConfiguration.NumberOfMinHashesPerTable);
var subFingerprints = modelService.ReadSubFingerprintsByHashBucketsHavingThreshold(tuple.Item2, queryConfiguration.ThresholdVotes);
foreach (var subFingerprint in subFingerprints)
{
int similarity = HashingUtils.CalculateHammingSimilarity(tuple.Item1, subFingerprint.Item1.Signature);
if (hammingSimilarities.ContainsKey(subFingerprint.Item1.TrackId))
{
hammingSimilarities[subFingerprint.Item1.TrackId] += similarity;
}
else
{
hammingSimilarities.Add(subFingerprint.Item1.TrackId, similarity);
}
}
}
if (hammingSimilarities.Any())
{
var topMatches = hammingSimilarities.OrderBy(pair => pair.Value).Take(queryConfiguration.MaximumNumberOfTracksToReturnAsResult);
List<ResultData> resultSet = topMatches.Select(match => new ResultData { Track = modelService.ReadTrackById(match.Key), Similarity = match.Value }).ToList();
return new QueryResult
{
IsSuccessful = true,
TotalNumberOfAnalyzedCandidates = hammingSimilarities.Count,
Results = resultSet
};
}
return new QueryResult();
}
public QueryConfigurator(IQueryConfiguration queryConfiguration)
{
_queryConfiguration = ArgumentValidation.IsNotNull(queryConfiguration, "queryConfiguration");
}