public PreprocessingData(IKaVESet <string> zips)
{
_allZips = zips;
_unindexedZips = Sets.NewHashSetFrom(_allZips);
_idsByZip = new Dictionary <string, IKaVESet <string> >();
_uncleansedZips = Sets.NewHashSet <string>();
}
public void Users(IKaVESet <User> users)
{
_log.Log();
_log.Log(new string('-', 60));
_log.Log();
_log.Log("identified {0} users:", users.Count);
var i = 0;
foreach (var u in users)
{
_log.Log();
_log.Log("#### user {0}", i++);
_log.Log();
_log.Log("Files:");
_log.Log();
foreach (var file in u.Files)
{
_log.Append("{0}, ", file);
}
_log.Log();
_log.Log("Identifier:");
_log.Log();
foreach (var id in u.Identifiers)
{
_log.Append("{0}, ", id);
}
}
}
public void Setup()
{
_actualProcessedZips = Sets.NewHashSet <string>();
_actualGroupedIds = null;
_actualGroups = Lists.NewList <IKaVESet <string> >();
_actualCleanedZips = Lists.NewList <string>();
_io = Mock.Of <IPreprocessingIo>();
Mock.Get(_io).Setup(io => io.FindRelativeZipPaths()).Returns(GetExpectedZips());
Mock.Get(_io)
.Setup(io => io.GetFullPath_In(It.IsAny <string>()))
.Returns <string>(relZip => "<dirIn>\\" + relZip);
Mock.Get(_io)
.Setup(io => io.GetFullPath_Merged(It.IsAny <string>()))
.Returns <string>(relZip => "<dirMerged>\\" + relZip);
Mock.Get(_io)
.Setup(io => io.GetFullPath_Out(It.IsAny <string>()))
.Returns <string>(relZip => "<dirOut>\\" + relZip);
_log = Mock.Of <IMultiThreadedPreprocessingLogger>();
Mock.Get(_log)
.Setup(l => l.Error(It.IsAny <int>(), It.IsAny <Exception>()))
.Callback <int, Exception>((taskId, ex) => _lastLoggedException = ex);
_sut = new MultiThreadedPreprocessing(
_io,
_log,
NumWorker,
IdReaderFactory,
CreateGrouper(),
GroupMergerFactory,
CleanerFactory);
}
public TypeShapeSolutionAnalysis(ISolution solution,
ILogger logger,
Action <ITypeShape> cbTypeShape) : base(solution, logger)
{
_cbTypeShape = cbTypeShape;
_seenClrNames = Sets.NewHashSet <string>();
}
public string Merge(IKaVESet <string> relZips)
{
Asserts.NotNull(relZips);
Asserts.That(relZips.Count > 0);
foreach (var relZip in relZips)
{
var zip = _io.GetFullPath_In(relZip);
Asserts.That(File.Exists(zip));
}
var relZipOut = relZips.First();
_log.NextGroup(relZips.Count, relZipOut);
var zipOut = _io.GetFullPath_Merged(relZipOut);
_io.EnsureParentExists(zipOut);
var numEvents = 0;
using (var wa = new WritingArchive(zipOut))
{
foreach (var e in ReadArchives(relZips))
{
numEvents++;
wa.Add(e);
}
}
_log.Result(numEvents);
return(relZipOut);
}
public void Run()
{
_upes = Lists.NewList <IUserProfileEvent>();
var users = Sets.NewHashSet <string>();
_keys = Sets.NewHashSet <string>();
_assignableSubmissions = Sets.NewHashSet <string>();
var zips = _io.FindCcZips().ToList();
var cur = 1;
var total = zips.Count;
foreach (var zipName in zips)
{
_printer.StartZip(zipName, cur++, total);
var userKey = GetUserKey(zipName);
users.Add(userKey);
_printer.FoundUserKey(userKey);
var zipKeys = GetKeysFrom(zipName);
_printer.FoundKeysInZip(zipKeys);
foreach (var key in zipKeys)
{
var combKey = string.Format("{0}\t{1}", key, userKey);
_keys.Add(combKey);
}
}
_printer.FoundUsers(users);
_printer.FoundKeys(_keys);
_printer.FoundUpes(_upes);
_printer.FoundAssignableZips(_assignableSubmissions);
}
public void Run()
{
Log(
"Simple smoketest. Reads all events from all zips found in the provided base dir and calculates their hash codes to access all their fields.");
Log("");
Log("base dir: {0}", _baseDir);
Log("number of threads: {0}", _numThreads);
Log("");
Log("Finding .zip files...");
_remainingZips = FindRelativeZipPaths();
_numZips = _remainingZips.Count;
Log("Found {0} files.", _numZips);
Log("");
var tasks = new Task[_numThreads];
for (var i = 0; i < _numThreads; i++)
{
var i1 = i;
tasks[i] = Task.Factory.StartNew(() => { Process(i1); });
}
Task.WaitAll(tasks);
Log("");
Log("done (press key)");
Console.Read();
}
private static void WriteCache(IKaVESet <string> ids, string zip)
{
var json = ids.ToFormattedJson();
var cacheFile = GetCacheFile(zip);
File.WriteAllText(cacheFile, json);
}
private IKaVESet <IDEEvent> GetAnswer(IDEEvent @event)
{
_answer = Sets.NewHashSet(@event);
foreach (var processor in GetProcessorsFor(@event))
{
processor(@event);
}
return(_answer);
}
private static ISet <User> FindMatchingEntries(IEnumerable <User> users,
IKaVESet <string> currentArchiveIdentifiers)
{
var matches = users.Where(
user =>
(user.Identifiers.Overlaps(currentArchiveIdentifiers)));
return(Sets.NewHashSetFrom(matches));
}
public ContextStatistics()
{
UniqueTypeDecl = Sets.NewHashSet <ITypeName>();
UniqueMethodDeclsOverrideOrImplementAsm = Sets.NewHashSet <IMethodName>();
UniqueAssemblies = Sets.NewHashSet <IAssemblyName>();
UniqueAsmMethods = Sets.NewHashSet <IMethodName>();
UniqueAsmFields = Sets.NewHashSet <IFieldName>();
UniqueAsmProperties = Sets.NewHashSet <IPropertyName>();
}
public void FoundAssignableZips(IKaVESet <string> zips)
{
Console.WriteLine(@"#### found {0} assignable submissions: ####", zips.Count);
foreach (var zip in zips)
{
Console.WriteLine(zip);
}
Console.WriteLine();
}
public void FoundUsers(IKaVESet <string> users)
{
Console.WriteLine(@"#### users:");
foreach (var user in users)
{
Console.WriteLine(user);
}
Console.WriteLine();
}
public void FoundKeys(IKaVESet <string> keys)
{
Console.WriteLine(@"#### keys:");
foreach (var key in keys)
{
Console.WriteLine(key);
}
Console.WriteLine();
}
public void FoundKeysInZip(IKaVESet <string> zipKeys)
{
Console.WriteLine(@"keys: [");
foreach (var key in zipKeys)
{
Console.WriteLine(@" {0}", key);
}
Console.WriteLine(@"]");
Console.WriteLine();
}
public TypeShape()
{
TypeHierarchy = new TypeHierarchy();
NestedTypes = Sets.NewHashSet <ITypeName>();
Delegates = Sets.NewHashSet <IDelegateTypeName>();
EventHierarchies = Sets.NewHashSet <IMemberHierarchy <IEventName> >();
Fields = Sets.NewHashSet <IFieldName>();
MethodHierarchies = Sets.NewHashSet <IMemberHierarchy <IMethodName> >();
PropertyHierarchies = Sets.NewHashSet <IMemberHierarchy <IPropertyName> >();
}
public void StoreIds([NotNull] string zip, [NotNull] IKaVESet <string> ids)
{
lock (_lock)
{
Asserts.NotNull(zip);
Asserts.NotNull(ids);
Asserts.That(_allZips.Contains(zip));
Asserts.Not(_idsByZip.ContainsKey(zip));
_idsByZip[zip] = ids;
}
}
public bool AcquireNextUnmergedZipGroup(out IKaVESet <string> zips)
{
lock (_lock)
{
zips = _zipGroups.FirstOrDefault();
if (zips != null)
{
_zipGroups.Remove(zips);
return(true);
}
return(false);
}
}
private IKaVESet <string> GetKeysFrom(string zipName)
{
IKaVESet <string> keys = Sets.NewHashSet <string>();
foreach (var cce in _io.ReadCce(zipName))
{
var date = cce.TriggeredAt ?? DateTime.MinValue;
var dateStr = string.Format("{0:0000}{1:00}{2:00}", date.Year, date.Month, date.Day);
keys.Add(dateStr);
}
return(keys);
}
public void Count(IKaVESet <IAssemblyName> asms)
{
foreach (var asm in asms)
{
if (_counts.ContainsKey(asm))
{
_counts[asm]++;
}
else
{
_counts[asm] = 1;
}
}
}
public void StoreZipGroups([NotNull] IKaVESet <IKaVESet <string> > zipGroups)
{
lock (_lock)
{
Asserts.That(_zipGroups == null);
Asserts.NotNull(zipGroups);
Asserts.Not(zipGroups.Count == 0);
foreach (var zipGroup in zipGroups)
{
Asserts.Not(zipGroup.Count == 0);
foreach (var zip in zipGroup)
{
Asserts.That(_allZips.Contains(zip));
}
}
_zipGroups = Sets.NewHashSetFrom(zipGroups);
}
}
public void LoggerTest()
{
var actualZips = new IDictionary <string, IKaVESet <string> > [1];
var actualUsers = new IKaVESet <User> [1];
Mock.Get(_log)
.Setup(l => l.Zips(It.IsAny <IDictionary <string, IKaVESet <string> > >()))
.Callback <IDictionary <string, IKaVESet <string> > >(m => { actualZips[0] = m; });
Mock.Get(_log)
.Setup(l => l.Users(It.IsAny <IKaVESet <User> >()))
.Callback <IKaVESet <User> >(us => { actualUsers[0] = us; });
Given("a", "0", "1");
Given("b", "1", "2");
Given("c", "3", "4");
new Grouper(_log).GroupRelatedZips(_zipToIds);
Mock.Get(_log).Verify(l => l.Init(), Times.Exactly(1));
Mock.Get(_log).Verify(l => l.Zips(It.IsAny <IDictionary <string, IKaVESet <string> > >()), Times.Exactly(1));
Mock.Get(_log).Verify(l => l.Users(It.IsAny <IKaVESet <User> >()), Times.Exactly(1));
// zips
var expectedZips = new Dictionary <string, IKaVESet <string> >
{
{ "a", Sets.NewHashSet("0", "1") },
{ "b", Sets.NewHashSet("1", "2") },
{ "c", Sets.NewHashSet("3", "4") }
};
CollectionAssert.AreEqual(expectedZips, actualZips[0]);
// users
var expectedUsers = Sets.NewHashSet(
new User {
Files = { "a", "b" }, Identifiers = { "0", "1", "2" }
},
new User {
Files = { "c" }, Identifiers = { "3", "4" }
});
CollectionAssert.AreEquivalent(expectedUsers, actualUsers[0]);
}
public override void Visit(IVariableDeclaration stmt, IKaVESet <IName> context)
{
context.Add(stmt.Type);
base.Visit(stmt, context);
}
public override void Visit(IMethodDeclaration stmt, IKaVESet <IName> context)
{
context.Add(stmt.Name);
base.Visit(stmt, context);
}
public override void Visit(ILambdaExpression expr, IKaVESet <IName> context)
{
context.Add(expr.Name);
base.Visit(expr, context);
}
public void Setup()
{
_zips = Sets.NewHashSet <string>();
}
public override void Visit(IInvocationExpression entity, IKaVESet <IName> context)
{
context.Add(entity.MethodName);
base.Visit(entity, context);
}
public User()
{
Identifiers = Sets.NewHashSet <string>();
Files = Sets.NewHashSet <string>();
}
public TestRunEvent()
{
Tests = Sets.NewHashSet <TestCaseResult>();
}
public string Merge(IKaVESet <string> relZips)
{
throw _e;
}
