private IEnumerable <string> ReadZip()
{
using (var ra = new ReadingArchive(_zipPath))
{
return(ra.GetAll <string>());
}
}
public void Run()
{
Console.Write("looking (recursively) for events in folder {0}\n", Path.GetFullPath(_eventsDir));
/*
* Each .zip that is contained in the eventsDir represents all events
* that we have collected for a specific user, the folder represents the
* first day when the user uploaded data.
*/
var userZips = FindUserZips();
var ZipIterator = 0;
foreach (var userZip in userZips)
{
ZipIterator += 1;
Console.Write("\n#### processing user zip: {0} #####\n", userZip);
// open the .zip file ...
using (IReadingArchive ra = new ReadingArchive(Path.Combine(_eventsDir, userZip)))
{
// ... and iterate over content.
while (ra.HasNext())
{
/*
* within the userZip, each stored event is contained as a
* single file that contains the Json representation of a
* subclass of IDEEvent.
*/
try
{
var e = ra.GetNext <IDEEvent>();
// the events can then be processed individually
foreach (Process task in _tasks)
{
task.process(e);
}
}
catch (System.InvalidOperationException e)
{
Console.WriteLine(e.Message);
}
catch (Newtonsoft.Json.JsonReaderException e)
{
Console.WriteLine(e.Message);
}
catch (Exception e)
{
Console.WriteLine("not tested error!" + e.Message);
}
}
}
//getThe result for this actual zip and stock it in the .txt Result file.
foreach (Process task in _tasks)
{
task.getResult(ZipIterator + " / " + userZips.Count);
}
}
}
private void Process(int taskId)
{
Log("({0}) starting", taskId);
string relZip;
while (GetNextZip(taskId, out relZip))
{
var zip = Path.Combine(_baseDir, relZip);
using (var ra = new ReadingArchive(zip))
{
while (ra.HasNext())
{
try
{
var e = ra.GetNext <IDEEvent>();
// ReSharper disable once UnusedVariable
var hc = e.GetHashCode();
}
catch
{
Log("Exception thrown in {0} ({1}).", zip, ra.CurrentInternalFileName);
throw;
}
}
}
}
Log("({0}) stopping", taskId);
}
public void Run()
{
Console.Write("looking (recursively) for events in folder {0}\n", Path.GetFullPath(_eventsDir));
/*
* Each .zip that is contained in the eventsDir represents all events
* that we have collected for a specific user, the folder represents the
* first day when the user uploaded data.
*/
var userZips = FindUserZips();
foreach (var userZip in userZips)
{
Console.Write("\n#### processing user zip: {0} #####\n", userZip);
// open the .zip file ...
using (IReadingArchive ra = new ReadingArchive(Path.Combine(_eventsDir, userZip)))
{
// ... and iterate over content.
while (ra.HasNext())
{
/*
* within the userZip, each stored event is contained as a
* single file that contains the Json representation of a
* subclass of IDEEvent.
*/
var e = ra.GetNext <IDEEvent>();
// the events can then be processed individually
process(e);
}
}
}
}
private void WriteEventStream(string zip, string relFile)
{
using (var ra = new ReadingArchive(zip))
{
var events = ra.GetAll <IDEEvent>();
_export.Write(events, relFile);
}
}
protected IList <T> ReadZip <T>(string file)
{
Assert.IsTrue(file.StartsWith(DirTestRoot));
using (var ra = new ReadingArchive(file))
{
return(ra.GetAll <T>());
}
}
public IEnumerable <IDEEvent> ReadEvents(string exportFile)
{
using (var ra = new ReadingArchive(exportFile))
{
while (ra.HasNext())
{
yield return(ra.GetNext <IDEEvent>());
}
}
}
private void AssertZipContent(string fileName, params string[] expectedsArr)
{
using (var ra = new ReadingArchive(fileName))
{
var expecteds = new List <string>();
expecteds.AddRange(expectedsArr);
var actuals = ra.GetAll <string>();
Assert.AreEqual(expecteds, actuals);
}
}
private static IEnumerable <IDEEvent> ReadEventsFromZip(string zip)
{
using (var ra = new ReadingArchive(zip))
{
while (ra.HasNext())
{
yield return(ra.GetNext <IDEEvent>());
}
}
}
private IEnumerable <IDEEvent> ReadEventsForUser(string userZip)
{
using (var ra = new ReadingArchive(_root + userZip))
{
while (ra.HasNext())
{
yield return(ra.GetNext <IDEEvent>());
}
}
}
private void Expect(string relFile, params IDEEvent[] expecteds)
{
var file = Path.Combine(MergedDir, relFile);
using (var ra = new ReadingArchive(file))
{
var actuals = ra.GetAll <IDEEvent>();
CollectionAssert.AreEqual(expecteds, actuals);
}
}
private EditLocationResults Analyze(string zip)
{
var res = new EditLocationResults {
Zip = zip
};
var file = _io.GetFullPath_In(zip);
using (var ra = new ReadingArchive(file))
{
var locAnal = new RelativeEditLocationAnalysis();
while (ra.HasNext())
{
var @event = ra.GetNext <IDEEvent>();
res.NumEvents++;
var complEvent = @event as CompletionEvent;
if (complEvent == null)
{
continue;
}
var fileName = complEvent.ActiveDocument.FileName;
if (fileName != null && !fileName.EndsWith(".cs"))
{
continue;
}
res.NumCompletionEvents++;
var loc = locAnal.Analyze(complEvent.Context2.SST);
if (!loc.HasEditLocation || loc.Size < 2)
{
continue;
}
res.NumLocations++;
if (complEvent.TerminatedState == TerminationState.Applied)
{
res.AppliedEditLocations.Add(loc);
}
else
{
res.OtherEditLocations.Add(loc);
}
}
}
return(res);
}
private void PrepareZip(params string[] entries)
{
using (var zipFile = new ZipFile())
{
var i = 0;
foreach (var entry in entries)
{
var fileName = (i++) + ".json";
var content = entry == EmptyString ? "" : entry.ToCompactJson();
zipFile.AddEntry(fileName, content);
}
zipFile.Save(_zipPath);
}
_sut = new ReadingArchive(_zipPath);
}
public IEnumerable <CompletionEvent> ReadCce(string zipName)
{
var fullPath = Path.Combine(_dirCcEvents, zipName);
using (var ra = new ReadingArchive(fullPath))
{
while (ra.HasNext())
{
var e = ra.GetNext <IDEEvent>() as CompletionEvent;
if (e != null)
{
yield return(e);
}
}
}
}
public IEnumerable <IDEEvent> Read(string zip)
{
var fullPath = Path.Combine(_dirEvents, zip);
using (var ra = new ReadingArchive(fullPath))
{
while (ra.HasNext())
{
var e = ra.GetNext <IDEEvent>();
if (e != null)
{
yield return(e);
}
}
}
}
public void Run(string rootDir)
{
int numRepos = 0;
int numSolutions = 0;
int numSSTs = 0;
long loc = 0;
foreach (var user in GetSubdirs(rootDir))
{
foreach (var repo in GetSubdirs(Path.Combine(rootDir, user)))
{
numRepos++;
Console.Write("##### {0}/{1} ##############################", user, repo);
var repoPath = Path.Combine(rootDir, user, repo);
foreach (var zip in GetArchives(repoPath))
{
numSolutions++;
Console.WriteLine();
Console.WriteLine("@@ {0} @@", zip);
var zipPath = Path.Combine(repoPath, zip);
using (var ra = new ReadingArchive(zipPath))
{
while (ra.HasNext())
{
numSSTs++;
Console.Write('.');
var ctx = ra.GetNext <Context>();
var sstloc = CountLoc(ctx.SST);
loc += sstloc;
}
}
}
}
}
Console.WriteLine("## RESULTS ##");
Console.WriteLine("#repos: {0}", numRepos);
Console.WriteLine("#solutions: {0}", numSolutions);
Console.WriteLine("#types: {0}", numSSTs);
Console.WriteLine("loc: {0}", loc);
}
public IUserProfileEvent TryGetUserProfile(string zipName)
{
var fullPath = Path.Combine(_dirAllEvents, zipName);
using (var ra = new ReadingArchive(fullPath))
{
while (ra.HasNext())
{
var e = ra.GetNext <IDEEvent>() as IUserProfileEvent;
if (e != null)
{
return(e);
}
}
}
return(null);
}
private void Run(int taskId)
{
string zip;
while (_zips.TryTake(out zip))
{
Console.WriteLine();
Console.WriteLine(@"({0}) Next Zip", taskId);
using (var ra = new ReadingArchive(zip))
{
while (ra.HasNext())
{
var context = ra.GetNext <Context>();
Console.Write('.');
context.SST.Accept(new NameFixTester(), -1);
}
}
}
}
public void AddTuples()
{
var q = new Query
{
type = new CoReTypeName("Ln/T")
};
var expected = Tuple.Create(q, q);
_sut.OpenCache();
_sut.AddTuple(expected);
_sut.CloseCache();
using (var ra = new ReadingArchive(Path.Combine(_dirHistories, "Ln", "T", "0.zip")))
{
var ts = ra.GetAll <Tuple <Query, Query> >();
Assert.AreEqual(1, ts.Count);
Assert.AreEqual(expected, ts[0]);
}
}
public void LastRead()
{
using (var wa = new WritingArchive(_zipPath))
{
wa.AddAsPlainText("x");
wa.AddAsPlainText("y");
wa.AddAsPlainText("z");
}
using (var ra = new ReadingArchive(_zipPath))
{
Assert.Null(ra.CurrentInternalFileName);
ra.GetNextAsPlainText();
Assert.AreEqual("0.json", ra.CurrentInternalFileName);
ra.GetNextAsPlainText();
Assert.AreEqual("1.json", ra.CurrentInternalFileName);
ra.GetNextAsPlainText();
Assert.AreEqual("2.json", ra.CurrentInternalFileName);
}
}
public void CreateArchive_HappyPath()
{
var expecteds = new List <string> {
"a", "b"
};
var zipFileName = Path.Combine(_testRoot, "a.zip");
using (var wa = _sut.CreateArchive(zipFileName))
{
foreach (var c in expecteds)
{
wa.Add(c);
}
}
Assert.True(File.Exists(zipFileName));
var actuals = new ReadingArchive(zipFileName).GetAll <string>();
CollectionAssert.AreEquivalent(expecteds, actuals);
}
private void CreateStatistics(int taskId)
{
string zip;
while (GetNextZip(out zip))
{
_log.CreatingStats(taskId, zip);
var file = _io.GetFullPath_In(zip);
using (var ra = new ReadingArchive(file))
{
var ctxs = ra.GetAllLazy <Context>();
foreach (var ctx in ctxs)
{
ctx.SST.Accept(
new EmDebugVisitor(typesAll, seenMethods, firstAll, superAll, elemAll, invAll),
ctx.TypeShape);
}
}
}
_log.FinishedStatCreation(taskId);
}
private IEnumerable <ICompletionEvent> ReadCce(string zipName)
{
var fullPath = Path.Combine(_dirIn, zipName);
using (var ra = new ReadingArchive(fullPath))
{
while (ra.HasNext())
{
var e = ra.GetNext <IDEEvent>();
var ce = e as CompletionEvent;
if (ce != null)
{
Console.Write('x');
yield return(ce);
}
else
{
Console.Write('.');
}
}
}
}
private void CreateStatistics(int taskId)
{
_log.StartingStatCreation(taskId);
var extractor = new InteractionStatisticsExtractor();
string zip;
while (GetNextZip(out zip))
{
_log.CreatingStats(taskId, zip);
var file = _io.GetFullPath_In(zip);
using (var ra = new ReadingArchive(file))
{
var es = ra.GetAllLazy <IDEEvent>();
var stats = extractor.CreateStatistics(es);
StoreResult(zip, stats);
}
}
_log.FinishedStatCreation(taskId);
}
private void ExtractEvents(IKaVEList <Event> events)
{
var logs = FindSSTLogs(_dirCtxs);
var generator = new EventStreamGenerator();
foreach (var log in logs)
{
Console.WriteLine("##################################################");
Console.WriteLine("reading {0}...", Path.GetFileName(log));
using (var ra = new ReadingArchive(log))
{
var ctxs = ra.GetAll <Context>();
Console.WriteLine("\tFound {0} contexts", ctxs.Count);
Console.Write("\tExtracting events... ");
foreach (var ctx in ctxs)
{
ctx.SST.Accept(generator, events);
Console.Write('.');
}
Console.WriteLine(" done");
}
}
}
public void ReadAsPlainText()
{
using (var wa = new WritingArchive(_zipPath))
{
wa.AddAsPlainText("x");
wa.AddAsPlainText("y");
wa.AddAsPlainText("z");
}
var actuals = new List <string>();
using (var ra = new ReadingArchive(_zipPath))
{
while (ra.HasNext())
{
actuals.Add(ra.GetNextAsPlainText());
}
}
var expecteds = new[] { "x", "y", "z" };
CollectionAssert.AreEqual(expecteds, actuals);
}
private IEnumerable <ICompletionEvent> ReadCce(string zipName)
{
var fullPath = Path.Combine(_dirIn, zipName);
using (var ra = new ReadingArchive(fullPath))
{
while (ra.HasNext())
{
var e = ra.GetNext <IDEEvent>() as CompletionEvent;
if (e != null)
{
if (e.TerminatedState == TerminationState.Applied)
{
var sel = e.LastSelectedProposal;
if (sel != null && sel.Name is IMethodName)
{
yield return(e);
}
}
}
}
}
}
public FailsafeIDEEventReadingArchive(string zip, Action <string, Exception> log)
{
_log = log;
_ra = new ReadingArchive(zip);
}
public void Run()
{
var startTime = DateTime.Now;
var ctxZips = FindInputFiles();
var numZips = ctxZips.Count();
var currentZip = 1;
var numTotalCtxs = 0;
var numTotalUsages = 0;
using (var cache = new ZipFolderLRUCache <CoReTypeName>(_dirOut, 1000))
{
foreach (var fileName in ctxZips)
{
try
{
Log("### processing zip {0}/{1}: {2}", currentZip++, numZips, fileName);
var numLocalCtxs = 0;
var numLocalUsages = 0;
var fullFileIn = _dirIn + fileName;
using (var ra = new ReadingArchive(fullFileIn))
{
Log("reading contexts...");
var ctxs = ra.GetAll <Context>();
var numCtxs = ctxs.Count;
Log("found {0} contexts, exporting usages...\n\n", numCtxs);
var colCounter = 0;
foreach (var ctx in ctxs)
{
if (colCounter == 25)
{
Console.WriteLine();
colCounter = 0;
}
colCounter++;
var usages = ExtractUsages(ctx);
var msg = usages.Count == 0
? "."
: string.Format("{0}", usages.Count);
Console.Write(msg.PadLeft(5));
foreach (var u in usages)
{
cache.GetArchive(u.type).Add(u);
}
numLocalCtxs++;
numLocalUsages += usages.Count;
}
}
Append("\n");
Log(
"--> {0} contexts, {1} usages\n\n",
numLocalCtxs,
numLocalUsages);
numTotalCtxs += numLocalCtxs;
numTotalUsages += numLocalUsages;
}
catch (Exception e)
{
Log("oops, something went wrong...\n{0}", e);
}
}
}
Log("finished! (started at {0})", startTime);
Log(
"found a total of {0} contexts and extracted {1} usages\n\n",
numTotalCtxs,
numTotalUsages);
}
public void Run()
{
Console.WriteLine("Grab Names from Contexts");
var ctx = FindInputFiles();
var numZips = ctx.Count();
var currentZip = 1;
var numTotalCtxs = 0;
var numTotalUsages = 0;
List <Tuple <string, List <string> > > ssts = new KaVEList <Tuple <string, List <string> > >();
foreach (var fileName in ctx)
{
Log("### processing zip {0}/{1}: {2}", currentZip++, numZips, fileName);
var fullFileIn = _dirIn + fileName;
using (var ra = new ReadingArchive(fullFileIn))
{
Log("reading contexts...");
var numCtxs = 0;
while (ra.HasNext())
{
var context = ra.GetNext <Context>();
var list = new KaVEList <string>();
// TODO: grab names in a NameToJsonConverter
numCtxs++;
}
Log("found {0} contexts\n\n", numCtxs);
if (_numMaxZips != -1 && currentZip == _numMaxZips + 1)
{
break;
}
}
}
var typeNameNullCount = 0;
var typeNameCount = 0;
var methodNameNullCount = 0;
var methodNameCount = 0;
List <Tuple <string, string> > wrongSyntaxTypeName = new KaVEList <Tuple <string, string> >();
foreach (var t in ssts)
{
foreach (var s in t.Item2)
{
var type = s.Split(':');
if (type[0].Equals("CSharp.PropertyName"))
{
typeNameCount++;
var name = s.Deserialize <IName>();
if (name.Identifier == "?")
{
wrongSyntaxTypeName.Add(new Tuple <string, string>(s, t.Item1));
typeNameNullCount++;
}
}
}
}
Log("{0} of {1} names are null", typeNameNullCount, typeNameCount);
Log("{0} of {1} names are null", methodNameNullCount, methodNameCount);
double percentageTypeNames = typeNameNullCount / (double)typeNameCount;
double percentageMethodNames = methodNameNullCount / (double)methodNameCount;
Log("TypeNames not parseable: {0}%\n", percentageTypeNames);
Log("PropertyName not parseable: {0}%\n\n", percentageMethodNames);
//showInvalidNames(wrongSyntaxTypeName);
Log("\n\n");
//showInvalidNames(wrongSyntaxMethodName);
if (_writeToFile)
{
Log("File with invalid names written to {0}", _dirOut);
writeToFile(wrongSyntaxTypeName, _dirOut + "/typename.txt");
}
//Log(wrongSyntax[0].Item1 + "\n\n");
//Log(wrongSyntax[0].Item2 + "\n\n");
}
