private IEnumerable <string> ReadZip() { using (var ra = new ReadingArchive(_zipPath)) { return(ra.GetAll <string>()); } }
public void Run() { Console.Write("looking (recursively) for events in folder {0}\n", Path.GetFullPath(_eventsDir)); /* * Each .zip that is contained in the eventsDir represents all events * that we have collected for a specific user, the folder represents the * first day when the user uploaded data. */ var userZips = FindUserZips(); var ZipIterator = 0; foreach (var userZip in userZips) { ZipIterator += 1; Console.Write("\n#### processing user zip: {0} #####\n", userZip); // open the .zip file ... using (IReadingArchive ra = new ReadingArchive(Path.Combine(_eventsDir, userZip))) { // ... and iterate over content. while (ra.HasNext()) { /* * within the userZip, each stored event is contained as a * single file that contains the Json representation of a * subclass of IDEEvent. */ try { var e = ra.GetNext <IDEEvent>(); // the events can then be processed individually foreach (Process task in _tasks) { task.process(e); } } catch (System.InvalidOperationException e) { Console.WriteLine(e.Message); } catch (Newtonsoft.Json.JsonReaderException e) { Console.WriteLine(e.Message); } catch (Exception e) { Console.WriteLine("not tested error!" + e.Message); } } } //getThe result for this actual zip and stock it in the .txt Result file. foreach (Process task in _tasks) { task.getResult(ZipIterator + " / " + userZips.Count); } } }
private void Process(int taskId) { Log("({0}) starting", taskId); string relZip; while (GetNextZip(taskId, out relZip)) { var zip = Path.Combine(_baseDir, relZip); using (var ra = new ReadingArchive(zip)) { while (ra.HasNext()) { try { var e = ra.GetNext <IDEEvent>(); // ReSharper disable once UnusedVariable var hc = e.GetHashCode(); } catch { Log("Exception thrown in {0} ({1}).", zip, ra.CurrentInternalFileName); throw; } } } } Log("({0}) stopping", taskId); }
public void Run() { Console.Write("looking (recursively) for events in folder {0}\n", Path.GetFullPath(_eventsDir)); /* * Each .zip that is contained in the eventsDir represents all events * that we have collected for a specific user, the folder represents the * first day when the user uploaded data. */ var userZips = FindUserZips(); foreach (var userZip in userZips) { Console.Write("\n#### processing user zip: {0} #####\n", userZip); // open the .zip file ... using (IReadingArchive ra = new ReadingArchive(Path.Combine(_eventsDir, userZip))) { // ... and iterate over content. while (ra.HasNext()) { /* * within the userZip, each stored event is contained as a * single file that contains the Json representation of a * subclass of IDEEvent. */ var e = ra.GetNext <IDEEvent>(); // the events can then be processed individually process(e); } } } }
private void WriteEventStream(string zip, string relFile) { using (var ra = new ReadingArchive(zip)) { var events = ra.GetAll <IDEEvent>(); _export.Write(events, relFile); } }
protected IList <T> ReadZip <T>(string file) { Assert.IsTrue(file.StartsWith(DirTestRoot)); using (var ra = new ReadingArchive(file)) { return(ra.GetAll <T>()); } }
public IEnumerable <IDEEvent> ReadEvents(string exportFile) { using (var ra = new ReadingArchive(exportFile)) { while (ra.HasNext()) { yield return(ra.GetNext <IDEEvent>()); } } }
private void AssertZipContent(string fileName, params string[] expectedsArr) { using (var ra = new ReadingArchive(fileName)) { var expecteds = new List <string>(); expecteds.AddRange(expectedsArr); var actuals = ra.GetAll <string>(); Assert.AreEqual(expecteds, actuals); } }
private static IEnumerable <IDEEvent> ReadEventsFromZip(string zip) { using (var ra = new ReadingArchive(zip)) { while (ra.HasNext()) { yield return(ra.GetNext <IDEEvent>()); } } }
private IEnumerable <IDEEvent> ReadEventsForUser(string userZip) { using (var ra = new ReadingArchive(_root + userZip)) { while (ra.HasNext()) { yield return(ra.GetNext <IDEEvent>()); } } }
private void Expect(string relFile, params IDEEvent[] expecteds) { var file = Path.Combine(MergedDir, relFile); using (var ra = new ReadingArchive(file)) { var actuals = ra.GetAll <IDEEvent>(); CollectionAssert.AreEqual(expecteds, actuals); } }
private EditLocationResults Analyze(string zip) { var res = new EditLocationResults { Zip = zip }; var file = _io.GetFullPath_In(zip); using (var ra = new ReadingArchive(file)) { var locAnal = new RelativeEditLocationAnalysis(); while (ra.HasNext()) { var @event = ra.GetNext <IDEEvent>(); res.NumEvents++; var complEvent = @event as CompletionEvent; if (complEvent == null) { continue; } var fileName = complEvent.ActiveDocument.FileName; if (fileName != null && !fileName.EndsWith(".cs")) { continue; } res.NumCompletionEvents++; var loc = locAnal.Analyze(complEvent.Context2.SST); if (!loc.HasEditLocation || loc.Size < 2) { continue; } res.NumLocations++; if (complEvent.TerminatedState == TerminationState.Applied) { res.AppliedEditLocations.Add(loc); } else { res.OtherEditLocations.Add(loc); } } } return(res); }
private void PrepareZip(params string[] entries) { using (var zipFile = new ZipFile()) { var i = 0; foreach (var entry in entries) { var fileName = (i++) + ".json"; var content = entry == EmptyString ? "" : entry.ToCompactJson(); zipFile.AddEntry(fileName, content); } zipFile.Save(_zipPath); } _sut = new ReadingArchive(_zipPath); }
public IEnumerable <CompletionEvent> ReadCce(string zipName) { var fullPath = Path.Combine(_dirCcEvents, zipName); using (var ra = new ReadingArchive(fullPath)) { while (ra.HasNext()) { var e = ra.GetNext <IDEEvent>() as CompletionEvent; if (e != null) { yield return(e); } } } }
public IEnumerable <IDEEvent> Read(string zip) { var fullPath = Path.Combine(_dirEvents, zip); using (var ra = new ReadingArchive(fullPath)) { while (ra.HasNext()) { var e = ra.GetNext <IDEEvent>(); if (e != null) { yield return(e); } } } }
public void Run(string rootDir) { int numRepos = 0; int numSolutions = 0; int numSSTs = 0; long loc = 0; foreach (var user in GetSubdirs(rootDir)) { foreach (var repo in GetSubdirs(Path.Combine(rootDir, user))) { numRepos++; Console.Write("##### {0}/{1} ##############################", user, repo); var repoPath = Path.Combine(rootDir, user, repo); foreach (var zip in GetArchives(repoPath)) { numSolutions++; Console.WriteLine(); Console.WriteLine("@@ {0} @@", zip); var zipPath = Path.Combine(repoPath, zip); using (var ra = new ReadingArchive(zipPath)) { while (ra.HasNext()) { numSSTs++; Console.Write('.'); var ctx = ra.GetNext <Context>(); var sstloc = CountLoc(ctx.SST); loc += sstloc; } } } } } Console.WriteLine("## RESULTS ##"); Console.WriteLine("#repos: {0}", numRepos); Console.WriteLine("#solutions: {0}", numSolutions); Console.WriteLine("#types: {0}", numSSTs); Console.WriteLine("loc: {0}", loc); }
public IUserProfileEvent TryGetUserProfile(string zipName) { var fullPath = Path.Combine(_dirAllEvents, zipName); using (var ra = new ReadingArchive(fullPath)) { while (ra.HasNext()) { var e = ra.GetNext <IDEEvent>() as IUserProfileEvent; if (e != null) { return(e); } } } return(null); }
private void Run(int taskId) { string zip; while (_zips.TryTake(out zip)) { Console.WriteLine(); Console.WriteLine(@"({0}) Next Zip", taskId); using (var ra = new ReadingArchive(zip)) { while (ra.HasNext()) { var context = ra.GetNext <Context>(); Console.Write('.'); context.SST.Accept(new NameFixTester(), -1); } } } }
public void AddTuples() { var q = new Query { type = new CoReTypeName("Ln/T") }; var expected = Tuple.Create(q, q); _sut.OpenCache(); _sut.AddTuple(expected); _sut.CloseCache(); using (var ra = new ReadingArchive(Path.Combine(_dirHistories, "Ln", "T", "0.zip"))) { var ts = ra.GetAll <Tuple <Query, Query> >(); Assert.AreEqual(1, ts.Count); Assert.AreEqual(expected, ts[0]); } }
public void LastRead() { using (var wa = new WritingArchive(_zipPath)) { wa.AddAsPlainText("x"); wa.AddAsPlainText("y"); wa.AddAsPlainText("z"); } using (var ra = new ReadingArchive(_zipPath)) { Assert.Null(ra.CurrentInternalFileName); ra.GetNextAsPlainText(); Assert.AreEqual("0.json", ra.CurrentInternalFileName); ra.GetNextAsPlainText(); Assert.AreEqual("1.json", ra.CurrentInternalFileName); ra.GetNextAsPlainText(); Assert.AreEqual("2.json", ra.CurrentInternalFileName); } }
public void CreateArchive_HappyPath() { var expecteds = new List <string> { "a", "b" }; var zipFileName = Path.Combine(_testRoot, "a.zip"); using (var wa = _sut.CreateArchive(zipFileName)) { foreach (var c in expecteds) { wa.Add(c); } } Assert.True(File.Exists(zipFileName)); var actuals = new ReadingArchive(zipFileName).GetAll <string>(); CollectionAssert.AreEquivalent(expecteds, actuals); }
private void CreateStatistics(int taskId) { string zip; while (GetNextZip(out zip)) { _log.CreatingStats(taskId, zip); var file = _io.GetFullPath_In(zip); using (var ra = new ReadingArchive(file)) { var ctxs = ra.GetAllLazy <Context>(); foreach (var ctx in ctxs) { ctx.SST.Accept( new EmDebugVisitor(typesAll, seenMethods, firstAll, superAll, elemAll, invAll), ctx.TypeShape); } } } _log.FinishedStatCreation(taskId); }
private IEnumerable <ICompletionEvent> ReadCce(string zipName) { var fullPath = Path.Combine(_dirIn, zipName); using (var ra = new ReadingArchive(fullPath)) { while (ra.HasNext()) { var e = ra.GetNext <IDEEvent>(); var ce = e as CompletionEvent; if (ce != null) { Console.Write('x'); yield return(ce); } else { Console.Write('.'); } } } }
private void CreateStatistics(int taskId) { _log.StartingStatCreation(taskId); var extractor = new InteractionStatisticsExtractor(); string zip; while (GetNextZip(out zip)) { _log.CreatingStats(taskId, zip); var file = _io.GetFullPath_In(zip); using (var ra = new ReadingArchive(file)) { var es = ra.GetAllLazy <IDEEvent>(); var stats = extractor.CreateStatistics(es); StoreResult(zip, stats); } } _log.FinishedStatCreation(taskId); }
private void ExtractEvents(IKaVEList <Event> events) { var logs = FindSSTLogs(_dirCtxs); var generator = new EventStreamGenerator(); foreach (var log in logs) { Console.WriteLine("##################################################"); Console.WriteLine("reading {0}...", Path.GetFileName(log)); using (var ra = new ReadingArchive(log)) { var ctxs = ra.GetAll <Context>(); Console.WriteLine("\tFound {0} contexts", ctxs.Count); Console.Write("\tExtracting events... "); foreach (var ctx in ctxs) { ctx.SST.Accept(generator, events); Console.Write('.'); } Console.WriteLine(" done"); } } }
public void ReadAsPlainText() { using (var wa = new WritingArchive(_zipPath)) { wa.AddAsPlainText("x"); wa.AddAsPlainText("y"); wa.AddAsPlainText("z"); } var actuals = new List <string>(); using (var ra = new ReadingArchive(_zipPath)) { while (ra.HasNext()) { actuals.Add(ra.GetNextAsPlainText()); } } var expecteds = new[] { "x", "y", "z" }; CollectionAssert.AreEqual(expecteds, actuals); }
private IEnumerable <ICompletionEvent> ReadCce(string zipName) { var fullPath = Path.Combine(_dirIn, zipName); using (var ra = new ReadingArchive(fullPath)) { while (ra.HasNext()) { var e = ra.GetNext <IDEEvent>() as CompletionEvent; if (e != null) { if (e.TerminatedState == TerminationState.Applied) { var sel = e.LastSelectedProposal; if (sel != null && sel.Name is IMethodName) { yield return(e); } } } } } }
public FailsafeIDEEventReadingArchive(string zip, Action <string, Exception> log) { _log = log; _ra = new ReadingArchive(zip); }
public void Run() { var startTime = DateTime.Now; var ctxZips = FindInputFiles(); var numZips = ctxZips.Count(); var currentZip = 1; var numTotalCtxs = 0; var numTotalUsages = 0; using (var cache = new ZipFolderLRUCache <CoReTypeName>(_dirOut, 1000)) { foreach (var fileName in ctxZips) { try { Log("### processing zip {0}/{1}: {2}", currentZip++, numZips, fileName); var numLocalCtxs = 0; var numLocalUsages = 0; var fullFileIn = _dirIn + fileName; using (var ra = new ReadingArchive(fullFileIn)) { Log("reading contexts..."); var ctxs = ra.GetAll <Context>(); var numCtxs = ctxs.Count; Log("found {0} contexts, exporting usages...\n\n", numCtxs); var colCounter = 0; foreach (var ctx in ctxs) { if (colCounter == 25) { Console.WriteLine(); colCounter = 0; } colCounter++; var usages = ExtractUsages(ctx); var msg = usages.Count == 0 ? "." : string.Format("{0}", usages.Count); Console.Write(msg.PadLeft(5)); foreach (var u in usages) { cache.GetArchive(u.type).Add(u); } numLocalCtxs++; numLocalUsages += usages.Count; } } Append("\n"); Log( "--> {0} contexts, {1} usages\n\n", numLocalCtxs, numLocalUsages); numTotalCtxs += numLocalCtxs; numTotalUsages += numLocalUsages; } catch (Exception e) { Log("oops, something went wrong...\n{0}", e); } } } Log("finished! (started at {0})", startTime); Log( "found a total of {0} contexts and extracted {1} usages\n\n", numTotalCtxs, numTotalUsages); }
public void Run() { Console.WriteLine("Grab Names from Contexts"); var ctx = FindInputFiles(); var numZips = ctx.Count(); var currentZip = 1; var numTotalCtxs = 0; var numTotalUsages = 0; List <Tuple <string, List <string> > > ssts = new KaVEList <Tuple <string, List <string> > >(); foreach (var fileName in ctx) { Log("### processing zip {0}/{1}: {2}", currentZip++, numZips, fileName); var fullFileIn = _dirIn + fileName; using (var ra = new ReadingArchive(fullFileIn)) { Log("reading contexts..."); var numCtxs = 0; while (ra.HasNext()) { var context = ra.GetNext <Context>(); var list = new KaVEList <string>(); // TODO: grab names in a NameToJsonConverter numCtxs++; } Log("found {0} contexts\n\n", numCtxs); if (_numMaxZips != -1 && currentZip == _numMaxZips + 1) { break; } } } var typeNameNullCount = 0; var typeNameCount = 0; var methodNameNullCount = 0; var methodNameCount = 0; List <Tuple <string, string> > wrongSyntaxTypeName = new KaVEList <Tuple <string, string> >(); foreach (var t in ssts) { foreach (var s in t.Item2) { var type = s.Split(':'); if (type[0].Equals("CSharp.PropertyName")) { typeNameCount++; var name = s.Deserialize <IName>(); if (name.Identifier == "?") { wrongSyntaxTypeName.Add(new Tuple <string, string>(s, t.Item1)); typeNameNullCount++; } } } } Log("{0} of {1} names are null", typeNameNullCount, typeNameCount); Log("{0} of {1} names are null", methodNameNullCount, methodNameCount); double percentageTypeNames = typeNameNullCount / (double)typeNameCount; double percentageMethodNames = methodNameNullCount / (double)methodNameCount; Log("TypeNames not parseable: {0}%\n", percentageTypeNames); Log("PropertyName not parseable: {0}%\n\n", percentageMethodNames); //showInvalidNames(wrongSyntaxTypeName); Log("\n\n"); //showInvalidNames(wrongSyntaxMethodName); if (_writeToFile) { Log("File with invalid names written to {0}", _dirOut); writeToFile(wrongSyntaxTypeName, _dirOut + "/typename.txt"); } //Log(wrongSyntax[0].Item1 + "\n\n"); //Log(wrongSyntax[0].Item2 + "\n\n"); }