public void Build(Options opts) { _files = new List<Tuple<string, int>>(); using(var rootTypes = File.CreateText(DataFileRootTypes)) { rootTypes.AutoFlush = true; foreach (var root in opts.Config["import-root"]) { Console.WriteLine("VSS Root: {0}", root); var rootItem = opts.DB.Value.VSSItem[root].Normalize(opts.DB.Value); rootTypes.WriteLine("{0} {1}", rootItem.Spec, rootItem.Type == 0 ? "d" : "f"); WalkItem(rootItem); } File.WriteAllLines(AllFilesList, _files.Select(t => string.Format("{0} {1}", t.Item1, t.Item2)).ToArray()); } Console.WriteLine("All files: " + AllFilesList); FilterFiles(opts); Console.WriteLine("Selected files: " + FilesList); }
public void Build(Options opts, IList<Tuple<string, int>> files) { var coDict = new Dictionary<DateTime, List<Tuple<string, string>>>(); var xrefsCo = new XRefMap(); var xrefs = new XRefMap(); foreach (var file in files.Select(t => t.Item1)) { Console.WriteLine(file); var item = opts.DB.Value.VSSItem[file]; foreach (IVSSItem vssLink in item.Links) { if (!String.Equals(item.Spec, vssLink.Spec, StringComparison.OrdinalIgnoreCase) && !vssLink.Deleted) xrefs.AddRef(item.Spec, vssLink.Spec); foreach (IVSSCheckout vssCheckout in vssLink.Checkouts) { xrefsCo.AddRef(item.Spec, vssCheckout.Username + " at " + vssCheckout.Date); var coDate = vssCheckout.Date.Date; List<Tuple<string, string>> list; if(!coDict.TryGetValue(coDate, out list)) { list = new List<Tuple<string, string>>(); coDict[coDate] = list; } list.Add(Tuple.Create(vssCheckout.Username, item.Spec)); } } } xrefs.Save(DataFileName); xrefs.Save(DataFileUiName, true); xrefsCo.Save(DataFileCoName, true); // save co dict by dates using (var tw = File.CreateText(DataFileCoByDateName)) { foreach (var kvp in coDict.OrderByDescending(kvp => kvp.Key)) { tw.WriteLine("{0:yyyy-MM-dd} ({1} days ago)", kvp.Key, (int)(DateTime.Now - kvp.Key).TotalDays); foreach (var tuple in kvp.Value) { tw.WriteLine("\t{0} at {1}", tuple.Item1, tuple.Item2); } tw.WriteLine(); } } Console.WriteLine("VSS links: {0} pcs", xrefs.Map.Count); Console.WriteLine("Checkouts: {0} pcs", xrefsCo.Map.Count); }
public TfsDriver(Options opts, TextWriter log, bool checkWcStatus) { _opts = opts; _execHelper = new ExecHelper(opts.TfExe, log, false, TimeSpan.FromSeconds(60), true); _commitMessageFile = Path.Combine(Path.GetTempPath(), Path.GetTempFileName()); CheckRepositoryValid(); if (checkWcStatus) CheckWorkingCopyStatus(); }
public void Build(Options opts, bool noPrompt) { if (opts.ImportDriver == "git") { if (!opts.IsGitRepoDirExternal) { if (noPrompt || MessageBox.Show("Repository and work tree will be recreated", "Confirm", MessageBoxButtons.OKCancel) == DialogResult.OK) { File.WriteAllText(Importer.DataFileName, "0\n"); GitDriver.Create(opts.GitExe, opts.GitRepoDir); } } } else if(opts.ImportDriver == "tfs") { if (noPrompt || MessageBox.Show("Work tree will be cleanup", "Confirm", MessageBoxButtons.OKCancel) == DialogResult.OK) { var driver = new TfsDriver(opts, Console.Out, false); driver.CleanupWorkingTree(); } } else if (opts.ImportDriver == "svn") { if (opts.IsSvnRepoDirExternal) { if (noPrompt || MessageBox.Show("Working copy will be recreated (but not repository)", "Confirm", MessageBoxButtons.OKCancel) == DialogResult.OK) { SvnDriver.Checkout(opts.SvnRepoUrl, opts.SvnWorkTreeDir); } } else { if (noPrompt || MessageBox.Show("Repository and Working copy will be recreated", "Confirm", MessageBoxButtons.OKCancel) == DialogResult.OK) { File.WriteAllText(Importer.DataFileName, "0\n"); SvnDriver.CreateRepo(opts.SvnRepoUrl); SvnDriver.Checkout(opts.SvnRepoUrl, opts.SvnWorkTreeDir); } } } else { throw new Exception("Unknown import driver: " + opts.ImportDriver); } }
public void Import(Options opts, List<Commit> commits, bool startNewSession, Action<float> progress = null) { StopImport = false; DogWatch = DateTimeOffset.Now; if(startNewSession && File.Exists(DataFileName)) File.Delete(DataFileName); _opts = opts; _unimportants = opts .Config["unimportant-diff"] .Select(v => { var sep = v.IndexOf('?'); if (sep == -1) throw new ApplicationException("Incorrect unimportant-diff: " + v + "\nAbsent separator '?' between filename and unimportant regex"); var fileNameRx = new Regex(v.Substring(0, sep), RegexOptions.IgnoreCase); var regex = new Regex(v.Substring(sep + 1), RegexOptions.IgnoreCase); return Tuple.Create(fileNameRx, regex); }) .ToList() ; // load censores _censors = LoadCensors(opts); var fromCommit = 0; if (!startNewSession) { if (File.Exists(DataFileName)) { fromCommit = File.ReadAllLines(DataFileName) .Select(x => Int32.Parse(x) + 1) .DefaultIfEmpty(0) .Last() ; } if (MessageBox.Show(string.Format("Cleanu work tree and start import from commit #{0} by {1}", fromCommit, commits[fromCommit].User), "Confirmation", MessageBoxButtons.OKCancel) != DialogResult.OK) return; if (opts.ImportDriver == "tfs") { new TfsDriver(opts, Console.Out, false).CleanupWorkingTree(); } } using (_cache = new VssFileCache(opts.CacheDir, _opts.SourceSafeIni)) using(var log = File.CreateText(LogFileName)) { log.AutoFlush = true; try { IDestinationDriver driver; if (opts.ImportDriver == "git") { driver = new GitDriver(opts.GitExe, opts.GitRepoDir, opts.GitDefaultAuthorDomain, log); } else if (opts.ImportDriver == "tfs") { driver = new TfsDriver(opts, log, true); } else { driver = new SvnDriver(opts.SvnWorkTreeDir, log); } for (var i = fromCommit; i < commits.Count; i++) { var c = commits[i]; Console.WriteLine("[{2,6}/{3}] Import: {0:yyyy-MMM-dd HH:ss:mm}, by {1}", c.At, c.User, i, commits.Count); if (progress != null) progress((float)i / commits.Count); DogWatch = DateTimeOffset.Now; driver.StartRevision(); DogWatch = DateTimeOffset.Now; LoadRevision(driver, c, log); DogWatch = DateTimeOffset.Now; driver.CommitRevision(commits[i].User, c.Comment, commits[i].At); DogWatch = DateTimeOffset.Now; // OK File.AppendAllText(DataFileName, i + "\n"); if (StopImport) break; } } catch (Exception ex) { log.WriteLine(ex.ToString()); throw; } } DogWatch = null; if (StopImport) { Console.WriteLine("Import interrupted."); } else { Console.WriteLine("Import complete."); if (opts.ImportDriver == "git" && !string.IsNullOrWhiteSpace(opts.GitStartAfterImport)) { Process.Start(opts.GitStartAfterImport, opts.GitStartAfterImportArgs.Replace("%REPODIR%", opts.GitRepoDir)); } } }
public static List<CensoreGroup> LoadCensors(Options opts) { return opts .Config["censore-group"] .Select(v => { var fileRxs = opts.Config[string.Format("censore-{0}-file-rx", v)].Select(x => new Regex(x, RegexOptions.IgnoreCase)).ToArray(); var encoding = Encoding.UTF8; var encodingStr = opts.Config[string.Format("censore-{0}-encoding", v)].FirstOrDefault(); if (encodingStr != null) { int codePage; if(int.TryParse(encodingStr, out codePage)) encoding = Encoding.GetEncoding(codePage); else if (encodingStr == "utf-8-no-bom") encoding = new UTF8Encoding(false); else encoding = Encoding.GetEncoding(encodingStr); } var replacements = new List<Tuple<Regex, string>>(); for (var i = 0; ; i++) { var rx = opts.Config[string.Format("censore-{0}-match{1}", v, i)].Select(x => new Regex(x, RegexOptions.IgnoreCase)).FirstOrDefault(); var replace = opts.Config[string.Format("censore-{0}-replace{1}", v, i)].FirstOrDefault(); if (i >= 1 && rx == null && replace == null) break; if (rx == null && replace == null) continue; var rpl = Tuple.Create(rx, replace); replacements.Add(rpl); } return new CensoreGroup { Name = v, FileNameRegex = fileRxs, Replacement = replacements, Encoding = encoding }; }) .ToList(); }
public void Build(Options opts, List<FileRevision> versions) { _opts = opts; if(File.Exists(DataFileName)) File.Delete(DataFileName); if (opts.UnimportantCheckinCommentRx.Length > 0) { // find unimportant revisons var unimportant = versions .Where(r => opts.UnimportantCheckinCommentRx.Any(rx => rx.IsMatch(r.Comment))) .ToList() ; // if unimportant revision is most recent - keep it var keep = new List<FileRevision>(); foreach (var g in unimportant.ToList().GroupBy(r => r.FileSpec)) { var maxRev = versions.Where(r => r.FileSpec == g.Key).Max(r => r.VssVersion); // try find unimportant with most recent revision. this revision should be kept var keepIt = g.FirstOrDefault(r => r.VssVersion == maxRev); if (keepIt != null) keep.Add(keepIt); } keep.ForEach(r => unimportant.Remove(r)); // remove unimportant foreach (var r in unimportant) { versions.Remove(r); } } var orderedRevisions = versions .OrderBy(r => r.At) .ThenBy(r => r.VssVersion) ; // perform mapping vss user -> author MapUsersInComment(orderedRevisions); var commits = SliceToCommits(orderedRevisions); // perfrom author -> commiter mapping MapUsers(commits); // save using(var wr = File.CreateText(DataFileName)) { foreach (var c in commits) { wr.WriteLine("Commit:{0} User:{1} Comment:{2}", c.At.Ticks, c.User, SerializeMultilineText(c.Comment)); c.Files.ToList().ForEach(f => { Debug.Assert(f.At.Kind == DateTimeKind.Utc); wr.WriteLine(" {0}:{1}:{2}", f.VssVersion, f.At.Ticks, f.FileSpec); }); } } Console.WriteLine("{0} commits produced.", commits.Count); Console.WriteLine("Build commits list complete. Check " + DataFileName); }
public CacheBuilder(Options opts) { _options = opts; }
static Int32 Main(string[] args) { _opts = new Options(args); try{ if (args.Length == 0) { args = new [] { "ui" }; } if(args.Any(a => a.StartsWith("/help")) || args.Any(a => a.StartsWith("-h")) || args.Any(a => a.StartsWith("--help"))) { ShowHelp(); return -1; } var verbs = args .Where(a => !a.StartsWith("-")) .Select(a => a.ToLowerInvariant()) .SelectMany(a => { if(a== "all") return new[] { "build-list", "build-list-stats", "build-versions", "build-links", "build-cache", "build-commits", "build-wc", "import", "build-scripts" }; return Enumerable.Repeat(a, 1); }) .ToList() ; if(verbs.Count == 0) { ShowHelp(); return -1; } var unkVerb = verbs.FirstOrDefault(v => v != "ui" && v != "build-list" && v != "build-list-stats" && v != "build-versions" && v != "build-links" && v != "build-cache" && v != "build-commits" && v != "build-wc" && v != "import" && v != "build-scripts"); if(unkVerb != null) { ShowHelp(unkVerb); return -1; } verbs.ForEach(x => ProcessStage(x, true)); } catch(ApplicationException ex) { Console.Error.WriteLine(ex.Message); if (_opts.Ask) { Console.WriteLine("Press any key..."); Console.ReadKey(); } return 1; } catch(Exception ex) { Console.Error.WriteLine(ex.ToString()); if (_opts.Ask) { Console.WriteLine("Press any key..."); Console.ReadKey(); } return 1; } return 0; }
void ReparseConfig() { var opts = new Options(new string[0]); opts.ReadConfig(Program.GetConfigPath()); labelActiveDriver.Text = string.Format("Active driver: {0}", opts.ImportDriver); }
public void Build(Options opts, IList<Tuple<string, int>> files, Action<float> progress = null) { var stopWatch = new Stopwatch(); stopWatch.Start(); using (var cache = new VssFileCache(opts.CacheDir + "-revs", opts.SourceSafeIni)) using(var wr = File.CreateText(DataFileName)) using(var log = File.CreateText(LogFileName)) { wr.AutoFlush = log.AutoFlush = true; var db = opts.DB.Value; var findex = 0; foreach (var spec in files.Select(t => t.Item1)) { findex++; try{ IVSSItem item = db.VSSItem[spec]; var head = item.VersionNumber; var timestamp = item.VSSVersion.Date.Ticks; var cachedData = cache.GetFilePath(spec, head, timestamp); if (cachedData != null) { Console.Write("c"); Save(wr, Load(cachedData)); // next file continue; } Console.Write("[{0,5}/{1,5}] {2} ", findex, files.Count, item.Spec); if (progress != null) progress((float)findex / files.Count); var rotationIndex = 0; var rotationArray = @"|/-\|/-\".ToCharArray(); var latestOnly = IsLatestOnly(opts, spec); var itemRevisions = new List<FileRevision>(); foreach (IVSSVersion ver in item.Versions) { Console.Write("{0}\b", rotationArray[rotationIndex++ % rotationArray.Length]); if (ver.Action.StartsWith("Labeled ") || ver.Action.StartsWith("Branched ")) continue; if (!ver.Action.StartsWith("Checked in ") && !ver.Action.StartsWith("Created ") && !ver.Action.StartsWith("Archived ") && !ver.Action.StartsWith("Rollback to")) { log.WriteLine("Unknown action: " + ver.Action); } var user = ver.Username.ToLowerInvariant(); var fileVersionInfo = new FileRevision { FileSpec = item.Spec, At = ver.Date.ToUniversalTime(), Comment = ver.Comment, VssVersion = ver.VersionNumber, User = user }; try { // can throw exception, but it is not critical fileVersionInfo.Physical = ver.VSSItem.Physical; } catch (Exception ex) { Console.WriteLine("ERROR: Get Physical: " + ex.Message); log.WriteLine("ERROR: Get Physical: {0}", spec); log.WriteLine(ex.ToString()); fileVersionInfo.Physical = "_UNKNOWN_"; } itemRevisions.Add(fileVersionInfo); if (latestOnly) break; Console.Write('.'); } Console.WriteLine(" "); if (itemRevisions.Count > 0) { // some time date of items wrong, but versions - correct. // sort items in correct order and fix dates itemRevisions = itemRevisions.OrderBy(i => i.VssVersion).ToList(); // fix time. make time of each next item greater than all previous var notEarlierThan = itemRevisions[0].At; for (int i = 1; i < itemRevisions.Count; i++) { if (itemRevisions[i].At < notEarlierThan) { itemRevisions[i].At = notEarlierThan + TimeSpan.FromMilliseconds(1); itemRevisions[i].Comment += "\n! Time was fixed during VSS -> SVN conversion. Time can be incorrect !\n"; itemRevisions[i].Comment = itemRevisions[i].Comment.Trim(); } notEarlierThan = itemRevisions[i].At; } } Save(wr, itemRevisions); var tempFile = Path.GetTempFileName(); try { using (var sw = new StreamWriter(tempFile, false, Encoding.UTF8)) Save(sw, itemRevisions); cache.AddFile(spec, head, timestamp, tempFile, false); } finally { if (File.Exists(tempFile)) File.Delete(tempFile); } } catch(Exception ex) { Console.WriteLine("ERROR: {0}", spec); log.WriteLine("ERROR: {0}", spec); log.WriteLine(ex.ToString()); } } } stopWatch.Stop(); Console.WriteLine("Build files versions list complete. Take: {0}", stopWatch.Elapsed); }
bool IsLatestOnly(Options opts, string spec) { return opts.LatestOnly.Contains(spec) || opts.LatestOnlyRx.Any(rx => rx.IsMatch(spec)); }
public void FilterFiles(Options opts) { var files = LoadFrom(AllFilesList); var isInclude = opts.IncludePredicate; var excluded = files.Where(t => !isInclude(t.Item1)).ToList(); files = files.Where(t => isInclude(t.Item1)).ToList(); // write included & excluded File.WriteAllLines(FilesList, files.Select(t => string.Format("{0} {1}", t.Item1, t.Item2)).ToArray()); File.WriteAllLines(ExcludedFilesList, excluded.Select(t => string.Format("{0} {1}", t.Item1, t.Item2)).ToArray()); // calc stats // filter by prefix if (opts.Prefix != null) { files = files .Where(t => t.Item1.Replace('\\', '/').StartsWith(opts.Prefix, StringComparison.OrdinalIgnoreCase)) .ToList() ; } // filter by filter if (opts.FilterRx != null) { files = files .Where(t => opts.FilterRx.IsMatch(t.Item1.Replace('\\', '/'))) .ToList() ; } // build extensions map Console.WriteLine("Files extensions:"); files .Select(t => Path.GetExtension(t.Item1)) .Select(e => e.ToLowerInvariant()) .GroupBy(e => e) .ToList() .ForEach(g => Console.Write("{0}({1}) ", g.Key, g.Count())) ; Console.WriteLine(); Console.WriteLine(); // dump extensions map using (var map = File.CreateText(DataExtsFileName)) { // overview map.WriteLine("== Overview =="); map.WriteLine("<all> : Count: {0,5}, Size: {1,10:0.00} Kb", files.Count, files.Sum(f => (double)f.Item2) / 1024.0); files .Select(t => new { Ext = (Path.GetExtension(t.Item1) ?? "").ToLowerInvariant(), Size = t.Item2 }) .GroupBy(x => x.Ext) .OrderBy(g => g.Sum(f => f.Size)) // .OrderBy(g => g.Key) .ToList() .ForEach(g => map.WriteLine("{0,-9}: Count: {1,5}, Size: {2,10:0.00} Kb, Avg size: {3,7:0.00} Kb", g.Key, g.Count(), g.Sum(f => f.Size) / 1024.0, g.Sum(f => f.Size) / 1024.0 / g.Count())) ; map.WriteLine(); map.WriteLine(); map.WriteLine("== Detailed =="); files .GroupBy(t => (Path.GetExtension(t.Item1) ?? "").ToLowerInvariant()) .ToList() .ForEach(g => { map.WriteLine("{0}({1}):", g.Key, g.Count()); foreach (var f in g.OrderByDescending(ff => ff.Item2)) { map.WriteLine("{0,10} {1}", f.Item2, f.Item1); } map.WriteLine(); }) ; } // dump files by size using (var map = File.CreateText(DataSizesFileName)) { files .Select(t => new { Spec = t.Item1, Size = t.Item2 }) .OrderByDescending(inf => inf.Size) .ToList() .ForEach(inf => map.WriteLine("{0,10:0.0} KiB {1}", inf.Size / 1024.0, inf.Spec)) ; } }
public void Build(Options opts, IList<Tuple<string, int>> importSpecs, Dictionary<string, bool> roots) { if(!Directory.Exists("scripts")) Directory.CreateDirectory("scripts"); using(var swRmLocal = File.CreateText("scripts\\remove-local.bat")) using(var swRmVss = File.CreateText("scripts\\remove-vss.bat")) { swRmVss.WriteLine("set PATH=%PATH%;C:\\Program Files (x86)\\Microsoft Visual SourceSafe"); swRmVss.WriteLine("set SSDIR={0}", Path.GetDirectoryName(opts.SourceSafeIni)); swRmVss.WriteLine("set SSUSER={0}", opts.Config["source-safe-user"].LastOrDefault() ?? "<TODO>"); swRmVss.WriteLine("set SSPWD={0}", opts.Config["source-safe-password"].LastOrDefault() ?? "<TODO>"); swRmVss.WriteLine(); // remove vss projects, local files foreach (var kvp in roots) { swRmVss.WriteLine("ss.exe DELETE \"{0}\"", kvp.Key); if(kvp.Value) swRmLocal.WriteLine("rd /S /Q \"{0}\"", kvp.Key.TrimStart("$/\\".ToCharArray()).Replace('/', '\\').TrimEnd('\\')); else swRmLocal.WriteLine("del /F \"{0}\"", kvp.Key.TrimStart("$/\\".ToCharArray()).Replace('/', '\\')); } } // generate script for update links information + new links.db file using(var sw = File.CreateText("scripts\\apply-link-tokens.bat")) { var file2Token = new XRefMap(); var linksDb = opts.Config["links-db-latest"].FirstOrDefault(); if (linksDb != null) { IDisposable disp = null; try { var user = opts.Config["links-db-user"].FirstOrDefault(); var password = opts.Config["links-db-password"].FirstOrDefault(); if (user != null && password != null) { disp = WindowsImpersonation.Impersonate(new NetworkCredential(user, password)); } // get max file if (Directory.Exists(linksDb)) { linksDb = Directory .GetFiles(linksDb, "*.gz", SearchOption.AllDirectories) .Select(f => new { Ind = Int32.Parse(Path.GetFileNameWithoutExtension(f)), Path = f }) .OrderByDescending(f => f.Ind) .First() .Path ; } using (var src = new GZipStream(File.OpenRead(linksDb), CompressionMode.Decompress)) using (var dst = File.Create(string.Format("_links_db_token_file.{0}.original.txt", Path.GetFileNameWithoutExtension(linksDb)))) src.CopyTo(dst); using (var sr = new StreamReader(new GZipStream(File.OpenRead(linksDb), CompressionMode.Decompress))) file2Token.LoadTokenFile(sr); } finally { if(disp != null) disp.Dispose(); } } var token2Files = file2Token.Inverse(); var importedLinks = new XRefMap(); importedLinks.Load(LinksBuilder.DataFileUiName); // 1. Build set with imported files var importedSet = new HashSet<string>(); foreach (var importedLink in importedLinks.Map.Keys) { importedSet.Add(importedLink); } // files which added to link DB. var add2DbSet = new HashSet<string>(); foreach (var importedLink in importedLinks.Map.Keys) { var otherLinks = importedLinks.Map[importedLink].ToList(); var links = otherLinks.ToList(); links.Insert(0, importedLink); // find or construct token for this bunch of files string token = null; foreach (var link in links) { if(file2Token.Map.ContainsKey(link)) { token = file2Token.Map[link][0]; break; } } // Build new token if(token == null) { token = Path.GetFileName(importedLink).ToLowerInvariant(); if(token2Files.Map.ContainsKey(token)) token = token + "!" + DateTimeOffset.UtcNow.Ticks; Debug.Assert(token2Files.Map.ContainsKey(token) == false); } // buld list which should be added to db var forAdd2Db = otherLinks.Where(l => !importedSet.Contains(l) && !add2DbSet.Contains(l) && !file2Token.Map.ContainsKey(l)).ToList(); // mark imported file with token var file = importedLink.Trim("$/\\".ToCharArray()); Console.WriteLine("ihs:link-token: {0}", file); sw.WriteLine("svn ps ihs:link-token \"{0}\" \"{1}\"", token, file); // add other links into DB forAdd2Db.ForEach(l => { Console.WriteLine("Add 2 DB: {0}", l); file2Token.AddRef(l, token); token2Files.AddRef(token, l); add2DbSet.Add(l); }); Console.WriteLine(); } // check all imported specs (except already handled) if they has token in db foreach (var imported in importSpecs.Select(t => t.Item1).Where(i => !importedSet.Contains(i))) { List<string> files; if(!file2Token.Map.TryGetValue(imported, out files)) continue; var token = files[0]; // mark imported file with token var file = imported.Trim("$/\\".ToCharArray()); Console.WriteLine("Mark with ihs:link-token: {0}", file); sw.WriteLine("svn ps ihs:link-token \"{0}\" \"{1}\"", token, file); } if (linksDb != null) { var updated = string.Format("_links_db_token_file.{0}.updated.txt", Path.GetFileNameWithoutExtension(linksDb)); file2Token.SaveTokenFile(updated); File.AppendAllText(updated, "# hash: skip"); } } }