private static IEnumerable<ICompletionValue> GetFiles(FileInfo[] files)
{
return files
.Select(x => x.Name.StartsWith("_") ? x.Name.Substring(1) : x.Name)
.Select(x => new FileSystemCompletionValue(
displayText: x,
completionText: Path.GetFileNameWithoutExtension(x),
closeValue: true
));
}
/// <summary>
/// Count occurrences of unique words in the given collection of files and write
/// a list of words in descending order to the specified file.
/// </summary>
/// <param name="inputFiles">A collection of files to read.</param>
/// <param name="outputFilePath">The file name for writing the results.</param>
/// <param name="ignoreCase">Indicates whether letter casing should be ignored.</param>
public void CountWords(FileInfo[] inputFiles, string outputFilePath, bool ignoreCase = true)
{
var index = new Dictionary<string, int>();
var docs = new Dictionary<string, List<int>>();
int doc = 0;
var fileWordSeqs = inputFiles.Select(f => GetFileTokens(f.FullName, ignoreCase));
foreach (var sequence in fileWordSeqs)
{
foreach (string word in sequence)
{
int count;
index[word] = index.TryGetValue(word, out count) ? count + 1 : 1;
List<int> docList;
if (docs.TryGetValue(word, out docList))
docList.Add(doc);
else
docs[word] = new List<int> { doc };
}
doc++;
}
// Trim rare words
var words = index.Keys.Where(word => docs.ContainsKey(word) && docs[word].Count < 3).ToList();
foreach (string word in words)
{
index.Remove(word);
docs.Remove(word);
}
var orderedWords = index.OrderByDescending(pair => pair.Value).Select(pair => pair.Key).ToList();
var json = JsonConvert.SerializeObject(orderedWords, Formatting.Indented);
File.WriteAllText(outputFilePath, json);
}
private void FindMatchingFiles(DirectoryInfo dir, FileInfo[] files)
{
FileInfo[] tempfiles = this.Recursive ? dir.GetFiles(this.SearchPattern, SearchOption.AllDirectories) : dir.GetFiles(this.SearchPattern, SearchOption.TopDirectoryOnly);
if (this.ModifiedAfterDate != Convert.ToDateTime("01/01/0001 00:00:00", CultureInfo.CurrentCulture) && this.ModifiedBeforeDate == Convert.ToDateTime("01/01/0001 00:00:00", CultureInfo.CurrentCulture))
{
files = (from f in tempfiles
where f.LastWriteTime > this.ModifiedAfterDate
select f).ToArray();
}
if (this.ModifiedBeforeDate != Convert.ToDateTime("01/01/0001 00:00:00", CultureInfo.CurrentCulture) && this.ModifiedAfterDate == Convert.ToDateTime("01/01/0001 00:00:00", CultureInfo.CurrentCulture))
{
files = (from f in tempfiles
where f.LastWriteTime < this.ModifiedBeforeDate
select f).ToArray();
}
if (this.ModifiedBeforeDate != Convert.ToDateTime("01/01/0001 00:00:00", CultureInfo.CurrentCulture) && this.ModifiedAfterDate != Convert.ToDateTime("01/01/0001 00:00:00", CultureInfo.CurrentCulture))
{
files = (from f in tempfiles
where f.LastWriteTime < this.ModifiedBeforeDate & f.LastWriteTime > this.ModifiedAfterDate
select f).ToArray();
}
if (this.ModifiedBeforeDate == Convert.ToDateTime("01/01/0001 00:00:00", CultureInfo.CurrentCulture) && this.ModifiedAfterDate == Convert.ToDateTime("01/01/0001 00:00:00", CultureInfo.CurrentCulture))
{
files = tempfiles;
}
this.items = files.Select(fileInfo => new TaskItem(fileInfo.FullName)).Cast<ITaskItem>().ToList();
}
private static void CompileCSharp(FileInfo[] sources, IEnumerable<string> dependencyPaths, string assemblyPath, bool isApp)
{
Directory.CreateDirectory(Path.GetDirectoryName(assemblyPath));
EmitResult result;
var assemblyFileName = Path.GetFileNameWithoutExtension(assemblyPath);
var syntaxTrees =
sources.Select(src => CSharpSyntaxTree.ParseText(File.ReadAllText(src.FullName), null, src.FullName)).ToArray();
var references = new List<PortableExecutableReference>()
{
MetadataReference.CreateFromFile(typeof(object).Assembly.Location),
MetadataReference.CreateFromFile(typeof(DynamicAttribute).Assembly.Location),
};
references.AddRange(dependencyPaths.Select(p => MetadataReference.CreateFromFile(p)));
using(var stream = File.Create(assemblyPath))
{
var compilation = CSharpCompilation.Create(assemblyFileName,
syntaxTrees,
references,
new CSharpCompilationOptions(isApp ? OutputKind.ConsoleApplication : OutputKind.DynamicallyLinkedLibrary)
);
// Trick to make version show up in properties dialog
using(var win32ResStream = compilation.CreateDefaultWin32Resources(versionResource: true, noManifest: false, manifestContents: null, iconInIcoFormat: null))
{
result = compilation.Emit(stream, win32Resources: win32ResStream);
}
}
if(!result.Success)
{
File.Delete(assemblyPath);
foreach(var group in result.Diagnostics.GroupBy(d => d.Location.SourceTree.FilePath))
{
Console.WriteLine($"In {group.Key}");
foreach(var diagnostic in group)
{
Console.WriteLine(diagnostic);
}
}
throw new BuildFailedException();
}
var binDir = Path.GetDirectoryName(assemblyPath);
foreach(var reference in references.Skip(2))
File.Copy(reference.FilePath, Path.Combine(binDir, Path.GetFileName(reference.FilePath)));
}
/// <summary>
/// Zips a set of files (that must be rooted at the given RootDir) to a set of zip files in the given OutputDir. The files will be prefixed with the given basename.
/// </summary>
/// <param name="InputFiles">Fully qualified list of files to zip (must be rooted at RootDir).</param>
/// <param name="RootDir">Root Directory where all files will be extracted.</param>
/// <param name="OutputDir">Location to place the set of zip files created.</param>
/// <param name="StagingDir">Location to create zip files before copying them to the OutputDir. If the OutputDir is on a remote file share, staging may be more efficient. Use null to avoid using a staging copy.</param>
/// <param name="ZipBaseName">The basename of the set of zip files.</param>
/// <returns>Some metrics about the zip process.</returns>
/// <remarks>
/// This function tries to zip the files in parallel as fast as it can. It makes no guarantees about how many zip files will be created or which files will be in which zip,
/// but it does try to reasonably balance the file sizes.
/// </remarks>
private static FileInfo[] ParallelZipFiles(FileInfo[] InputFiles, DirectoryReference RootDir, DirectoryReference OutputDir, DirectoryReference StagingDir, string ZipBaseName)
{
// First get the sizes of all the files. We won't parallelize if there isn't enough data to keep the number of zips down.
var FilesInfo = InputFiles
.Select(InputFile => new { File = new FileReference(InputFile), FileSize = InputFile.Length })
.ToList();
// Profiling results show that we can zip 100MB quite fast and it is not worth parallelizing that case and creating a bunch of zips that are relatively small.
const long MinFileSizeToZipInParallel = 1024 * 1024 * 100L;
var bZipInParallel = FilesInfo.Sum(FileInfo => FileInfo.FileSize) >= MinFileSizeToZipInParallel;
// order the files in descending order so our threads pick up the biggest ones first.
// We want to end with the smaller files to more effectively fill in the gaps
var FilesToZip = new ConcurrentQueue<FileReference>(FilesInfo.OrderByDescending(FileInfo => FileInfo.FileSize).Select(FileInfo => FileInfo.File));
// We deliberately avoid Parallel.ForEach here because profiles have shown that dynamic partitioning creates
// too many zip files, and they can be of too varying size, creating uneven work when unzipping later,
// as ZipFile cannot unzip files in parallel from a single archive.
// We can safely assume the build system will not be doing more important things at the same time, so we simply use all our logical cores,
// which has shown to be optimal via profiling, and limits the number of resulting zip files to the number of logical cores.
//
// Sadly, mono implementation of System.IO.Compression is really poor (as of 2015/Aug), causing OOM when parallel zipping a large set of files.
// However, Ionic is MUCH slower than .NET's native implementation (2x+ slower in our build farm), so we stick to the faster solution on PC.
// The code duplication in the threadprocs is unfortunate here, and hopefully we can settle on .NET's implementation on both platforms eventually.
List<Thread> ZipThreads;
ConcurrentBag<FileInfo> ZipFiles = new ConcurrentBag<FileInfo>();
DirectoryReference ZipDir = StagingDir ?? OutputDir;
if (Utils.IsRunningOnMono)
{
ZipThreads = (
from CoreNum in Enumerable.Range(0, bZipInParallel ? Environment.ProcessorCount : 1)
let ZipFileName = FileReference.Combine(ZipDir, string.Format("{0}{1}.zip", ZipBaseName, bZipInParallel ? "-" + CoreNum.ToString("00") : ""))
select new Thread(() =>
{
// don't create the zip unless we have at least one file to add
FileReference File;
if (FilesToZip.TryDequeue(out File))
{
// Create one zip per thread using the given basename
using (var ZipArchive = new Ionic.Zip.ZipFile(ZipFileName.FullName) { CompressionLevel = Ionic.Zlib.CompressionLevel.BestSpeed })
{
// pull from the queue until we are out of files.
do
{
// use fastest compression. In our best case we are CPU bound, so this is a good tradeoff,
// cutting overall time by 2/3 while only modestly increasing the compression ratio (22.7% -> 23.8% for RootEditor PDBs).
// This is in cases of a super hot cache, so the operation was largely CPU bound.
ZipArchive.AddFile(File.FullName, CommandUtils.ConvertSeparators(PathSeparator.Slash, File.Directory.MakeRelativeTo(RootDir)));
} while (FilesToZip.TryDequeue(out File));
ZipArchive.Save();
}
// if we are using a staging dir, copy to the final location and delete the staged copy.
FileInfo ZipFile = new FileInfo(ZipFileName.FullName);
if (StagingDir != null)
{
FileInfo NewZipFile = ZipFile.CopyTo(CommandUtils.MakeRerootedFilePath(ZipFile.FullName, StagingDir.FullName, OutputDir.FullName));
ZipFile.Delete();
ZipFile = NewZipFile;
}
ZipFiles.Add(ZipFile);
}
})).ToList();
}
else
{
ZipThreads = (
from CoreNum in Enumerable.Range(0, bZipInParallel ? Environment.ProcessorCount : 1)
let ZipFileName = FileReference.Combine(ZipDir, string.Format("{0}{1}.zip", ZipBaseName, bZipInParallel ? "-" + CoreNum.ToString("00") : ""))
select new Thread(() =>
{
// don't create the zip unless we have at least one file to add
FileReference File;
if (FilesToZip.TryDequeue(out File))
{
// Create one zip per thread using the given basename
using (var ZipArchive = System.IO.Compression.ZipFile.Open(ZipFileName.FullName, System.IO.Compression.ZipArchiveMode.Create))
{
// pull from the queue until we are out of files.
do
{
// use fastest compression. In our best case we are CPU bound, so this is a good tradeoff,
// cutting overall time by 2/3 while only modestly increasing the compression ratio (22.7% -> 23.8% for RootEditor PDBs).
// This is in cases of a super hot cache, so the operation was largely CPU bound.
// Also, sadly, mono appears to have a bug where nothing you can do will properly set the LastWriteTime on the created entry,
// so we have to ignore timestamps on files extracted from a zip, since it may have been created on a Mac.
ZipFileExtensions.CreateEntryFromFile(ZipArchive, File.FullName, CommandUtils.ConvertSeparators(PathSeparator.Slash, File.MakeRelativeTo(RootDir)), System.IO.Compression.CompressionLevel.Fastest);
} while (FilesToZip.TryDequeue(out File));
}
// if we are using a staging dir, copy to the final location and delete the staged copy.
FileInfo ZipFile = new FileInfo(ZipFileName.FullName);
if (StagingDir != null)
{
FileInfo NewZipFile = ZipFile.CopyTo(CommandUtils.MakeRerootedFilePath(ZipFile.FullName, StagingDir.FullName, OutputDir.FullName));
ZipFile.Delete();
ZipFile = NewZipFile;
}
ZipFiles.Add(ZipFile);
}
})).ToList();
}
ZipThreads.ForEach(thread => thread.Start());
ZipThreads.ForEach(thread => thread.Join());
return ZipFiles.OrderBy(x => x.Name).ToArray();
}
/// <summary>
/// Creates a manifest from a flat list of files (in many folders) and a BaseFolder from which they are rooted.
/// </summary>
/// <param name="InFiles">List of full file paths</param>
/// <param name="RootDir">Root folder for all the files. All files must be relative to this RootDir.</param>
public TempStorageManifest(FileInfo[] InFiles, DirectoryReference RootDir)
{
Files = InFiles.Select(x => new TempStorageFile(x, RootDir)).ToArray();
}
public static string ResolveJavaArchiveLoadRequest(string context, string name, FileInfo[] ImplicitReferences)
{
Func<string[], string> f =
list =>
{
var r = name.Replace(".", "/") + ".class";
foreach (var item in list)
{
var c = default(string[]);
if (!FileContentLookup.ContainsKey(item))
{
// https://sites.google.com/a/jsc-solutions.net/backlog/knowledge-base/2014/201402/20140209
Console.WriteLine("JavaArchiveExtensions.ResolveJavaArchiveLoadRequest " +
new
{
jar = Path.GetFileNameWithoutExtension(item),
context,
name
}
);
var zip = ZIPArchive.GetFiles(item);
FileContentLookup[item] = zip.Select(k => k.Name).ToArray();
}
c = FileContentLookup[item];
var u = c.FirstOrDefault(k => k == r);
if (u != null)
return item;
}
return null;
};
var x0 = f(ImplicitReferences.Select(k => k.FullName).ToArray());
if (x0 != null)
return x0;
Console.WriteLine("JavaArchiveExtensions.ResolveJavaArchiveLoadRequest " +
new
{
context,
name
}
);
var x1 = f(ContextToFileArray(context));
//MessageBox.Show("@ JavaArchiveResolve: " + name);
//if (name == "com.amazonaws.AmazonWebServiceRequest")
// return @"C:\util\aws-android-sdk-0.2.0\lib\aws-android-sdk-0.2.0-core.jar";
return x1;
}
/// <summary>
/// Определяет тип кривых файлы которых находятся в указанной папке
/// </summary>
/// <param name="fiList"></param>
/// <returns></returns>
private string DetectTypeOfCurves(FileInfo[] fiList)
{
string[] types = {".txt", ".s1p", ".s2p", ".s3p", ".s4p"};
string res = fiList.Select(fi => Path.GetExtension(fi.FullName.ToLower())).FirstOrDefault(ext => types.Contains(ext));
return res ?? "";
}
private static HashFile[] ReadHashFiles(FileInfo[] files, long maxSize)
{
Console.Write("Reading Hashfiles");
var hashFiles = files
.Select(f => HashFile.Create(f))
.Where(h => h != null && h.SourceFile != null && h.SourceFile.Exists && (maxSize == -1 || h.SourceFile.Length <= maxSize))
.ToArray();
var filtered =
(
from file in hashFiles
join hash in hashdata.Values
on file.SourceName.ToLower()
equals hash.Path.ToLower()
into hashlist
from h in hashlist.DefaultIfEmpty()
where
h == null ||
// h.Updated.AddMinutes(20) < DateTime.Now ||
(file.SourceFile.Length != h.FileLength || file.SourceFile.LastWriteTime != h.LastWriteTime)
select file
).ToArray();
Console.WriteLine(" - Loaded {0} of {2} Hashfile{1}", filtered.Length, hashFiles.Length == 1 ? "" : "s", hashFiles.Length);
return filtered.Length < 1 ? null : filtered;
}
public ImageFolder(
DirectoryInfo thisDir, DirectoryInfo rootDir, FileInfo[] imageFiles)
{
this.thisDir = thisDir;
this.rootDir = rootDir;
this.files = imageFiles.Select(x => new ImageFile(x)).ToArray();
}
