private static IDictionary<string, FileStats> AnalyzeDirectory(int level, string dir)
{
IDictionary<string,FileStats> dirStats = new Dictionary<string,FileStats>();
foreach (string filepath in Directory.EnumerateFiles(dir))
{
FileInfo file = new FileInfo(filepath);
string extension = file.Extension;
int lines = 0;
using(StreamReader reader = new StreamReader(filepath))
{
while(reader.ReadLine() != null)
{
lines++;
}
}
if(dirStats.ContainsKey(extension))
{
dirStats[extension].Count++;
dirStats[extension].Lines += lines;
}
else
{
FileStats fs = new FileStats(extension);
fs.Count = 1;
fs.Lines = lines;
dirStats.Add(extension, fs);
}
}
WriteStats(true, level, dir, dirStats);
bool hassubDir = false;
foreach (string subdir in Directory.EnumerateDirectories(dir))
{
string localDir = subdir.Substring(subdir.LastIndexOf('\\') + 1);
bool skipdir = false;
foreach(string dirFilter in DIR_FILTERS)
{
if(localDir.StartsWith(dirFilter))
{
skipdir = true;
break; ;
}
}
if (skipdir) continue;
IDictionary<string,FileStats> subdirStats = AnalyzeDirectory(level+1, subdir);
AddStats(dirStats, subdirStats);
hassubDir = true;
}
if (hassubDir)
{
WriteStats(false, level, dir, dirStats);
}
return dirStats;
}
public FileEntry CreateFile(FileHandle fileHandle, FileStats stats)
{
if (stats.Type.HasFlag(FileType.File))
{
return(new RegularFileEntry(this, fileHandle, stats));
}
if (stats.Type.HasFlag(FileType.Directory))
{
return(new DirectoryEntry(this, fileHandle, stats));
}
return(new UnknownFileEntry(this, fileHandle, stats));
}
public LocalFileHandle(string path, FileStats stats)
{
if (path == "/")
{
Path = path;
_fileType = FileType.Directory;
_isRoot = true;
return;
}
Path = path;
_fileType = stats.Type;
_isRoot = false;
}
public async Task Setup()
{
var services = new ServiceCollection();
services.AddSingleton(typeof(ILogger <>), typeof(NullLogger <>));
services.AddSingleton <ILogger, NullLogger>();
services.AddSingleton <ILoggerFactory, NullLoggerFactory>();
services.TryAddSingletonFileService(builder => builder.TryAddMemoryFileSystem("memory"));
_serviceProvider = services.BuildServiceProvider();
_fileService = _serviceProvider.GetRequiredService <IFileService>();
var root = await _fileService.CreateFileHandle(Url.Parse("file://memory/"));
_exampleMp4FileHandle = await _fileService.CreateFile(
root,
"example.mp4",
Resources.ReadEmbeddedFile(typeof(RendererBenchmark).Assembly, "/Resources/example.mp4"));
_exampleMp4FileStats = await _fileService.Stat(_exampleMp4FileHandle);
_exampleMp3FileHandle = await _fileService.CreateFile(
root,
"example.mp3",
Resources.ReadEmbeddedFile(typeof(RendererBenchmark).Assembly, "/Resources/example.mp3"));
_exampleMp3FileStats = await _fileService.Stat(_exampleMp3FileHandle);
_exampleMp3WithCoverFileHandle = await _fileService.CreateFile(
root,
"example-with-cover.mp3",
Resources.ReadEmbeddedFile(typeof(RendererBenchmark).Assembly, "/Resources/example-with-cover.mp3"));
_exampleMp3WithCoverFileStats = await _fileService.Stat(_exampleMp3WithCoverFileHandle);
_examplePngFileHandle = await _fileService.CreateFile(
root,
"example.png",
Resources.ReadEmbeddedFile(typeof(RendererBenchmark).Assembly, "/Resources/example.png"));
_examplePngFileStats = await _fileService.Stat(_examplePngFileHandle);
_renderContext = new ThumbnailsRenderContext();
_audioFileRenderer = new AudioFileRenderer(_fileService);
_videoFileRenderer = new VideoFileRenderer(_fileService);
_renderOption = new ThumbnailsRenderOption {
Size = 512
};
}
private void btnChooseFile_Click(object sender, RoutedEventArgs e)
{
var dlg = new CommonOpenFileDialog()
{
Title = "Vælg en fil",
IsFolderPicker = false,
AddToMostRecentlyUsedList = false,
AllowNonFileSystemItems = false,
EnsureFileExists = true,
EnsurePathExists = true,
EnsureReadOnly = false,
EnsureValidNames = true,
Multiselect = false,
ShowPlacesList = true
};
if (dlg.ShowDialog() == CommonFileDialogResult.Ok)
{
// TODO:
// Show the path (found in dlg.FileName) in the correct textbox so the user can see what was chosen
tbxFilePath.Text = dlg.FileName;
FileInfo fileInfo = new FileInfo(dlg.FileName);
cbxAdvSearch.IsEnabled = false;
cbxModUTC.IsEnabled = true;
// TODO:
// Use the fileInfo to create a FileStats object and show the relevant data for the user
FileStats fileStats = StatsCreator.CreateFileStats(fileInfo);
lblName.Content = fileStats.Name;
lblType.Content = "File";
CalculateSize(fileStats.FileSize);
lblCreationTime.Content = fileStats.FileCreated;
lblLastModified.Content = fileStats.LastModified;
lblFileExtension.Content = fileStats.FileExtension;
wpFileCount.Visibility = Visibility.Collapsed;
wpFileExtension.Visibility = Visibility.Visible;
cbxAdvSearch.IsChecked = false;
wpLastModified.Visibility = Visibility.Visible;
wpCreationTime.Visibility = Visibility.Visible;
imgIcon.Source = BitmapToImageSource(fileStats.GetBitmap());
}
}
private void Execute(IGeneratedAlgorithm ca)
{
// If there is a message update listener, try to add in the engine to listen to messages.
if (this.MessageUpdated != null)
{
DebuggingSupport.TryAddRemoveEventListenerDynamic(ca, OnMessageUpdated, add: true);
}
// Register the ProgressChanged handler only while doing inference within InferenceEngine.
// We do not want the handler to run if the user accesses the GeneratedAlgorithms directly.
ca.ProgressChanged += OnProgressChanged;
try
{
Stopwatch s = null;
if (ShowTimings)
{
s = new Stopwatch();
s.Start();
FileStats.Clear();
}
if (ResetOnObservedValueChanged)
{
ca.Execute(NumberOfIterations);
}
else
{
ca.Update(NumberOfIterations - ca.NumberOfIterationsDone);
}
if (s != null)
{
long elapsed = s.ElapsedMilliseconds;
Console.WriteLine("Inference time was {1}ms (max {0} iterations)",
NumberOfIterations, elapsed);
if (FileStats.ReadCount > 0 || FileStats.WriteCount > 0)
{
Console.WriteLine("{0} file reads {1} file writes", FileStats.ReadCount, FileStats.WriteCount);
}
}
}
finally
{
ca.ProgressChanged -= OnProgressChanged;
if (this.MessageUpdated != null)
{
DebuggingSupport.TryAddRemoveEventListenerDynamic(ca, OnMessageUpdated, add: false);
}
}
}
private void cbxModUTC_Click(object sender, RoutedEventArgs e)
{
FileInfo fileInfo = new FileInfo(tbxFilePath.Text);
FileStats directoryStats = StatsCreator.CreateFileStats(fileInfo);
if (cbxModUTC.IsChecked == true)
{
lblLastModified.Content = directoryStats.LastModifiedUTC;
lblCreationTime.Content = directoryStats.FileCreatedUTC;
}
else if (cbxModUTC.IsChecked == false)
{
lblLastModified.Content = directoryStats.LastModified;
lblCreationTime.Content = directoryStats.FileCreated;
}
}
/// <summary>
/// Removes the stats of the given file from the DB.
/// </summary>
/// <param name="file"></param>
public void RemoveFileStats(FileStats file)
{
SQLiteParameter path = new SQLiteParameter("@path");
path.Value = file.FilePath;
path.DbType = System.Data.DbType.String;
using (SQLiteCommand cmd = new SQLiteCommand(dbConn))
{
cmd.Parameters.Add(path);
// Remove words related to file.
cmd.CommandText = "DELETE FROM Words WHERE file=@path";
cmd.ExecuteNonQuery();
// Remove the stats for file
cmd.CommandText = "DELETE FROM Files WHERE path=@path";
cmd.ExecuteNonQuery();
}
}
public ActionResult UploadFile()
{
//recieving file from form
HttpPostedFileBase hpf = Request.Files[0] as HttpPostedFileBase;
//converting file contenttosting
string FileContent = new StreamReader(hpf.InputStream).ReadToEnd();
//processingfile content
FileStats result = new FileStats();
result = FileAnalyzer.AnalyzeFile(FileContent);
result.FileName = Path.GetFileName(hpf.FileName);
//adding file to db
fileStatsRepository.InsertFileStats(result);
fileStatsRepository.Save();
//displaying results
return(View("Index", result));
}
private Metric CreateStat(BuildInfo current, CommitStats commitStats, FileStats fileStats)
{
var stat = new Metric();
stat.BuildCommit = current.CommitHash;
stat.BuildResult = (int)current.BuildResult;
stat.Commit = commitStats.Commit;
stat.ExportDateUtc = DateTime.UtcNow;
stat.NumberOfDistinctCommitters = fileStats.NumberOfDistinctCommitters;
stat.NumberOfModifiedLines = fileStats.NumberOfModifiedLines;
stat.NumberOfRevisions = fileStats.NumberOfRevisions;
stat.OldPath = fileStats.OldPath;
stat.Path = fileStats.Path;
stat.BuildCommitDateTimeLocal = commitStats.LocalDateTime;
stat.BuildDateTimeLocal = current.BuildDateTimeLocal;
stat.Author = commitStats.Author;
stat.BuildProjectName = current.BuildProjectName;
stat.ProjectName = this.appConfiguration.ImportProjectName;
stat.ObjectId = fileStats.GitObjectId;
return(stat);
}
/// <summary>
/// Provides the list of unknown words and quantities for given
/// language, project and file.
/// </summary>
/// <param name="fs">FileStats</param>
/// <returns></returns>
public Dictionary <string, int> GetUnknownWords(FileStats fs)
{
Dictionary <string, int> words = new Dictionary <string, int>();
SQLiteParameter fileParam = new SQLiteParameter("@file");
fileParam.Value = fs.FilePath;
fileParam.DbType = System.Data.DbType.String;
string sql = "SELECT word, quantity FROM Words " +
"WHERE file=@file ORDER BY quantity DESC";
using (SQLiteCommand cmd = new SQLiteCommand(sql, dbConn))
{
cmd.Parameters.Add(fileParam);
SQLiteDataReader dr = cmd.ExecuteReader();
while (dr.Read())
{
words[dr.GetString(0)] = dr.GetInt32(1);
}
dr.Close();
}
return(words);
}
private void ClearMemory() {
fDelete.Clear();
fInfo.Clear();
fStats = new FileStats();
fDelete = new List<FileInformation>();
fInfo = new List<FileInformation>();
}
internal static extern int Stat(string path, out FileStats output);
internal static extern int FStat(int fileDescriptor, out FileStats output);
private async ValueTask <long> IndexFile(
IDbTransaction transaction,
string path,
FileHandle fileHandle,
FileStats fileStats,
FileChangeEventBuilder eventBuilder)
{
if (path == "/")
{
return(0);
}
var fileType = fileStats.Type;
var isDirectory = fileStats.Type.HasFlag(FileType.Directory);
var oldFile = await _databaseTable.SelectByPathAsync(transaction, path);
if (oldFile == null)
{
long?parentId = await CreateDirectory(transaction, PathLib.Dirname(path), eventBuilder);
var newId = await _databaseTable.InsertAsync(
transaction,
path,
parentId,
fileType.HasFlag(FileType.Directory),
fileHandle.Identifier,
fileStats.Hash.ContentTag);
eventBuilder.Created(fileHandle);
return(newId);
}
if (oldFile.IsDirectory != isDirectory)
{
await Delete(transaction, oldFile, eventBuilder);
var parentId = await CreateDirectory(transaction, PathLib.Dirname(path), eventBuilder);
var newId = await _databaseTable.InsertAsync(
transaction,
path,
parentId,
fileType.HasFlag(FileType.Directory),
fileHandle.Identifier,
fileStats.Hash.ContentTag);
eventBuilder.Created(fileHandle);
return(newId);
}
if (oldFile.IdentifierTag == null)
{
await _databaseTable.UpdateIdentifierTagByIdAsync(
transaction,
oldFile.Id,
fileHandle.Identifier,
fileStats.Hash.ContentTag);
eventBuilder.Created(fileHandle);
return(oldFile.Id);
}
if (oldFile.IdentifierTag != fileHandle.Identifier)
{
await Delete(transaction, oldFile, eventBuilder);
var parentId = await CreateDirectory(transaction, PathLib.Dirname(path), eventBuilder);
var newId = await _databaseTable.InsertAsync(
transaction,
path,
parentId,
fileType.HasFlag(FileType.Directory),
fileHandle.Identifier,
fileStats.Hash.ContentTag);
eventBuilder.Created(fileHandle);
return(newId);
}
if (oldFile.ContentTag != fileStats.Hash.ContentTag)
{
await _databaseTable.UpdateStatsByIdAsync(
transaction,
oldFile.Id,
fileStats.Hash.ContentTag);
await _databaseTable.DeletePropertyOnFileUpdated(transaction, oldFile.Id);
eventBuilder.Changed(fileHandle);
return(oldFile.Id);
}
return(oldFile.Id);
}
public async ValueTask <ReadOnlyMemory <byte>?> GetProperty(string path, FileHandle fileHandle, FileStats fileStats, string key)
{
using var transaction = new SqliteTransaction(_context, ITransaction.TransactionMode.Mutation);
var eventBuilder = new FileChangeEventBuilder();
var fileId = await IndexFile(transaction, path, fileHandle, fileStats, eventBuilder);
var property = await _databaseTable.SelectPropertyAsync(transaction, fileId, key);
await transaction.CommitAsync();
await EmitFileChangeEvents(eventBuilder.BuildFileEvents());
return(property?.Value);
}
private async Task RunStats(AbsolutePath rootPath)
{
const int dividerWidth = 20;
try
{
var allContentFiles = default(FileStats);
var sharedFiles = default(FileStats);
var privateFiles = default(FileStats);
var cacheDeduplicatedFiles = default(FileStats);
var outputDeduplicatedFiles = default(FileStats);
var processFileInfoActionBlock = new ActionBlock <FileInfo>(
fileInfo =>
{
var hardLinkCount = 0;
try
{
hardLinkCount = _fileSystem.GetHardLinkCount(fileInfo.FullPath);
}
catch (Exception ex)
{
_logger.Warning("Failed to open content file at {0}. {1}", fileInfo.FullPath, ex);
}
_logger.Diagnostic(
"Processing file=[{0}] size=[{1}] hardLinkCount=[{2}]", fileInfo.FullPath, fileInfo.Length, hardLinkCount);
allContentFiles.Increase(1, fileInfo.Length);
if (hardLinkCount == 1)
{
privateFiles.Increase(1, fileInfo.Length);
}
else
{
if (hardLinkCount > 1)
{
sharedFiles.Increase(1, fileInfo.Length);
cacheDeduplicatedFiles.Increase(hardLinkCount - 1, fileInfo.Length);
if (hardLinkCount > 2)
{
outputDeduplicatedFiles.Increase(hardLinkCount - 2, fileInfo.Length);
}
}
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
}
);
var processFileInfoTask =
processFileInfoActionBlock.PostAllAndComplete(EnumerateBlobPathsFromDisk(rootPath));
var metadataFileInfos = _fileSystem.EnumerateFiles(rootPath, EnumerateOptions.None).ToList();
var metadataFiles = new FileStats(
metadataFileInfos.Count, metadataFileInfos.Select(fileInfo => fileInfo.Length).Aggregate((s1, s2) => s1 + s2));
await processFileInfoTask.ConfigureAwait(false);
_logger.Always("Statistics for cache at {0}", rootPath);
_logger.Always(new string('=', dividerWidth));
_logger.Always(
"{0:N0} total content files ({1}) in cache",
allContentFiles.FileCount,
allContentFiles.ByteCount.ToSizeExpression());
_logger.Always(
"{0:N0} private content files ({1}) in cache",
privateFiles.FileCount,
privateFiles.ByteCount.ToSizeExpression());
_logger.Always(
"{0:N0} non-content files ({1})",
metadataFiles.FileCount,
metadataFiles.ByteCount.ToSizeExpression());
_logger.Always(string.Empty);
_logger.Always("Shared Content");
_logger.Always(new string('-', dividerWidth));
_logger.Always(
"{0:N0} shared files ({1}) in cache",
sharedFiles.FileCount,
sharedFiles.ByteCount.ToSizeExpression());
_logger.Always(
"Use of hard links avoided duplication on disk of {0} for {1} savings",
cacheDeduplicatedFiles.ByteCount.ToSizeExpression(),
Percent(cacheDeduplicatedFiles.ByteCount, sharedFiles.ByteCount + cacheDeduplicatedFiles.ByteCount));
_logger.Always(
"Use of hard links avoided duplication in output directories of {0} for {1} savings",
outputDeduplicatedFiles.ByteCount.ToSizeExpression(),
Percent(outputDeduplicatedFiles.ByteCount, sharedFiles.ByteCount + outputDeduplicatedFiles.ByteCount));
}
catch (Exception exception)
{
_logger.Error(exception.Message);
_logger.Debug(exception);
}
}
protected FileEntry(ApplicationEntry applicationEntry, FileHandle fileHandle, FileStats stats)
{
ApplicationEntry = applicationEntry;
_rawFileHandle = fileHandle;
Stats = stats;
}
public void InsertFileStats(FileStats FileStats)
{
context.FileStatsSet.Add(FileStats);
}
private static IDictionary <string, FileStats> AnalyzeDirectory(int level, string dir)
{
IDictionary <string, FileStats> dirStats = new Dictionary <string, FileStats>();
foreach (string filepath in Directory.EnumerateFiles(dir))
{
FileInfo file = new FileInfo(filepath);
string extension = file.Extension;
int lines = 0;
using (StreamReader reader = new StreamReader(filepath))
{
while (reader.ReadLine() != null)
{
lines++;
}
}
if (dirStats.ContainsKey(extension))
{
dirStats[extension].Count++;
dirStats[extension].Lines += lines;
}
else
{
FileStats fs = new FileStats(extension);
fs.Count = 1;
fs.Lines = lines;
dirStats.Add(extension, fs);
}
}
WriteStats(true, level, dir, dirStats);
bool hassubDir = false;
foreach (string subdir in Directory.EnumerateDirectories(dir))
{
string localDir = subdir.Substring(subdir.LastIndexOf('\\') + 1);
bool skipdir = false;
foreach (string dirFilter in DIR_FILTERS)
{
if (localDir.StartsWith(dirFilter))
{
skipdir = true;
break;;
}
}
if (skipdir)
{
continue;
}
IDictionary <string, FileStats> subdirStats = AnalyzeDirectory(level + 1, subdir);
AddStats(dirStats, subdirStats);
hassubDir = true;
}
if (hassubDir)
{
WriteStats(false, level, dir, dirStats);
}
return(dirStats);
}
public UnknownFileEntry(ApplicationEntry applicationEntry, FileHandle fileHandle, FileStats stats)
: base(applicationEntry, fileHandle, stats)
{
}
/// <summary>
/// Adds new stats into pending to DB state.
/// </summary>
/// <param name="stats"></param>
public void UpdateStats(FileStats stats)
{
statList.Add(stats);
}
public DirectoryEntry(ApplicationEntry applicationEntry, FileHandle fileHandle, FileStats stats)
: base(applicationEntry, fileHandle, stats)
{
}
private async ValueTask IndexFile(string path, FileHandle fileHandle, FileStats stats)
{
await _hintFileTracker.HintFile(path, fileHandle, stats);
}
public RegularFileEntry(ApplicationEntry applicationEntry, FileHandle fileHandle, FileStats stats)
: base(applicationEntry, fileHandle, stats)
{
}
/// <summary>
/// This is the call back method called by the FaspManager on any event
/// </summary>
/// <param name="asperaEvent">The event for callback.</param>
/// <param name="sessionStats">The Session levels statistics for the Session that triggered this event</param>
/// <param name="fileStats">Statistics of the last file transferred in this session</param>
public void fileSessionEvent(TransferEvent asperaEvent, SessionStats sessionStats, FileStats fileStats)
{
Console.WriteLine("Job Name: " + sessionStats.Cookie);
Console.WriteLine("\tJob State: " + sessionStats.State);
Console.WriteLine("\tTarget Rate: " + sessionStats.TargetRateKbps + "Kbps");
if (sessionStats.ElapsedUSec > 0)
Console.WriteLine("\tAvg Actual Rate: " + (sessionStats.TotalTransferredBytes * 8 * 1000 / sessionStats.ElapsedUSec) + "Kbps");
Console.WriteLine("\tSession Downloaded: " + (sessionStats.TotalTransferredBytes / 1000) + "KB");
if (fileStats != null)
{
Console.WriteLine("\tFileName: " + fileStats.name);
Console.WriteLine("\tFile Downloaded: " + (fileStats.writtenBytes / 1000) + "KB");
}
Console.Write("\n");
// And stop the fasp manager
if (asperaEvent == TransferEvent.SESSION_STOP || asperaEvent == TransferEvent.SESSION_ERROR)
{
currentFile.IsUploaded = true;
currentFile.UploadEndTime = DateTime.Now;
try
{
if (FaspManager.getInstance().getSessionIDList().Count == 1)
{
//FaspManager.destroy();
MoveFileToArchive(currentFile);
}
}
catch (Exception)
{
}
}
if (asperaEvent == TransferEvent.SESSION_ERROR)
{
Console.WriteLine(sessionStats.ErrorDescription);
}
}
