public async Task <List <RepositoryDependency> > ReadAsync(string owner, string name, string branch, DateTime?asOf)
{
var dependencies = new List <RepositoryDependency>();
var files = await repositorySourceManager.ReadFilesAsync(owner, name, branch, asOf).ConfigureAwait(false);
var packageJsonFiles = files.Where(file => file.Name == "package.json");
if (packageJsonFiles != null)
{
foreach (var packageJsonFile in packageJsonFiles)
{
var packageJsonContent = await repositorySourceManager.ReadFileContentAsync(owner, name, branch, packageJsonFile.FullPath, asOf).ConfigureAwait(false);
JObject jObject = null;
try
{
jObject = JObject.Parse(packageJsonContent);
}
catch (Exception ex)
{
var exceptionMessage = $"Error parsing JSON from {owner} - {name} - {packageJsonFile.FullPath}";
throw new ArgumentException(exceptionMessage, ex);
}
var npmProdDependencies = jObject["dependencies"];
if (npmProdDependencies != null)
{
foreach (var token in npmProdDependencies)
{
var property = token as JProperty;
var dependency = new RepositoryDependency();
dependency.Environment = "Production";
dependency.Source = "npm";
dependency.Name = property.Name;
var cleansedVersionMatch = Regex.Match(property.Value.ToString(), @"[\d\.]+");
dependency.Version = cleansedVersionMatch.Value;
dependency.MajorVersion = Regex.Match(dependency.Version, @"\d+").Value;
dependencies.Add(dependency);
}
}
var npmDevDependencies = jObject["devDependencies"];
if (npmDevDependencies != null)
{
foreach (var token in npmDevDependencies)
{
var property = token as JProperty;
var dependency = new RepositoryDependency();
dependency.Environment = "Development";
dependency.Source = "npm";
dependency.Name = property.Name;
var cleansedVersionMatch = Regex.Match(property.Value.ToString(), @"[\d\.]+");
dependency.Version = cleansedVersionMatch.Value;
dependency.MajorVersion = Regex.Match(dependency.Version, @"\d+").Value;
dependencies.Add(dependency);
}
}
}
}
return(dependencies);
}
public async Task <List <RepositoryDependency> > ReadAsync(string owner, string name, string branch, DateTime?asOf)
{
var dependencies = new List <RepositoryDependency>();
var files = await repositorySourceManager.ReadFilesAsync(owner, name, branch, asOf).ConfigureAwait(false);
// Check for .NET framework NuGet packages
var packageConfigFiles = files.Where(file => file.Name == "packages.config");
if (packageConfigFiles != null)
{
foreach (var packageConfigFile in packageConfigFiles)
{
var packageConfigContent = await repositorySourceManager.ReadFileContentAsync(owner, name, branch, packageConfigFile.FullPath, asOf).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(packageConfigContent))
{
string byteOrderMarkUtf8 = Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble());
if (packageConfigContent.StartsWith(byteOrderMarkUtf8))
{
packageConfigContent = packageConfigContent.Remove(0, byteOrderMarkUtf8.Length);
// For some reason removing the UTF preamble sometimes removes the first XML character
if (!packageConfigContent.StartsWith("<"))
{
packageConfigContent = $"<{packageConfigContent}";
}
}
var xdoc = XDocument.Parse(packageConfigContent);
foreach (var descendant in xdoc.Elements().First().Descendants())
{
var dependency = new RepositoryDependency();
dependency.RepoPath = packageConfigFile.FullPath;
dependency.Source = "NuGet";
dependency.Name = descendant.Attribute("id").Value;
var versionString = descendant.Attribute("version").Value;
var preReleaseSemanticVersionMatch = Regex.Match(versionString, @"-.*?\Z");
if (preReleaseSemanticVersionMatch.Success)
{
versionString = versionString.TrimEnd(preReleaseSemanticVersionMatch.Value.ToCharArray());
dependency.PreReleaseSemanticVersion = preReleaseSemanticVersionMatch.Value.TrimStart('-');
}
dependency.Version = versionString;
dependency.MajorVersion = Regex.Match(dependency.Version, @"\d+").Value;
dependencies.Add(dependency);
}
}
}
}
// Check for Nuget packaes defined in project files from .NET Standard/Core projects OR from .NET Standard/Framework
// dual targeted projects
var dotNetProjectFiles = files.Where(file => file.Name.EndsWith(".csproj") || file.Name.EndsWith(".vbproj"));
if (dotNetProjectFiles != null && dotNetProjectFiles.Any())
{
foreach (var dotNetProjectFile in dotNetProjectFiles)
{
var projectFileContent = await repositorySourceManager.ReadFileContentAsync(owner, name, branch, dotNetProjectFile.FullPath, asOf).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(projectFileContent))
{
string byteOrderMarkUtf8 = Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble());
if (projectFileContent.StartsWith(byteOrderMarkUtf8))
{
projectFileContent = projectFileContent.Remove(0, byteOrderMarkUtf8.Length);
// For some reason removing the UTF preamble sometimes removes the first XML character
if (!projectFileContent.StartsWith("<"))
{
projectFileContent = $"<{projectFileContent}";
}
}
var xDoc = XDocument.Parse(projectFileContent);
var packageReferenceElements = xDoc.Descendants().Where(descendant => descendant.Name.LocalName == "PackageReference");
if (packageReferenceElements != null)
{
foreach (var packageReferenceElement in packageReferenceElements)
{
var dependency = new RepositoryDependency();
dependency.RepoPath = dotNetProjectFile.FullPath;
dependency.Source = "NuGet";
dependency.Name = packageReferenceElement.FirstAttribute.Value;
string versionString = null;
// Figure out if this is a refernce from a .NET Standard/Core project file
if (packageReferenceElement.Attributes().Count() == 2 && packageReferenceElement.LastAttribute.Name == "Version")
{
versionString = packageReferenceElement.LastAttribute.Value;
}
// Figure out if this is a refernce from a .NET Framework project file that has been dual targeted with .NET standard
else if (packageReferenceElement.Attributes().Count() == 1 && packageReferenceElement.HasElements && packageReferenceElement.Elements().First().Name.LocalName == "Version")
{
versionString = packageReferenceElement.Elements().First().Value;
}
dependency.Version = versionString;
// Some project dependencies like "Microsoft.AspNetCore.App" do not have version elements
// TODO: Move this parsing logic in to the dependency manager class maybe?
if (dependency.Version != null)
{
var preReleaseSemanticVersionMatch = Regex.Match(versionString, @"-.*?\Z");
if (preReleaseSemanticVersionMatch.Success)
{
versionString = versionString.TrimEnd(preReleaseSemanticVersionMatch.Value.ToCharArray());
dependency.PreReleaseSemanticVersion = preReleaseSemanticVersionMatch.Value.TrimStart('-');
}
dependency.MajorVersion = Regex.Match(dependency.Version, @"\d+").Value;
}
dependencies.Add(dependency);
}
}
}
}
}
return(dependencies);
}
public async Task CreateAsync(RepositoryAnalysis repositoryAnalysis)
{
var parsedRepoUrl = ParseRepositoryUrl();
// maybe change the name of this var to reflect how it's different than the repositoryAnalysis.repsoitoryId?
var repositoryId = $"{parsedRepoUrl.Host}|{parsedRepoUrl.Owner}|{parsedRepoUrl.Name}";
var repository = await repositoryManager.ReadAsync(repositoryId, repositoryAnalysis.AsOf).ConfigureAwait(false);
DateTime?repositoryLastUpdatedOn = null;
// If a last updated time for the repo was provided, use that to hopefully save an API call
if (repositoryAnalysis.RepositoryLastUpdatedOn.HasValue)
{
repositoryLastUpdatedOn = repositoryAnalysis.RepositoryLastUpdatedOn.Value;
}
else
{
var repositorySummary = await repositorySourceManager.ReadRepositorySummaryAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name).ConfigureAwait(false);
repositoryLastUpdatedOn = repositorySummary.UpdatedAt;
}
if (repositoryAnalysis.ForceCompleteRefresh || repository == null || repositoryLastUpdatedOn > repository.CurrentState.RepositoryLastUpdatedOn)
{
// Do repository summary call to get the commit Id of the latest commit and the date that commit was pushed for the snapshot
// populate the snapshot date with the corresponding manager calls (E.G. ScrapeDependenciesAsync)
// Do full repository read to get all the current state stuff (including calls to get derived data like devops integrations)
var sourceRepository = await repositorySourceManager.ReadRepositoryAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name).ConfigureAwait(false);
var repositoryCurrentState = new RepositoryCurrentState();
repositoryCurrentState.Id = repositoryId;
repositoryCurrentState.Name = sourceRepository.Name;
repositoryCurrentState.Owner = parsedRepoUrl.Owner;
repositoryCurrentState.DefaultBranch = sourceRepository.DefaultBranchName;
repositoryCurrentState.HasIssues = sourceRepository.IssueCount > 0;
repositoryCurrentState.HasProjects = sourceRepository.ProjectCount > 0;
repositoryCurrentState.HasPullRequests = sourceRepository.PullRequestCount > 0;
repositoryCurrentState.RepositoryCreatedOn = sourceRepository.CreatedAt;
repositoryCurrentState.RepositoryLastUpdatedOn = sourceRepository.PushedAt;
repositoryCurrentState.Teams = sourceRepository.Teams;
repositoryCurrentState.Topics = sourceRepository.TopicNames?.Select(name => new RepositoryTopic {
Name = name
}).ToList();
repositoryCurrentState.DevOpsIntegrations = await ScrapeDevOpsIntegrations(repositoryCurrentState.Name).ConfigureAwait(false);
// Need to pick a branch for the snapshot stuff
string branchName = null;
if (sourceRepository.BranchNames.Contains("master"))
{
branchName = "master";
}
else if (sourceRepository.BranchNames.Contains("development"))
{
branchName = "development";
}
else if (!string.IsNullOrWhiteSpace(sourceRepository.DefaultBranchName))
{
branchName = sourceRepository.DefaultBranchName;
}
RepositorySnapshot repositorySnapshot = null;
repositorySnapshot = new RepositorySnapshot();
// Have to set the windows in the manager
repositorySnapshot.RepositoryCurrentStateRepositoryId = repositoryCurrentState.Id;
repositorySnapshot.TakenOn = DateTime.Now;
repositorySnapshot.BranchUsed = branchName;
if (branchName != null)
{
repositorySnapshot.Dependencies = await ScrapeDependenciesAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name, branchName, repositoryAnalysis.AsOf).ConfigureAwait(false);
repositorySnapshot.Files = await repositorySourceManager.ReadFilesAsync(parsedRepoUrl.Owner, parsedRepoUrl.Name, branchName, repositoryAnalysis.AsOf).ConfigureAwait(false);
}
else
{
repositorySnapshot.Dependencies = new List <RepositoryDependency>();
repositorySnapshot.Files = new List <RepositoryFile>();
}
repositorySnapshot.TypesAndImplementations = await ScrapeRepositoryTypeAndImplementation(
parsedRepoUrl.Name,
repositorySnapshot.Files,
repositorySnapshot.Dependencies,
repositoryCurrentState.Topics?.Select(topic => topic.Name),
new BacklogInfo { HasIssues = repositoryCurrentState.HasIssues ?? false },
repositoryAnalysis.AsOf).ConfigureAwait(false);
var updatedRepository = new Repository
{
CurrentState = repositoryCurrentState,
Snapshot = repositorySnapshot
};
await repositoryManager.UpsertAsync(updatedRepository, repositoryAnalysis.AsOf).ConfigureAwait(false);
}
(string Owner, string Name, string Host) ParseRepositoryUrl()
{
var repositoryUri = new Uri(repositoryAnalysis.RepositoryId);
var owner = repositoryUri.Segments[1].TrimEnd('/');
var name = repositoryUri.Segments[2].TrimEnd('/');
var host = repositoryUri.Host;
return(owner, name, host);
}
}
public async Task <List <RepositoryDependency> > ReadAsync(string owner, string name, string branch, DateTime?asOf)
{
var dependencies = new List <RepositoryDependency>();
var files = await repositorySourceManager.ReadFilesAsync(owner, name, branch, asOf).ConfigureAwait(false);
var dotNetProjectFiles = files.Where(file => file.Name.EndsWith(".csproj") || file.Name.EndsWith(".vbproj"));
if (dotNetProjectFiles != null && dotNetProjectFiles.Any())
{
foreach (var dotNetProjectFile in dotNetProjectFiles)
{
var projectFileContent = await this.repositorySourceManager.ReadFileContentAsync(owner, name, branch, dotNetProjectFile.FullPath, asOf).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(projectFileContent))
{
string byteOrderMarkUtf8 = Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble());
if (projectFileContent.StartsWith(byteOrderMarkUtf8))
{
projectFileContent = projectFileContent.Remove(0, byteOrderMarkUtf8.Length);
// For some reason removing the UTF preamble sometimes removes the first XML character
if (!projectFileContent.StartsWith("<"))
{
projectFileContent = $"<{projectFileContent}";
}
}
var xDoc = XDocument.Parse(projectFileContent);
// Look for a .NET framework element
var targetFrameworkVersionElement = xDoc.Descendants().FirstOrDefault(descendant => descendant.Name.LocalName == "TargetFrameworkVersion");
if (targetFrameworkVersionElement != null)
{
var dotNetFrameworkVersion = targetFrameworkVersionElement.Value.TrimStart('v');
var dotNetFrameworkDependency = new RepositoryDependency();
dotNetFrameworkDependency.Name = ".NET Framework";
dotNetFrameworkDependency.Version = dotNetFrameworkVersion;
dotNetFrameworkDependency.MajorVersion = Regex.Match(dotNetFrameworkDependency.Version, @"\d+").Value;
dotNetFrameworkDependency.Environment = "Production";
dotNetFrameworkDependency.Source = "Visual Studio Project File";
dotNetFrameworkDependency.RepoPath = dotNetProjectFile.FullPath;
dependencies.Add(dotNetFrameworkDependency);
}
else
{
// Look for a .NET core / standard element
var targetFrameworkElement = xDoc.Descendants().FirstOrDefault(descendant => descendant.Name.LocalName == "TargetFramework");
if (targetFrameworkElement != null)
{
// AFAIK only these two names are used in this element
//netstandard
//netcoreapp
var match = Regex.Match(targetFrameworkElement.Value, @"([A-z]+)([\d|\.]+)");
var appType = match.Groups[1].Value;
var version = match.Groups[2].Value;
var netCoreStandardDependency = new RepositoryDependency();
if (appType.ToLower() == "netstandard")
{
netCoreStandardDependency.Name = ".NET Standard";
}
else if (appType.ToLower() == "netcoreapp")
{
netCoreStandardDependency.Name = ".NET Core";
}
else
{
throw new ArgumentException($"Unrecognized .NET application type of {appType}");
}
netCoreStandardDependency.Version = version;
netCoreStandardDependency.MajorVersion = Regex.Match(netCoreStandardDependency.Version, @"\d+").Value;
netCoreStandardDependency.Source = "Visual Studio Project File";
netCoreStandardDependency.RepoPath = dotNetProjectFile.FullPath;
dependencies.Add(netCoreStandardDependency);
}
}
}
}
}
return(dependencies);
}
public async Task <List <RepositoryDependency> > ReadAsync(string owner, string name, string branch, DateTime?asOf)
{
var dependencies = new List <RepositoryDependency>();
var files = await repositorySourceManager.ReadFilesAsync(owner, name, branch, asOf).ConfigureAwait(false);
var mavenFiles = files.Where(file => file.Name == "pom.xml");
if (mavenFiles != null)
{
foreach (var mavenFile in mavenFiles)
{
var mavenFileContent = await repositorySourceManager.ReadFileContentAsync(owner, name, branch, mavenFile.FullPath, asOf).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(mavenFileContent))
{
string byteOrderMarkUtf8 = Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble());
if (mavenFileContent.StartsWith(byteOrderMarkUtf8))
{
mavenFileContent = mavenFileContent.Remove(0, byteOrderMarkUtf8.Length);
// For some reason removing the UTF preamble sometimes removes the first XML character
if (!mavenFileContent.StartsWith("<"))
{
mavenFileContent = $"<{mavenFileContent}";
}
}
var xDoc = XDocument.Parse(mavenFileContent);
var propertiesElement = xDoc.Descendants().FirstOrDefault(descendant => descendant.Name.LocalName == "properties");
var propertiesNameToValueMap = new Dictionary <string, string>();
if (propertiesElement != null)
{
foreach (var propertyElement in propertiesElement.Elements())
{
propertiesNameToValueMap.Add(propertyElement.Name.LocalName, propertyElement.Value);
}
// Replace any property names in the property values
foreach (var key in propertiesNameToValueMap.Keys.ToList())
{
var value = propertiesNameToValueMap[key];
var match = Regex.Match(value, @"\${(.*)}");
if (match.Success)
{
propertiesNameToValueMap[key] = Regex.Replace(value, @"\${.*}", propertiesNameToValueMap[match.Groups[1].Value]);
}
}
}
var dependencyElements = xDoc.Descendants().Where(descendant => descendant.Name.LocalName == "dependency");
foreach (var dependencyElement in dependencyElements)
{
var repositoryDependency = new RepositoryDependency();
repositoryDependency.Name = dependencyElement.Elements().First(element => element.Name.LocalName == "artifactId").Value;
var match = Regex.Match(repositoryDependency.Name, @"\${(.*)}");
if (match.Success)
{
var propertyName = match.Groups[1].Value;
repositoryDependency.Name = Regex.Replace(repositoryDependency.Name, @"\${.*}", propertiesNameToValueMap[propertyName]);
}
repositoryDependency.Version = dependencyElement.Elements().First(element => element.Name.LocalName == "version").Value;
match = Regex.Match(repositoryDependency.Version, @"\${(.*)}");
if (match.Success)
{
var propertyName = match.Groups[1].Value;
repositoryDependency.Version = Regex.Replace(repositoryDependency.Version, @"\${.*}", propertiesNameToValueMap[propertyName]);
}
repositoryDependency.MajorVersion = Regex.Match(repositoryDependency.Version, @"\A\d+").Value;
repositoryDependency.RepoPath = mavenFile.FullPath;
repositoryDependency.Source = "Maven";
if (!dependencies.Any(
dependency => dependency.Name == repositoryDependency.Name &&
dependency.Version == repositoryDependency.Version &&
dependency.RepoPath == repositoryDependency.RepoPath))
{
dependencies.Add(repositoryDependency);
}
}
}
}
}
return(dependencies);
}
