public void ProcessFiles(UpdateJobContext jobContext)
{
var repoData = _repository.Get(jobContext.RepositoryId);
if (repoData == null) return;
if (string.IsNullOrEmpty(repoData.PageContentTypeName)) repoData.PageContentTypeName = WellKnownConstants.DefaultRepoPageContentType;
var repoSettings = new BitbucketRepositorySettings(repoData, _encryptionService);
var urlMappings = repoData.UrlMappings();
// Ordering so files on the top of the folder hierarchy and index files are first. This way subsequent files will be able to find
// their parent.
foreach (var file in jobContext.Files.OrderBy(f => f.Path.Count(c => c == '/')).ThenBy(f => f.Path.IsIndexFilePath() ? 0 : 1))
{
Process(file, urlMappings, repoData, repoSettings, jobContext);
}
}
public void Repopulate(int repositoryId)
{
var repoData = GetRepositoryDataOrThrow(repositoryId);
var repoSettings = new BitbucketRepositorySettings(repoData, _encryptionService);
var lastChangeset = _apiService.FetchFromRepo<ChangesetsResponse>(repoSettings, "changesets?limit=1").Changesets.FirstOrDefault();
if (lastChangeset == null) return;
Func<string, List<FolderSrcFile>> recursivelyFetchFileList = null;
recursivelyFetchFileList =
(path) =>
{
var responseData = _apiService.FetchFromRepo<FolderSrcResponse>(repoSettings, UriHelper.Combine("src", lastChangeset.Revision.ToString(), path, "/"));
if (responseData.Directories == null) throw new ApplicationException("The path " + path + " was not found in the repo.");
if (responseData.Directories.Count == 0) return responseData.Files;
var files = new List<FolderSrcFile>();
foreach (var directory in responseData.Directories)
{
files.AddRange(recursivelyFetchFileList(UriHelper.Combine(path, directory)));
}
files.AddRange(responseData.Files);
return files;
};
foreach (var mapping in repoData.UrlMappings())
{
var jobFiles = new List<UpdateJobFile>();
if (!mapping.RepoPath.IsMarkdownFilePath())
{
var files = recursivelyFetchFileList(mapping.RepoPath);
foreach (var file in files)
{
jobFiles.Add(new UpdateJobFile(file.Path, UpdateJobfileType.AddedOrModified));
}
}
else
{
jobFiles.Add(new UpdateJobFile(mapping.RepoPath, UpdateJobfileType.AddedOrModified));
}
var jobContext = new UpdateJobContext(
repositoryId,
lastChangeset.Node,
lastChangeset.Revision,
jobFiles,
true);
_jobManager.CreateJob(Industry, jobContext, 99);
}
repoData.LastCheckedNode = lastChangeset.Node;
repoData.LastCheckedRevision = lastChangeset.Revision;
repoData.LastProcessedNode = "";
repoData.LastProcessedRevision = -1;
}
private void ProcessFile(UpdateJobFile file, string localPath, BitbucketRepositoryDataRecord repoData, BitbucketRepositorySettings repoSettings, UpdateJobContext jobContext)
{
if (String.IsNullOrEmpty(Path.GetExtension(localPath))) return;
if (file.Type != UpdateJobfileType.Removed)
{
var sizeProbe = _apiService.FetchFromRepo<FolderSrcResponse>(repoSettings, UriHelper.Combine("src", jobContext.Revision.ToString(), Path.GetDirectoryName(file.Path)));
var size = sizeProbe.Files.Where(f => f.Path == file.Path).Single().Size;
if (size > repoData.MaximalFileSizeKB * 1024) return;
_fileService.SaveFile(localPath, _apiService.FetchFromRepo(repoSettings, UriHelper.Combine("raw", jobContext.Revision.ToString(), file.Path)));
}
else
{
_fileService.DeleteFile(localPath);
}
}
private void ProcessPage(UpdateJobFile file, string localPath, BitbucketRepositoryDataRecord repoData, BitbucketRepositorySettings repoSettings, UpdateJobContext jobContext)
{
localPath = localPath.Replace("Index", "").Replace(".md", "");
if (file.Path.IsIndexFilePath()) localPath = UriHelper.Combine(localPath, "/"); // Trailing slash for index files directly fetched
var repoBasePath = UriHelper.Combine("bitbucket.org", repoData.AccountName, repoData.Slug);
var fullRepoFilePath = UriHelper.Combine(repoBasePath, file.Path);
ContentItem page = null;
if (file.Type != UpdateJobfileType.Removed)
{
var src = _apiService.FetchFromRepo<FileSrcResponse>(repoSettings, UriHelper.Combine("src", jobContext.Revision.ToString(), file.Path));
if (file.Type != UpdateJobfileType.Added) page = FetchPage(repoData.PageContentTypeName, fullRepoFilePath);
var isNew = page == null;
if (isNew) page = _contentManager.New(repoData.PageContentTypeName);
var pagePart = page.As<MarkdownPagePart>();
// We're updating the path for the repo file in the DB if it doesn't match the one that comes from the repo.
// This is needed to overcome the issue caused by the case-sensitive URLs on BitBucket, when the casing of any character changes in the file path.
if (pagePart.RepoPath != fullRepoFilePath)
{
var autoroutePart = page.As<AutoroutePart>();
autoroutePart.CustomPattern = localPath;
autoroutePart.UseCustomPattern = true;
autoroutePart.DisplayAlias = localPath;
page.As<MarkdownPagePart>().RepoPath = fullRepoFilePath;
}
// Searching for the (first) title in the markdown text. Doesn't work if the text is less than or equal to a single line.
var lines = Regex.Split(src.Data, "\r\n|\r|\n").ToList();
int i = 1;
var titleFound = false;
var title = string.Empty;
while (!titleFound && i < lines.Count)
{
// If this line consists of just equals signs, the above line is a H1
if (Regex.IsMatch(lines[i], "^[=]+$"))
{
title = lines[i - 1];
titleFound = true;
lines.RemoveAt(i - 1);
lines.RemoveAt(i);
}
// Or if it starts with a single hashmark
else if (lines[i - 1].StartsWith("#"))
{
title = lines[i - 1].Substring(1).Trim();
titleFound = true;
lines.RemoveAt(i - 1);
}
i++;
}
page.As<TitlePart>().Title = Regex.Replace(title, @"\\([\`*_{}[\]()#+-.!])", match => match.Groups[1].Value);
// Cleaning leading line breaks
while (lines.Count > 0 && string.IsNullOrEmpty(lines[0].Trim())) lines.RemoveAt(0);
page.As<BodyPart>().Text = string.Join(Environment.NewLine, lines);
var parent = FindParent(repoData.PageContentTypeName, repoBasePath, fullRepoFilePath);
if (parent != null) page.As<CommonPart>().Container = parent;
// This is needed after the title is set, because slug generation needs it
if (isNew) _contentManager.Create(page);
// This is needed so published handlers can run
_contentManager.Unpublish(page);
_contentManager.Publish(page);
}
else
{
page = FetchPage(repoData.PageContentTypeName, fullRepoFilePath);
if (page == null) return;
_contentManager.Remove(page);
}
}
private void Process(UpdateJobFile file, IEnumerable<UrlMapping> urlMappings, BitbucketRepositoryDataRecord repoData, BitbucketRepositorySettings repoSettings, UpdateJobContext jobContext)
{
var mapping = urlMappings.Where(urlMapping => file.Path.StartsWith(urlMapping.RepoPath)).FirstOrDefault();
if (mapping == null) return;
var localPath = file.Path;
if (!String.IsNullOrEmpty(mapping.RepoPath)) localPath = localPath.Replace(mapping.RepoPath, mapping.LocalPath.Trim('/'));
else localPath = UriHelper.Combine(mapping.LocalPath, localPath);
if (file.Path.IsMarkdownFilePath()) ProcessPage(file, localPath, repoData, repoSettings, jobContext);
else if (repoData.MirrorFiles) ProcessFile(file, localPath, repoData, repoSettings, jobContext);
}