public void CreateBundle(ObjectId[] objectIds, string bundleName, BundleOdbBackend bundleBackend, ISet<ObjectId> disableCompressionIds, Dictionary<string, ObjectId> indexMap, IList<string> dependencies)
{
if (bundleBackend == null)
throw new InvalidOperationException("Can't pack files.");
if (objectIds.Length == 0)
return;
var packUrl = bundleBackend.BundleDirectory + bundleName + BundleOdbBackend.BundleExtension; // we don't want the pack to be compressed in the APK on android
// Create pack
BundleOdbBackend.CreateBundle(packUrl, backendRead1, objectIds, disableCompressionIds, indexMap, dependencies);
}
private static void CleanUnknownBundles(BundleOdbBackend outputBundleBackend, Dictionary <string, ResolvedBundle> resolvedBundles)
{
// Delete previous bundles
outputBundleBackend.DeleteBundles(bundleFile =>
{
// Ensure we have proper extension, otherwise delete
if (Path.GetExtension(bundleFile) != BundleOdbBackend.BundleExtension)
{
return(true);
}
// Only keep bundles that are supposed to be output with same BundleBackend
var bundleName = Path.GetFileNameWithoutExtension(bundleFile);
return(!resolvedBundles.TryGetValue(Path.GetFileNameWithoutExtension(bundleFile), out ResolvedBundle bundle) ||
bundle.BundleBackend != outputBundleBackend);
});
}
/// <summary>
/// Gets the listing from a bundle.
/// </summary>
/// <param name="bundlePath">The bundle path.</param>
/// <returns>System.Collections.Generic.List<SiliconStudio.Paradox.StorageTool.ObjectEntry>.</returns>
private static List <ObjectEntry> GetBundleListing(string bundlePath)
{
if (bundlePath == null)
{
throw new ArgumentNullException("bundlePath");
}
if (Path.GetExtension(bundlePath) != BundleOdbBackend.BundleExtension)
{
throw new StorageAppException("Invalid bundle file [{0}] not having extension [{1}]".ToFormat(bundlePath, BundleOdbBackend.BundleExtension));
}
if (!File.Exists(bundlePath))
{
throw new StorageAppException("Bundle file [{0}] not found".ToFormat(bundlePath));
}
BundleDescription bundle;
using (var stream = File.OpenRead(bundlePath))
{
bundle = BundleOdbBackend.ReadBundleDescription(stream);
}
var objectInfos = bundle.Objects.ToDictionary(x => x.Key, x => x.Value);
var entries = new List <ObjectEntry>();
foreach (var locationIds in bundle.Assets)
{
var entry = new ObjectEntry {
Location = locationIds.Key, Id = locationIds.Value
};
BundleOdbBackend.ObjectInfo objectInfo;
if (objectInfos.TryGetValue(entry.Id, out objectInfo))
{
entry.Size = objectInfo.EndOffset - objectInfo.StartOffset;
entry.SizeNotCompressed = objectInfo.SizeNotCompressed;
}
entries.Add(entry);
}
return(entries);
}
/// <summary>
/// Initializes a new instance of the <see cref="ObjectDatabase" /> class.
/// </summary>
/// <param name="vfsMainUrl">The VFS main URL.</param>
/// <param name="indexName">Name of the index file.</param>
/// <param name="vfsAdditionalUrl">The VFS additional URL. It will be used only if vfsMainUrl is read-only.</param>
public ObjectDatabase(string vfsMainUrl, string indexName, string vfsAdditionalUrl = null, bool loadDefaultBundle = true)
{
if (vfsMainUrl == null) throw new ArgumentNullException(nameof(vfsMainUrl));
// Create the merged asset index map
ContentIndexMap = new ObjectDatabaseContentIndexMap();
// Try to open file backends
bool isReadOnly = Platform.Type != PlatformType.Windows;
var backend = new FileOdbBackend(vfsMainUrl, indexName, isReadOnly);
ContentIndexMap.Merge(backend.ContentIndexMap);
if (backend.IsReadOnly)
{
backendRead1 = backend;
if (vfsAdditionalUrl != null)
{
backendWrite = backendRead2 = new FileOdbBackend(vfsAdditionalUrl, indexName, false);
ContentIndexMap.Merge(backendWrite.ContentIndexMap);
}
}
else
{
backendWrite = backendRead1 = backend;
}
ContentIndexMap.WriteableContentIndexMap = backendWrite.ContentIndexMap;
BundleBackend = new BundleOdbBackend(vfsMainUrl);
// Try to open "default" pack file synchronously
if (loadDefaultBundle)
{
try
{
BundleBackend.LoadBundle("default", ContentIndexMap).GetAwaiter().GetResult();
}
catch (FileNotFoundException)
{
}
}
}
/// <summary>
/// Builds bundles. It will automatically analyze assets and chunks to determine dependencies and what should be embedded in which bundle.
/// Bundle descriptions will be loaded from <see cref="Package.Bundles" /> provided by the <see cref="packageSession" />, and copied to <see cref="outputDirectory" />.
/// </summary>
/// <param name="logger">The builder logger.</param>
/// <param name="packageSession">The project session.</param>
/// <param name="rootPackage">The root package.</param>
/// <param name="indexName">Name of the index file.</param>
/// <param name="outputDirectory">The output directory.</param>
/// <param name="disableCompressionIds">The object id that should be kept uncompressed in the bundle (everything else will be compressed using LZ4).</param>
/// <param name="useIncrementalBundles">Specifies if incremental bundles should be used, or writing a complete new one.</param>
/// <exception cref="System.InvalidOperationException">
/// </exception>
public void Build(Logger logger, PackageSession packageSession, Package rootPackage, string indexName, string outputDirectory, ISet <ObjectId> disableCompressionIds, bool useIncrementalBundles, List <string> bundleFiles)
{
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (packageSession == null)
{
throw new ArgumentNullException("packageSession");
}
if (indexName == null)
{
throw new ArgumentNullException("indexName");
}
if (outputDirectory == null)
{
throw new ArgumentNullException("outputDirectory");
}
if (disableCompressionIds == null)
{
throw new ArgumentNullException("disableCompressionIds");
}
// Load index maps and mount databases
using (var objDatabase = new ObjectDatabase(VirtualFileSystem.ApplicationDatabasePath, indexName, null, false))
{
logger.Info("Generate bundles: Scan assets and their dependencies...");
// Prepare list of bundles gathered from all projects
var bundles = new List <Bundle>();
foreach (var project in packageSession.Packages)
{
bundles.AddRange(project.Bundles);
}
var databaseFileProvider = new DatabaseFileProvider(objDatabase.ContentIndexMap, objDatabase);
// Pass1: Create ResolvedBundle from user Bundle
var resolvedBundles = new Dictionary <string, ResolvedBundle>();
foreach (var bundle in bundles)
{
if (resolvedBundles.ContainsKey(bundle.Name))
{
throw new InvalidOperationException(string.Format("Two bundles with name {0} found", bundle.Name));
}
resolvedBundles.Add(bundle.Name, new ResolvedBundle(bundle));
}
// Pass2: Enumerate all assets which directly or indirectly belong to an bundle
var bundleAssets = new HashSet <string>();
foreach (var bundle in resolvedBundles)
{
// For each project, we apply asset selectors of current bundle
// This will give us a list of "root assets".
foreach (var assetSelector in bundle.Value.Source.Selectors)
{
foreach (var assetLocation in assetSelector.Select(packageSession, objDatabase.ContentIndexMap))
{
bundle.Value.AssetUrls.Add(assetLocation);
}
}
// Compute asset dependencies, and fill bundleAssets with list of all assets contained in bundles (directly or indirectly).
foreach (var assetUrl in bundle.Value.AssetUrls)
{
CollectReferences(databaseFileProvider, bundle.Value.Source, bundleAssets, assetUrl);
}
}
// Pass3: Create a default bundle that contains all assets not contained in any bundle (directly or indirectly)
var defaultBundle = new Bundle {
Name = "default"
};
var resolvedDefaultBundle = new ResolvedBundle(defaultBundle);
bundles.Add(defaultBundle);
resolvedBundles.Add(defaultBundle.Name, resolvedDefaultBundle);
foreach (var asset in objDatabase.ContentIndexMap.GetMergedIdMap())
{
if (!bundleAssets.Contains(asset.Key))
{
resolvedDefaultBundle.AssetUrls.Add(asset.Key);
}
}
// Pass4: Resolve dependencies
foreach (var bundle in resolvedBundles)
{
// Every bundle depends implicitely on default bundle
if (bundle.Key != "default")
{
bundle.Value.Dependencies.Add(resolvedBundles["default"]);
}
// Add other explicit dependencies
foreach (var dependencyName in bundle.Value.Source.Dependencies)
{
ResolvedBundle dependency;
if (!resolvedBundles.TryGetValue(dependencyName, out dependency))
{
throw new InvalidOperationException(string.Format("Could not find dependency {0} when processing bundle {1}", dependencyName, bundle.Value.Name));
}
bundle.Value.Dependencies.Add(dependency);
}
}
logger.Info("Generate bundles: Assign assets to bundles...");
// Pass5: Topological sort (a.k.a. build order)
// If there is a cyclic dependency, an exception will be thrown.
var sortedBundles = TopologicalSort(resolvedBundles.Values, assetBundle => assetBundle.Dependencies);
// Pass6: Find which ObjectId belongs to which bundle
foreach (var bundle in sortedBundles)
{
// Add objects created by dependencies
foreach (var dep in bundle.Dependencies)
{
// ObjectIds
bundle.DependencyObjectIds.UnionWith(dep.DependencyObjectIds);
bundle.DependencyObjectIds.UnionWith(dep.ObjectIds);
// IndexMap
foreach (var asset in dep.DependencyIndexMap.Concat(dep.IndexMap))
{
if (!bundle.DependencyIndexMap.ContainsKey(asset.Key))
{
bundle.DependencyIndexMap.Add(asset.Key, asset.Value);
}
}
}
// Collect assets (object ids and partial index map) from given asset urls
// Those not present in dependencies will be added to this bundle
foreach (var assetUrl in bundle.AssetUrls)
{
CollectBundle(databaseFileProvider, bundle, assetUrl);
}
}
logger.Info("Generate bundles: Compress and save bundles to HDD...");
var vfsToDisposeList = new List <IVirtualFileProvider>();
// Mount VFS for output database (currently disabled because already done in ProjectBuilder.CopyBuildToOutput)
using (var provider = VirtualFileSystem.MountFileSystem("/data_output", outputDirectory))
{
VirtualFileSystem.CreateDirectory("/data_output/db");
// Mount output database and delete previous bundles that shouldn't exist anymore (others should be overwritten)
using (var outputDatabase = new ObjectDatabase("/data_output/db", VirtualFileSystem.ApplicationDatabaseIndexName, loadDefaultBundle: false))
{
try
{
outputDatabase.LoadBundle("default").GetAwaiter().GetResult();
}
catch (Exception)
{
logger.Info("Generate bundles: Tried to load previous 'default' bundle but it was invalid. Deleting it...");
outputDatabase.BundleBackend.DeleteBundles(x => Path.GetFileNameWithoutExtension(x) == "default");
}
var outputBundleBackend = outputDatabase.BundleBackend;
var outputGroupBundleBackends = new Dictionary <string, BundleOdbBackend>();
if (rootPackage.OutputGroupDirectories != null)
{
foreach (var item in rootPackage.OutputGroupDirectories)
{
var path = Path.Combine(rootPackage.RootDirectory, item.Value);
var vfsPath = "/data_group_" + item.Key;
var vfsDatabasePath = vfsPath + "/db";
// Mount VFS for output database (currently disabled because already done in ProjectBuilder.CopyBuildToOutput)
vfsToDisposeList.Add(VirtualFileSystem.MountFileSystem(vfsPath, path));
VirtualFileSystem.CreateDirectory(vfsDatabasePath);
outputGroupBundleBackends.Add(item.Key, new BundleOdbBackend(vfsDatabasePath));
}
}
// Pass7: Assign bundle backends
foreach (var bundle in sortedBundles)
{
BundleOdbBackend bundleBackend;
if (bundle.Source.OutputGroup == null)
{
// No output group, use OutputDirectory
bundleBackend = outputBundleBackend;
}
else if (!outputGroupBundleBackends.TryGetValue(bundle.Source.OutputGroup, out bundleBackend))
{
// Output group not found in OutputGroupDirectories, let's issue a warning and fallback to OutputDirectory
logger.Warning($"Generate bundles: Could not find OutputGroup {bundle.Source.OutputGroup} for bundle {bundle.Name} in ProjectBuildProfile.OutputGroupDirectories");
bundleBackend = outputBundleBackend;
}
bundle.BundleBackend = bundleBackend;
}
CleanUnknownBundles(outputBundleBackend, resolvedBundles);
foreach (var bundleBackend in outputGroupBundleBackends)
{
CleanUnknownBundles(bundleBackend.Value, resolvedBundles);
}
// Pass8: Pack actual data
foreach (var bundle in sortedBundles)
{
// Compute dependencies (by bundle names)
var dependencies = bundle.Dependencies.Select(x => x.Name).Distinct().ToList();
BundleOdbBackend bundleBackend;
if (bundle.Source.OutputGroup == null)
{
// No output group, use OutputDirectory
bundleBackend = outputBundleBackend;
}
else if (!outputGroupBundleBackends.TryGetValue(bundle.Source.OutputGroup, out bundleBackend))
{
// Output group not found in OutputGroupDirectories, let's issue a warning and fallback to OutputDirectory
logger.Warning($"Generate bundles: Could not find OutputGroup {bundle.Source.OutputGroup} for bundle {bundle.Name} in ProjectBuildProfile.OutputGroupDirectories");
bundleBackend = outputBundleBackend;
}
var topBundleUrl = objDatabase.CreateBundle(bundle.ObjectIds.ToArray(), bundle.Name, bundleBackend, disableCompressionIds, bundle.IndexMap, dependencies, useIncrementalBundles);
// Expand list of incremental bundles
BundleOdbBackend.ReadBundleHeader(topBundleUrl, out var bundleUrls);
foreach (var bundleUrl in bundleUrls)
{
bundleFiles.Add(VirtualFileSystem.GetAbsolutePath(bundleUrl));
}
}
// Dispose VFS created for groups
foreach (var vfsToDispose in vfsToDisposeList)
{
try
{
vfsToDispose.Dispose();
}
catch (Exception ex)
{
logger.Error($"Unable to dispose VFS [{vfsToDispose.RootPath}]", ex);
}
}
}
}
}
logger.Info("Generate bundles: Done");
}
