public bool SaveBackup(string fileName, RegionData regiondata) { try { bool oldFileExists = File.Exists(fileName); //Do new style saving here! GZipStream m_saveStream = new GZipStream(new FileStream(fileName + ".tmp", FileMode.Create), CompressionMode.Compress); TarArchiveWriter writer = new TarArchiveWriter(m_saveStream); GZipStream m_loadStream = new GZipStream(new FileStream(fileName, FileMode.Open), CompressionMode.Decompress); TarArchiveReader reader = new TarArchiveReader(m_loadStream); writer.WriteDir("parcels"); foreach (LandData parcel in regiondata.Parcels) { OSDMap parcelMap = parcel.ToOSD(); var binary = OSDParser.SerializeLLSDBinary(parcelMap); writer.WriteFile("parcels/" + parcel.GlobalID.ToString(), binary); binary = null; parcelMap = null; } writer.WriteDir("newstyleterrain"); writer.WriteDir("newstylerevertterrain"); writer.WriteDir("newstylewater"); writer.WriteDir("newstylerevertwater"); writer.WriteDir("regioninfo"); byte[] regionData = OSDParser.SerializeLLSDBinary(regiondata.RegionInfo.PackRegionInfoData()); writer.WriteFile("regioninfo/regioninfo", regionData); try { writer.WriteFile("newstyleterrain/" + regiondata.RegionInfo.RegionID.ToString() + ".terrain", regiondata.Terrain); writer.WriteFile( "newstylerevertterrain/" + regiondata.RegionInfo.RegionID.ToString() + ".terrain", regiondata.RevertTerrain); if (regiondata.Water != null) { writer.WriteFile("newstylewater/" + regiondata.RegionInfo.RegionID.ToString() + ".terrain", regiondata.Water); writer.WriteFile( "newstylerevertwater/" + regiondata.RegionInfo.RegionID.ToString() + ".terrain", regiondata.RevertWater); } } catch (Exception ex) { MainConsole.Instance.WarnFormat("[Backup]: Exception caught: {0}", ex); } IDictionary<UUID, AssetType> assets = new Dictionary<UUID, AssetType>(); List<UUID> entitiesToSave = new List<UUID>(); foreach (ISceneEntity entity in regiondata.Groups) { try { if (entity.IsAttachment || ((entity.RootChild.Flags & PrimFlags.Temporary) == PrimFlags.Temporary) || ((entity.RootChild.Flags & PrimFlags.TemporaryOnRez) == PrimFlags.TemporaryOnRez)) continue; if (entity.HasGroupChanged || !oldFileExists) { entity.HasGroupChanged = false; //Write all entities writer.WriteFile("entities/" + entity.UUID.ToString(), entity.ToBinaryXml2()); } else entitiesToSave.Add(entity.UUID); } catch (Exception ex) { MainConsole.Instance.WarnFormat("[Backup]: Exception caught: {0}", ex); entitiesToSave.Add(entity.UUID); } } if (oldFileExists) { byte[] data; string filePath; TarArchiveReader.TarEntryType entryType; //Load the archive data that we need try { while ((data = reader.ReadEntry(out filePath, out entryType)) != null) { if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; if (filePath.StartsWith("entities/")) { UUID entityID = UUID.Parse(filePath.Remove(0, 9)); if (entitiesToSave.Contains(entityID)) { writer.WriteFile(filePath, data); entitiesToSave.Remove(entityID); } } data = null; } } catch (Exception ex) { MainConsole.Instance.WarnFormat("[Backup]: Exception caught: {0}", ex); } if (entitiesToSave.Count > 0) { MainConsole.Instance.Fatal(entitiesToSave.Count + " PRIMS WERE NOT GOING TO BE SAVED! FORCE SAVING NOW! "); foreach (ISceneEntity entity in regiondata.Groups) { if (entitiesToSave.Contains(entity.UUID)) { if (entity.IsAttachment || ((entity.RootChild.Flags & PrimFlags.Temporary) == PrimFlags.Temporary) || ((entity.RootChild.Flags & PrimFlags.TemporaryOnRez) == PrimFlags.TemporaryOnRez)) continue; //Write all entities byte[] xml = entity.ToBinaryXml2(); writer.WriteFile("entities/" + entity.UUID.ToString(), xml); xml = null; } } } } reader.Close(); writer.Close(); m_loadStream.Close(); m_saveStream.Close(); GC.Collect(); } catch (Exception ex) { MainConsole.Instance.Warn("[ProtobufRegionLoader]: Failed to save backup: " + ex.ToString()); return false; } return true; }
public RegionData LoadBackup(string file) { if (!File.Exists(file)) return null; var stream = ArchiveHelpers.GetStream(file); if (stream == null) return null; GZipStream m_loadStream = new GZipStream(stream, CompressionMode.Decompress); TarArchiveReader reader = new TarArchiveReader(m_loadStream); List<uint> foundLocalIDs = new List<uint>(); RegionData regiondata = new RegionData(); regiondata.Init(); byte[] data; string filePath; TarArchiveReader.TarEntryType entryType; System.Collections.Concurrent.ConcurrentQueue<byte[]> groups = new System.Collections.Concurrent.ConcurrentQueue<byte[]>(); //Load the archive data that we need while ((data = reader.ReadEntry(out filePath, out entryType)) != null) { if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; if (filePath.StartsWith("parcels/")) { //Only use if we are not merging LandData parcel = new LandData(); OSD parcelData = OSDParser.DeserializeLLSDBinary(data); parcel.FromOSD((OSDMap) parcelData); if (parcel.OwnerID != UUID.Parse("05948863-b678-433e-87a4-e44d17678d1d")) //The default owner of the 'default' region regiondata.Parcels.Add(parcel); } else if (filePath.StartsWith("newstyleterrain/")) { regiondata.Terrain = data; } else if (filePath.StartsWith("newstylerevertterrain/")) { regiondata.RevertTerrain = data; } else if (filePath.StartsWith("newstylewater/")) { regiondata.Water = data; } else if (filePath.StartsWith("newstylerevertwater/")) { regiondata.RevertWater = data; } else if (filePath.StartsWith("entities/")) { groups.Enqueue(data); } else if (filePath.StartsWith("regioninfo/")) { RegionInfo info = new RegionInfo(); info.FromOSD((OSDMap) OSDParser.DeserializeLLSDBinary(data)); regiondata.RegionInfo = info; } data = null; } m_loadStream.Close(); m_loadStream = null; int threadCount = groups.Count > 16 ? 16 : groups.Count; System.Threading.Thread[] threads = new System.Threading.Thread[threadCount]; for (int i = 0; i < threadCount; i++) { threads[i] = new System.Threading.Thread(() => { byte[] groupData; while (groups.TryDequeue(out groupData)) { MemoryStream ms = new MemoryStream(groupData); ISceneEntity sceneObject = SceneEntitySerializer.SceneObjectSerializer .FromXml2Format(ref ms, null); ms.Close(); ms = null; data = null; if (sceneObject != null) { foreach ( ISceneChildEntity part in sceneObject.ChildrenEntities()) { lock (foundLocalIDs) { if ( !foundLocalIDs.Contains( part.LocalId)) foundLocalIDs.Add(part.LocalId); else part.LocalId = 0; //Reset it! Only use it once! } } regiondata.Groups.Add( sceneObject as SceneObjectGroup); } } }); threads[i].Start(); } for (int i = 0; i < threadCount; i++) threads[i].Join(); foundLocalIDs.Clear(); return regiondata; }
/// <summary> /// Save a backup of the sim /// </summary> /// <param name = "appendedFilePath">The file path where the backup will be saved</param> protected virtual void SaveBackup(string appendedFilePath, bool saveAssets) { if (appendedFilePath == "/") appendedFilePath = ""; if (m_scene.RegionInfo.HasBeenDeleted) return; IBackupModule backupModule = m_scene.RequestModuleInterface<IBackupModule>(); if (backupModule != null && backupModule.LoadingPrims) //Something is changing lots of prims { MainConsole.Instance.Info("[Backup]: Not saving backup because the backup module is loading prims"); return; } //Save any script state saves that might be around IScriptModule[] engines = m_scene.RequestModuleInterfaces<IScriptModule>(); try { if (engines != null) { #if (!ISWIN) foreach (IScriptModule engine in engines) { if (engine != null) { engine.SaveStateSaves(); } } #else foreach (IScriptModule engine in engines.Where(engine => engine != null)) { engine.SaveStateSaves(); } #endif } } catch (Exception ex) { MainConsole.Instance.WarnFormat("[Backup]: Exception caught: {0}", ex); } MainConsole.Instance.Info("[FileBasedSimulationData]: Saving backup for region " + m_scene.RegionInfo.RegionName); string fileName = appendedFilePath + m_scene.RegionInfo.RegionName + m_saveAppendedFileName + ".abackup"; if (File.Exists(fileName)) { //Do new style saving here! GZipStream m_saveStream = new GZipStream(new FileStream(fileName + ".tmp", FileMode.Create), CompressionMode.Compress); TarArchiveWriter writer = new TarArchiveWriter(m_saveStream); GZipStream m_loadStream = new GZipStream(new FileStream(fileName, FileMode.Open), CompressionMode.Decompress); TarArchiveReader reader = new TarArchiveReader(m_loadStream); writer.WriteDir("parcels"); IParcelManagementModule module = m_scene.RequestModuleInterface<IParcelManagementModule>(); if (module != null) { List<ILandObject> landObject = module.AllParcels(); foreach (ILandObject parcel in landObject) { OSDMap parcelMap = parcel.LandData.ToOSD(); var binary = OSDParser.SerializeLLSDBinary(parcelMap); writer.WriteFile("parcels/" + parcel.LandData.GlobalID.ToString(), binary); binary = null; parcelMap = null; } } writer.WriteDir("newstyleterrain"); writer.WriteDir("newstylerevertterrain"); writer.WriteDir("newstylewater"); writer.WriteDir("newstylerevertwater"); ITerrainModule tModule = m_scene.RequestModuleInterface<ITerrainModule>(); if (tModule != null) { try { byte[] sdata = WriteTerrainToStream(tModule.TerrainMap); writer.WriteFile("newstyleterrain/" + m_scene.RegionInfo.RegionID.ToString() + ".terrain", sdata); sdata = null; sdata = WriteTerrainToStream(tModule.TerrainRevertMap); writer.WriteFile( "newstylerevertterrain/" + m_scene.RegionInfo.RegionID.ToString() + ".terrain", sdata); sdata = null; if (tModule.TerrainWaterMap != null) { sdata = WriteTerrainToStream(tModule.TerrainWaterMap); writer.WriteFile("newstylewater/" + m_scene.RegionInfo.RegionID.ToString() + ".terrain", sdata); sdata = null; sdata = WriteTerrainToStream(tModule.TerrainWaterRevertMap); writer.WriteFile( "newstylerevertwater/" + m_scene.RegionInfo.RegionID.ToString() + ".terrain", sdata); sdata = null; } } catch (Exception ex) { MainConsole.Instance.WarnFormat("[Backup]: Exception caught: {0}", ex); } } IDictionary<UUID, AssetType> assets = new Dictionary<UUID, AssetType>(); UuidGatherer assetGatherer = new UuidGatherer(m_scene.AssetService); ISceneEntity[] saveentities = m_scene.Entities.GetEntities(); List<UUID> entitiesToSave = new List<UUID>(); foreach (ISceneEntity entity in saveentities) { try { if (entity.IsAttachment || ((entity.RootChild.Flags & PrimFlags.Temporary) == PrimFlags.Temporary) || ((entity.RootChild.Flags & PrimFlags.TemporaryOnRez) == PrimFlags.TemporaryOnRez)) continue; if (entity.HasGroupChanged) { entity.HasGroupChanged = false; //Write all entities byte[] xml = ((ISceneObject) entity).ToBinaryXml2(); writer.WriteFile("entities/" + entity.UUID.ToString(), xml); xml = null; } else entitiesToSave.Add(entity.UUID); if (saveAssets) assetGatherer.GatherAssetUuids(entity, assets, m_scene); } catch (Exception ex) { MainConsole.Instance.WarnFormat("[Backup]: Exception caught: {0}", ex); entitiesToSave.Add(entity.UUID); } } byte[] data; string filePath; TarArchiveReader.TarEntryType entryType; //Load the archive data that we need try { while ((data = reader.ReadEntry(out filePath, out entryType)) != null) { if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; if (filePath.StartsWith("entities/")) { UUID entityID = UUID.Parse(filePath.Remove(0, 9)); if (entitiesToSave.Contains(entityID)) { writer.WriteFile(filePath, data); entitiesToSave.Remove(entityID); } } data = null; } } catch (Exception ex) { MainConsole.Instance.WarnFormat("[Backup]: Exception caught: {0}", ex); } if (entitiesToSave.Count > 0) { MainConsole.Instance.Fatal(entitiesToSave.Count + " PRIMS WERE NOT GOING TO BE SAVED! FORCE SAVING NOW! "); foreach (ISceneEntity entity in saveentities) { if (entitiesToSave.Contains(entity.UUID)) { if (entity.IsAttachment || ((entity.RootChild.Flags & PrimFlags.Temporary) == PrimFlags.Temporary) || ((entity.RootChild.Flags & PrimFlags.TemporaryOnRez) == PrimFlags.TemporaryOnRez)) continue; //Write all entities byte[] xml = ((ISceneObject) entity).ToBinaryXml2(); writer.WriteFile("entities/" + entity.UUID.ToString(), xml); xml = null; } } } if (saveAssets) { foreach (UUID assetID in new List<UUID>(assets.Keys)) { try { WriteAsset(assetID.ToString(), m_scene.AssetService.Get(assetID.ToString()), writer); } catch (Exception ex) { MainConsole.Instance.WarnFormat("[Backup]: Exception caught: {0}", ex); } } } reader.Close(); writer.Close(); m_loadStream.Close(); m_saveStream.Close(); GC.Collect(); if (m_keepOldSave && !m_oldSaveHasBeenSaved) { //Havn't moved it yet, so make sure the directory exists, then move it m_oldSaveHasBeenSaved = true; if (!Directory.Exists(m_oldSaveDirectory)) Directory.CreateDirectory(m_oldSaveDirectory); File.Copy(fileName + ".tmp", Path.Combine(m_oldSaveDirectory, m_scene.RegionInfo.RegionName + SerializeDateTime() + m_saveAppendedFileName + ".abackup")); } //Just remove the file File.Delete(fileName); } else { //Add the .temp since we might need to make a backup and so that if something goes wrong, we don't corrupt the main backup GZipStream m_saveStream = new GZipStream(new FileStream(fileName + ".tmp", FileMode.Create), CompressionMode.Compress); TarArchiveWriter writer = new TarArchiveWriter(m_saveStream); IAuroraBackupArchiver archiver = m_scene.RequestModuleInterface<IAuroraBackupArchiver>(); //Turn off prompting so that we don't ask the user questions every time we need to save the backup archiver.AllowPrompting = false; archiver.SaveRegionBackup(writer, m_scene); archiver.AllowPrompting = true; m_saveStream.Close(); writer.Close(); GC.Collect(); } File.Move(fileName + ".tmp", fileName); ISceneEntity[] entities = m_scene.Entities.GetEntities(); try { #if (!ISWIN) foreach (ISceneEntity entity in entities) { if (entity.HasGroupChanged) { entity.HasGroupChanged = false; } } #else foreach (ISceneEntity entity in entities.Where(entity => entity.HasGroupChanged)) { entity.HasGroupChanged = false; } #endif } catch (Exception ex) { MainConsole.Instance.WarnFormat("[Backup]: Exception caught: {0}", ex); } //Now make it the full file again MapTileNeedsGenerated = true; MainConsole.Instance.Info("[FileBasedSimulationData]: Saved Backup for region " + m_scene.RegionInfo.RegionName); }
protected virtual void ReadBackup(IScene scene) { MainConsole.Instance.Debug("[FileBasedSimulationData]: Reading file for " + scene.RegionInfo.RegionName); List<uint> foundLocalIDs = new List<uint>(); var stream = ArchiveHelpers.GetStream((m_loadDirectory == "" || m_loadDirectory == "/") ? m_fileName : Path.Combine(m_loadDirectory, m_fileName)); if(stream == null) return; GZipStream m_loadStream = new GZipStream(stream, CompressionMode.Decompress); TarArchiveReader reader = new TarArchiveReader(m_loadStream); byte[] data; string filePath; TarArchiveReader.TarEntryType entryType; System.Collections.Concurrent.ConcurrentQueue<byte[]> groups = new System.Collections.Concurrent.ConcurrentQueue<byte[]>(); //Load the archive data that we need while ((data = reader.ReadEntry(out filePath, out entryType)) != null) { if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; if (filePath.StartsWith("parcels/")) { //Only use if we are not merging LandData parcel = new LandData(); OSD parcelData = OSDParser.DeserializeLLSDBinary(data); parcel.FromOSD((OSDMap) parcelData); m_parcels.Add(parcel); } else if (filePath.StartsWith("terrain/")) { m_oldstyleterrain = data; } else if (filePath.StartsWith("revertterrain/")) { m_oldstylerevertTerrain = data; } else if (filePath.StartsWith("newstyleterrain/")) { m_terrain = data; } else if (filePath.StartsWith("newstylerevertterrain/")) { m_revertTerrain = data; } else if (filePath.StartsWith("newstylewater/")) { m_water = data; } else if (filePath.StartsWith("newstylerevertwater/")) { m_revertWater = data; } else if (filePath.StartsWith("entities/")) { groups.Enqueue(data); } data = null; } m_loadStream.Close(); m_loadStream = null; int threadCount = groups.Count > 16 ? 16 : groups.Count; System.Threading.Thread[] threads = new System.Threading.Thread[threadCount]; for (int i = 0; i < threadCount; i++) { threads[i] = new System.Threading.Thread(() => { byte[] groupData; while(groups.TryDequeue(out groupData)) { MemoryStream ms = new MemoryStream(groupData); SceneObjectGroup sceneObject = SceneObjectSerializer.FromXml2Format(ref ms, scene); ms.Close(); ms = null; data = null; if (sceneObject != null) { foreach (ISceneChildEntity part in sceneObject.ChildrenEntities()) { lock (foundLocalIDs) { if (!foundLocalIDs.Contains(part.LocalId)) foundLocalIDs.Add(part.LocalId); else part.LocalId = 0; //Reset it! Only use it once! } } m_groups.Add(sceneObject); } } }); threads[i].Start(); } for (int i = 0; i < threadCount; i++) threads[i].Join(); foundLocalIDs.Clear(); GC.Collect(); }
/// <summary> /// Execute the request /// </summary> /// <returns> /// A list of the inventory nodes loaded. If folders were loaded then only the root folders are /// returned /// </returns> public HashSet<InventoryNodeBase> Execute(bool loadAll) { try { string filePath = "ERROR"; int successfulAssetRestores = 0; int failedAssetRestores = 0; int successfulItemRestores = 0; HashSet<InventoryNodeBase> loadedNodes = loadAll ? new HashSet<InventoryNodeBase>() : null; List<InventoryFolderBase> folderCandidates = InventoryArchiveUtils.FindFolderByPath( m_registry.RequestModuleInterface<IInventoryService>(), m_userInfo.PrincipalID, m_invPath); if (folderCandidates.Count == 0) { // Possibly provide an option later on to automatically create this folder if it does not exist MainConsole.Instance.ErrorFormat("[INVENTORY ARCHIVER]: Inventory path {0} does not exist", m_invPath); return loadedNodes; } InventoryFolderBase rootDestinationFolder = folderCandidates[0]; archive = new TarArchiveReader(m_loadStream); // In order to load identically named folders, we need to keep track of the folders that we have already // resolved Dictionary<string, InventoryFolderBase> resolvedFolders = new Dictionary<string, InventoryFolderBase>(); byte[] data; TarArchiveReader.TarEntryType entryType; while ((data = archive.ReadEntry(out filePath, out entryType)) != null) { if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH)) { if (LoadAsset(filePath, data)) successfulAssetRestores++; else failedAssetRestores++; if ((successfulAssetRestores)%50 == 0) MainConsole.Instance.InfoFormat( "[INVENTORY ARCHIVER]: Loaded {0} assets...", successfulAssetRestores); } else if (filePath.StartsWith(ArchiveConstants.INVENTORY_PATH)) { filePath = filePath.Substring(ArchiveConstants.INVENTORY_PATH.Length); // Trim off the file portion if we aren't already dealing with a directory path if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType) filePath = filePath.Remove(filePath.LastIndexOf("/") + 1); InventoryFolderBase foundFolder = ReplicateArchivePathToUserInventory( filePath, rootDestinationFolder, ref resolvedFolders, ref loadedNodes); if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType) { InventoryItemBase item = LoadItem(data, foundFolder); if (item != null) { successfulItemRestores++; if ((successfulItemRestores)%50 == 0) MainConsole.Instance.InfoFormat( "[INVENTORY ARCHIVER]: Loaded {0} items...", successfulItemRestores); // If we aren't loading the folder containing the item then well need to update the // viewer separately for that item. if (loadAll && !loadedNodes.Contains(foundFolder)) loadedNodes.Add(item); } item = null; } } data = null; } MainConsole.Instance.InfoFormat( "[INVENTORY ARCHIVER]: Successfully loaded {0} assets with {1} failures", successfulAssetRestores, failedAssetRestores); MainConsole.Instance.InfoFormat("[INVENTORY ARCHIVER]: Successfully loaded {0} items", successfulItemRestores); return loadedNodes; } finally { m_loadStream.Close(); } }
private void DearchiveRegion0DotStar() { int successfulAssetRestores = 0; int failedAssetRestores = 0; string filePath = "NONE"; DateTime start = DateTime.Now; TarArchiveReader archive = new TarArchiveReader(m_loadStream); byte[] data; TarArchiveReader.TarEntryType entryType; if (!m_skipAssets) m_threadpool = new Aurora.Framework.AuroraThreadPool(new Aurora.Framework.AuroraThreadPoolStartInfo() { Threads = 1, priority = System.Threading.ThreadPriority.BelowNormal }); IBackupModule backup = m_scene.RequestModuleInterface<IBackupModule>(); if (!m_merge) { DateTime before = DateTime.Now; MainConsole.Instance.Info("[ARCHIVER]: Clearing all existing scene objects"); if (backup != null) backup.DeleteAllSceneObjects(); MainConsole.Instance.Info("[ARCHIVER]: Cleared all existing scene objects in " + (DateTime.Now - before).Minutes + ":" + (DateTime.Now - before).Seconds); } IScriptModule[] modules = m_scene.RequestModuleInterfaces<IScriptModule>(); //Disable the script engine so that it doesn't load in the background and kill OAR loading foreach (IScriptModule module in modules) { module.Disabled = true; } //Disable backup for now as well if (backup != null) backup.LoadingPrims = true; IRegionSerialiserModule serialiser = m_scene.RequestModuleInterface<IRegionSerialiserModule>(); int sceneObjectsLoadedCount = 0; //We save the groups so that we can back them up later List<SceneObjectGroup> groupsToBackup = new List<SceneObjectGroup>(); List<LandData> landData = new List<LandData>(); IUserManagement UserManager = m_scene.RequestModuleInterface<IUserManagement>(); try { while ((data = archive.ReadEntry(out filePath, out entryType)) != null) { //MainConsole.Instance.DebugFormat( // "[ARCHIVER]: Successfully read {0} ({1} bytes)", filePath, data.Length); if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; if (filePath.StartsWith(ArchiveConstants.OBJECTS_PATH)) { /* MainConsole.Instance.DebugFormat("[ARCHIVER]: Loading xml with raw size {0}", serialisedSceneObject.Length); // Really large xml files (multi megabyte) appear to cause // memory problems // when loading the xml. But don't enable this check yet if (serialisedSceneObject.Length > 5000000) { MainConsole.Instance.Error("[ARCHIVER]: Ignoring xml since size > 5000000);"); continue; } */ SceneObjectGroup sceneObject = (SceneObjectGroup)serialiser.DeserializeGroupFromXml2(data, m_scene); if (sceneObject == null) { //! big error! MainConsole.Instance.Error("Error reading SOP XML (Please mantis this!): " + m_asciiEncoding.GetString(data)); continue; } foreach (SceneObjectPart part in sceneObject.ChildrenList) { if (part.CreatorData == null || part.CreatorData == string.Empty) part.CreatorID = ResolveUserUuid(part.CreatorID, part.CreatorID, part.CreatorData, part.AbsolutePosition, landData); if (UserManager != null) UserManager.AddUser(part.CreatorID, part.CreatorData); part.OwnerID = ResolveUserUuid(part.OwnerID, part.CreatorID, part.CreatorData, part.AbsolutePosition, landData); part.LastOwnerID = ResolveUserUuid(part.LastOwnerID, part.CreatorID, part.CreatorData, part.AbsolutePosition, landData); // And zap any troublesome sit target information part.SitTargetOrientation = new Quaternion(0, 0, 0, 1); part.SitTargetPosition = new Vector3(0, 0, 0); // Fix ownership/creator of inventory items // Not doing so results in inventory items // being no copy/no mod for everyone lock (part.TaskInventory) { TaskInventoryDictionary inv = part.TaskInventory; foreach (KeyValuePair<UUID, TaskInventoryItem> kvp in inv) { kvp.Value.OwnerID = ResolveUserUuid(kvp.Value.OwnerID, kvp.Value.CreatorID, kvp.Value.CreatorData, part.AbsolutePosition, landData); if (kvp.Value.CreatorData == null || kvp.Value.CreatorData == string.Empty) kvp.Value.CreatorID = ResolveUserUuid(kvp.Value.CreatorID, kvp.Value.CreatorID, kvp.Value.CreatorData, part.AbsolutePosition, landData); if (UserManager != null) UserManager.AddUser(kvp.Value.CreatorID, kvp.Value.CreatorData); } } } //Add the offsets of the region Vector3 newPos = new Vector3(sceneObject.AbsolutePosition.X + m_offsetX, sceneObject.AbsolutePosition.Y + m_offsetY, sceneObject.AbsolutePosition.Z + m_offsetZ); if (m_flipX) newPos.X = m_scene.RegionInfo.RegionSizeX - newPos.X; if (m_flipY) newPos.Y = m_scene.RegionInfo.RegionSizeY - newPos.Y; sceneObject.SetAbsolutePosition(false, newPos); if (m_scene.SceneGraph.AddPrimToScene(sceneObject)) { groupsToBackup.Add(sceneObject); sceneObject.ScheduleGroupUpdate(PrimUpdateFlags.ForcedFullUpdate); sceneObjectsLoadedCount++; sceneObject.CreateScriptInstances(0, false, StateSource.RegionStart, UUID.Zero); } sceneObjectsLoadedCount++; if (sceneObjectsLoadedCount % 250 == 0) MainConsole.Instance.Info("[ARCHIVER]: Loaded " + sceneObjectsLoadedCount + " objects..."); } else if (!m_skipAssets && filePath.StartsWith(ArchiveConstants.ASSETS_PATH)) { if (LoadAsset(filePath, data)) successfulAssetRestores++; else failedAssetRestores++; if ((successfulAssetRestores + failedAssetRestores) % 250 == 0) MainConsole.Instance.Info("[ARCHIVER]: Loaded " + successfulAssetRestores + " assets and failed to load " + failedAssetRestores + " assets..."); } else if (filePath.StartsWith(ArchiveConstants.TERRAINS_PATH)) { LoadTerrain(filePath, data); } else if (!m_merge && filePath.StartsWith(ArchiveConstants.SETTINGS_PATH)) { LoadRegionSettings(filePath, data); } else if (filePath.StartsWith(ArchiveConstants.LANDDATA_PATH)) { LandData parcel = LandDataSerializer.Deserialize(m_utf8Encoding.GetString(data)); parcel.OwnerID = ResolveUserUuid(parcel.OwnerID, UUID.Zero, "", Vector3.Zero, null); landData.Add(parcel); } else if (filePath == ArchiveConstants.CONTROL_FILE_PATH) { LoadControlFile(filePath, data); } } //MainConsole.Instance.Debug("[ARCHIVER]: Reached end of archive"); } catch (Exception e) { MainConsole.Instance.ErrorFormat( "[ARCHIVER]: Aborting load with error in archive file {0}. {1}", filePath, e); m_errorMessage += e.ToString(); m_scene.EventManager.TriggerOarFileLoaded(UUID.Zero.Guid, m_errorMessage); return; } finally { archive.Close(); m_loadStream.Close(); m_loadStream.Dispose(); //Reeanble now that we are done foreach (IScriptModule module in modules) { module.Disabled = false; } //Reset backup too if (backup != null) backup.LoadingPrims = false; } //Now back up the prims foreach (SceneObjectGroup grp in groupsToBackup) { //Backup! grp.HasGroupChanged = true; } if (!m_skipAssets && m_useAsync && !AssetSaverIsRunning) m_threadpool.QueueEvent(SaveAssets, 0); if (!m_skipAssets) { MainConsole.Instance.InfoFormat("[ARCHIVER]: Restored {0} assets", successfulAssetRestores); if (failedAssetRestores > 0) { MainConsole.Instance.ErrorFormat("[ARCHIVER]: Failed to load {0} assets", failedAssetRestores); m_errorMessage += String.Format("Failed to load {0} assets", failedAssetRestores); } } // Try to retain the original creator/owner/lastowner if their uuid is present on this grid // otherwise, use the master avatar uuid instead // Reload serialized parcels MainConsole.Instance.InfoFormat("[ARCHIVER]: Loading {0} parcels. Please wait.", landData.Count); IParcelManagementModule parcelManagementModule = m_scene.RequestModuleInterface<IParcelManagementModule>(); if (!m_merge && parcelManagementModule != null) parcelManagementModule.ClearAllParcels(); if (landData.Count > 0) { m_scene.EventManager.TriggerIncomingLandDataFromStorage(landData, new Vector2(m_offsetX, m_offsetY)); //Update the database as well! if (parcelManagementModule != null) { foreach (LandData parcel in landData) { parcelManagementModule.UpdateLandObject(parcelManagementModule.GetLandObject(parcel.LocalID)); } } } else if (parcelManagementModule != null) parcelManagementModule.ResetSimLandObjects(); MainConsole.Instance.InfoFormat("[ARCHIVER]: Restored {0} parcels.", landData.Count); //Clean it out landData.Clear(); MainConsole.Instance.InfoFormat("[ARCHIVER]: Successfully loaded archive in " + (DateTime.Now - start).Minutes + ":" + (DateTime.Now - start).Seconds); m_validUserUuids.Clear(); m_scene.EventManager.TriggerOarFileLoaded(UUID.Zero.Guid, m_errorMessage); }
private void DearchiveRegion0DotStar() { int successfulAssetRestores = 0; int failedAssetRestores = 0; string filePath = "NONE"; DateTime start = DateTime.Now; TarArchiveReader archive = new TarArchiveReader(m_loadStream); if (!m_skipAssets) m_threadpool = new Aurora.Framework.AuroraThreadPool(new Aurora.Framework.AuroraThreadPoolStartInfo() { Threads = 1, priority = System.Threading.ThreadPriority.BelowNormal }); IBackupModule backup = m_scene.RequestModuleInterface<IBackupModule>(); if (!m_merge) { DateTime before = DateTime.Now; MainConsole.Instance.Info("[ARCHIVER]: Clearing all existing scene objects"); if (backup != null) backup.DeleteAllSceneObjects(); MainConsole.Instance.Info("[ARCHIVER]: Cleared all existing scene objects in " + (DateTime.Now - before).Minutes + ":" + (DateTime.Now - before).Seconds); } IScriptModule[] modules = m_scene.RequestModuleInterfaces<IScriptModule>(); //Disable the script engine so that it doesn't load in the background and kill OAR loading foreach (IScriptModule module in modules) { module.Disabled = true; } //Disable backup for now as well if (backup != null) backup.LoadingPrims = true; IRegionSerialiserModule serialiser = m_scene.RequestModuleInterface<IRegionSerialiserModule>(); int sceneObjectsLoadedCount = 0; //We save the groups so that we can back them up later List<SceneObjectGroup> groupsToBackup = new List<SceneObjectGroup>(); List<LandData> landData = new List<LandData>(); IUserFinder UserManager = m_scene.RequestModuleInterface<IUserFinder>(); // must save off some stuff until after assets have been saved and recieved new uuids // keeping these collection local because I am sure they will get large and garbage collection is better that way List<byte[]> seneObjectGroups = new List<byte[]>(); Dictionary<UUID, UUID> assetBinaryChangeRecord = new Dictionary<UUID, UUID>(); Queue<UUID> assets2Save = new Queue<UUID>(); try { byte[] data; TarArchiveReader.TarEntryType entryType; while ((data = archive.ReadEntry(out filePath, out entryType)) != null) { if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; if (filePath.StartsWith(ArchiveConstants.OBJECTS_PATH)) { seneObjectGroups.Add(data); } else if (!m_skipAssets && filePath.StartsWith(ArchiveConstants.ASSETS_PATH)) { AssetBase asset; if (LoadAsset(filePath, data, out asset)) { successfulAssetRestores++; if (m_useAsync) lock (AssetsToAdd) AssetsToAdd.Add(asset); else { if (asset.IsBinaryAsset) { UUID aid = asset.ID; asset.ID = m_scene.AssetService.Store(asset); if (asset.ID != aid && asset.ID != UUID.Zero) assetBinaryChangeRecord.Add(aid, asset.ID); } else { if (!assetNonBinaryCollection.ContainsKey(asset.ID)) { assetNonBinaryCollection.Add(asset.ID, asset); // I need something I can safely loop through assets2Save.Enqueue(asset.ID); } } } } else failedAssetRestores++; if ((successfulAssetRestores + failedAssetRestores) % 250 == 0) MainConsole.Instance.Info("[ARCHIVER]: Loaded " + successfulAssetRestores + " assets and failed to load " + failedAssetRestores + " assets..."); } else if (filePath.StartsWith(ArchiveConstants.TERRAINS_PATH)) { LoadTerrain(filePath, data); } else if (!m_merge && filePath.StartsWith(ArchiveConstants.SETTINGS_PATH)) { LoadRegionSettings(filePath, data); } else if (filePath.StartsWith(ArchiveConstants.LANDDATA_PATH)) { LandData parcel = LandDataSerializer.Deserialize(m_utf8Encoding.GetString(data)); parcel.OwnerID = ResolveUserUuid(parcel.OwnerID, UUID.Zero, "", Vector3.Zero, null); landData.Add(parcel); } else if (filePath == ArchiveConstants.CONTROL_FILE_PATH) { LoadControlFile(data); } } // Save Assets int savingAssetsCount = 0; while (assets2Save.Count > 0) { try { UUID assetid = assets2Save.Dequeue(); SaveNonBinaryAssets(assetid, assetNonBinaryCollection[assetid], assetBinaryChangeRecord); savingAssetsCount++; if ((savingAssetsCount) % 250 == 0) MainConsole.Instance.Info("[ARCHIVER]: Saving " + savingAssetsCount + " assets..."); } catch (Exception ex) { MainConsole.Instance.Info("[ARCHIVER]: Exception in saving an asset: " + ex.ToString()); } } foreach (byte[] data2 in seneObjectGroups) { byte[] data3 = data2; string stringData = Utils.BytesToString(data3); MatchCollection mc = Regex.Matches(stringData, sPattern); bool didChange = false; if (mc.Count >= 1) { foreach (Match match in mc) { UUID thematch = new UUID(match.Value); UUID newvalue = thematch; if (assetNonBinaryCollection.ContainsKey(thematch)) newvalue = assetNonBinaryCollection[thematch].ID; else if (assetBinaryChangeRecord.ContainsKey(thematch)) newvalue = assetBinaryChangeRecord[thematch]; if (thematch == newvalue) continue; stringData = stringData.Replace(thematch.ToString().Trim(), newvalue.ToString().Trim()); didChange = true; } } if (didChange) data3 = Utils.StringToBytes(stringData); SceneObjectGroup sceneObject = (SceneObjectGroup)serialiser.DeserializeGroupFromXml2(data3, m_scene); if (sceneObject == null) { //! big error! MainConsole.Instance.Error("Error reading SOP XML (Please mantis this!): " + m_asciiEncoding.GetString(data3)); continue; } foreach (SceneObjectPart part in sceneObject.ChildrenList) { if (string.IsNullOrEmpty(part.CreatorData)) part.CreatorID = ResolveUserUuid(part.CreatorID, part.CreatorID, part.CreatorData, part.AbsolutePosition, landData); if (UserManager != null) UserManager.AddUser(part.CreatorID, part.CreatorData); part.OwnerID = ResolveUserUuid(part.OwnerID, part.CreatorID, part.CreatorData, part.AbsolutePosition, landData); part.LastOwnerID = ResolveUserUuid(part.LastOwnerID, part.CreatorID, part.CreatorData, part.AbsolutePosition, landData); // And zap any troublesome sit target information part.SitTargetOrientation = new Quaternion(0, 0, 0, 1); part.SitTargetPosition = new Vector3(0, 0, 0); // Fix ownership/creator of inventory items // Not doing so results in inventory items // being no copy/no mod for everyone lock (part.TaskInventory) { TaskInventoryDictionary inv = part.TaskInventory; foreach (KeyValuePair<UUID, TaskInventoryItem> kvp in inv) { kvp.Value.OwnerID = ResolveUserUuid(kvp.Value.OwnerID, kvp.Value.CreatorID, kvp.Value.CreatorData, part.AbsolutePosition, landData); if (string.IsNullOrEmpty(kvp.Value.CreatorData)) kvp.Value.CreatorID = ResolveUserUuid(kvp.Value.CreatorID, kvp.Value.CreatorID, kvp.Value.CreatorData, part.AbsolutePosition, landData); if (UserManager != null) UserManager.AddUser(kvp.Value.CreatorID, kvp.Value.CreatorData); } } } //Add the offsets of the region Vector3 newPos = new Vector3(sceneObject.AbsolutePosition.X + m_offsetX, sceneObject.AbsolutePosition.Y + m_offsetY, sceneObject.AbsolutePosition.Z + m_offsetZ); if (m_flipX) newPos.X = m_scene.RegionInfo.RegionSizeX - newPos.X; if (m_flipY) newPos.Y = m_scene.RegionInfo.RegionSizeY - newPos.Y; sceneObject.SetAbsolutePosition(false, newPos); if (m_scene.SceneGraph.AddPrimToScene(sceneObject)) { groupsToBackup.Add(sceneObject); sceneObject.ScheduleGroupUpdate(PrimUpdateFlags.ForcedFullUpdate); sceneObject.CreateScriptInstances(0, false, StateSource.RegionStart, UUID.Zero, true); } sceneObjectsLoadedCount++; if (sceneObjectsLoadedCount % 250 == 0) MainConsole.Instance.Info("[ARCHIVER]: Loaded " + sceneObjectsLoadedCount + " objects..."); } assetNonBinaryCollection.Clear(); assetBinaryChangeRecord.Clear(); seneObjectGroups.Clear(); } catch (Exception e) { MainConsole.Instance.ErrorFormat( "[ARCHIVER]: Aborting load with error in archive file {0}. {1}", filePath, e); m_errorMessage += e.ToString(); m_scene.EventManager.TriggerOarFileLoaded(UUID.Zero.Guid, m_errorMessage); return; } finally { archive.Close(); m_loadStream.Close(); m_loadStream.Dispose(); //Reeanble now that we are done foreach (IScriptModule module in modules) { module.Disabled = false; } //Reset backup too if (backup != null) backup.LoadingPrims = false; } //Now back up the prims foreach (SceneObjectGroup grp in groupsToBackup) { //Backup! grp.HasGroupChanged = true; } if (!m_skipAssets && m_useAsync && !AssetSaverIsRunning) m_threadpool.QueueEvent(SaveAssets, 0); if (!m_skipAssets) { MainConsole.Instance.InfoFormat("[ARCHIVER]: Restored {0} assets", successfulAssetRestores); if (failedAssetRestores > 0) { MainConsole.Instance.ErrorFormat("[ARCHIVER]: Failed to load {0} assets", failedAssetRestores); m_errorMessage += String.Format("Failed to load {0} assets", failedAssetRestores); } } // Try to retain the original creator/owner/lastowner if their uuid is present on this grid // otherwise, use the master avatar uuid instead // Reload serialized parcels MainConsole.Instance.InfoFormat("[ARCHIVER]: Loading {0} parcels. Please wait.", landData.Count); IParcelManagementModule parcelManagementModule = m_scene.RequestModuleInterface<IParcelManagementModule>(); if (!m_merge && parcelManagementModule != null) parcelManagementModule.ClearAllParcels(); if (landData.Count > 0) { m_scene.EventManager.TriggerIncomingLandDataFromStorage(landData, new Vector2(m_offsetX, m_offsetY)); //Update the database as well! if (parcelManagementModule != null) { foreach (LandData parcel in landData) { parcelManagementModule.UpdateLandObject(parcelManagementModule.GetLandObject(parcel.LocalID)); } } } else if (parcelManagementModule != null) parcelManagementModule.ResetSimLandObjects(); MainConsole.Instance.InfoFormat("[ARCHIVER]: Restored {0} parcels.", landData.Count); //Clean it out landData.Clear(); MainConsole.Instance.InfoFormat("[ARCHIVER]: Successfully loaded archive in " + (DateTime.Now - start).Minutes + ":" + (DateTime.Now - start).Seconds); m_validUserUuids.Clear(); m_scene.EventManager.TriggerOarFileLoaded(UUID.Zero.Guid, m_errorMessage); }
private void LoadIAR(string fileName) { //Load the iar into memory TarArchiveReader archive = new TarArchiveReader (new GZipStream (ArchiveHelpers.GetStream (fileName), CompressionMode.Decompress)); byte[] data; TarArchiveReader.TarEntryType entryType; string filePath; InventoryFolderBase rootDestFolder = new InventoryFolderBase (UUID.Zero, UUID.Zero); Dictionary<string, InventoryFolderBase> resolvedFolders = new Dictionary<string, InventoryFolderBase> (); while ((data = archive.ReadEntry (out filePath, out entryType)) != null) { if (filePath.StartsWith (ArchiveConstants.ASSETS_PATH)) { LoadAsset (filePath, data); } else if (filePath.StartsWith (ArchiveConstants.INVENTORY_PATH)) { filePath = filePath.Substring (ArchiveConstants.INVENTORY_PATH.Length); // Trim off the file portion if we aren't already dealing with a directory path if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType) filePath = filePath.Remove (filePath.LastIndexOf ("/") + 1); InventoryFolderBase foundFolder = ReplicateArchivePathToUserInventory ( filePath, rootDestFolder, ref resolvedFolders); if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType) { LoadItem (data, foundFolder); } } } archive.Close (); //Got the .iar loaded into memory now // Time to put it into the GUI RebuildTreeView (); }
protected virtual void ReadBackup(IScene scene) { MainConsole.Instance.Info("[FileBasedSimulationData]: Reading file for " + scene.RegionInfo.RegionName); List<uint> foundLocalIDs = new List<uint>(); GZipStream m_loadStream; try { m_loadStream = new GZipStream( ArchiveHelpers.GetStream(((m_loadDirectory == "" || m_loadDirectory == "/") ? m_fileName : Path.Combine(m_loadDirectory, m_fileName))), CompressionMode.Decompress); } catch { if (CheckForOldDataBase()) SaveBackup(m_saveDirectory, false); return; } TarArchiveReader reader = new TarArchiveReader(m_loadStream); byte[] data; string filePath; TarArchiveReader.TarEntryType entryType; //Load the archive data that we need while ((data = reader.ReadEntry(out filePath, out entryType)) != null) { if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; if (filePath.StartsWith("parcels/")) { //Only use if we are not merging LandData parcel = new LandData(); OSD parcelData = OSDParser.DeserializeLLSDBinary(data); parcel.FromOSD((OSDMap) parcelData); m_parcels.Add(parcel); } else if (filePath.StartsWith("terrain/")) { m_oldstyleterrain = data; } else if (filePath.StartsWith("revertterrain/")) { m_oldstylerevertTerrain = data; } else if (filePath.StartsWith("newstyleterrain/")) { m_terrain = data; } else if (filePath.StartsWith("newstylerevertterrain/")) { m_revertTerrain = data; } else if (filePath.StartsWith("newstylewater/")) { m_water = data; } else if (filePath.StartsWith("newstylerevertwater/")) { m_revertWater = data; } else if (filePath.StartsWith("entities/")) { MemoryStream ms = new MemoryStream(data); SceneObjectGroup sceneObject = SceneObjectSerializer.FromXml2Format(ref ms, scene); ms.Close(); ms = null; data = null; foreach (ISceneChildEntity part in sceneObject.ChildrenEntities()) { if (!foundLocalIDs.Contains(part.LocalId)) foundLocalIDs.Add(part.LocalId); else part.LocalId = 0; //Reset it! Only use it once! } m_groups.Add(sceneObject); } data = null; } m_loadStream.Close(); m_loadStream = null; foundLocalIDs.Clear(); GC.Collect(); }
public void LoadRegionBackup(TarArchiveReader reader, IScene scene) { IAuroraBackupModule[] modules = scene.RequestModuleInterfaces<IAuroraBackupModule>(); byte[] data; string filePath; TarArchiveReader.TarEntryType entryType; foreach (IAuroraBackupModule module in modules) module.BeginLoadModuleFromArchive(scene); while ((data = reader.ReadEntry(out filePath, out entryType)) != null) { if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; foreach (IAuroraBackupModule module in modules) module.LoadModuleFromArchive(data, filePath, entryType, scene); } reader.Close(); foreach (IAuroraBackupModule module in modules) module.EndLoadModuleFromArchive(scene); }
public void LoadFromFile() { GZipStream m_loadStream = new GZipStream(ArchiveHelpers.GetStream(m_fileName), CompressionMode.Decompress); TarArchiveReader reader = new TarArchiveReader(m_loadStream); byte[] data; string filePath; TarArchiveReader.TarEntryType entryType; #region Our Region Info IParcelServiceConnector parcelService = Aurora.DataManager.DataManager.RequestPlugin<IParcelServiceConnector>(); SceneManager sceneManager = m_simulationBase.ApplicationRegistry.RequestModuleInterface<SceneManager>(); RegionInfo regionInfo = new RegionInfo(); IScene fakeScene = new Scene(); fakeScene.AddModuleInterfaces(m_simulationBase.ApplicationRegistry.GetInterfaces()); #endregion while ((data = reader.ReadEntry(out filePath, out entryType)) != null) { if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; if (filePath.StartsWith("estate/")) { string estateData = Encoding.UTF8.GetString(data); EstateSettings settings = new EstateSettings(WebUtils.ParseXmlResponse(estateData)); regionInfo.EstateSettings = settings; } else if (filePath.StartsWith("regioninfo/")) { regionInfo.UnpackRegionInfoData((OSDMap)OSDParser.DeserializeLLSDBinary(data)); } } if (!m_useExistingRegionInfo) { regionInfo.RegionID = UUID.Random(); regionInfo.RegionName = MainConsole.Instance.Prompt("Region Name: ", regionInfo.RegionName); regionInfo.RegionLocX = int.Parse(MainConsole.Instance.Prompt("Region Position X: ", regionInfo.RegionLocX.ToString())); regionInfo.RegionLocY = int.Parse(MainConsole.Instance.Prompt("Region Position Y: ", regionInfo.RegionLocY.ToString())); regionInfo.InternalEndPoint.Port = int.Parse(MainConsole.Instance.Prompt("HTTP Port: ", regionInfo.InternalEndPoint.Port.ToString())); } //ISimulationDataStore simulationStore = sceneManager.SimulationDataService; //Hijack the old simulation service and replace it with ours sceneManager.SimulationDataService = new OverridenFileBasedSimulationData (m_fileName, m_saveNewArchiveAtClose); ///Now load the region! sceneManager.AllRegions++; sceneManager.StartNewRegion (regionInfo); }