1

replay/genericpersistentmanager.h

@@ -0,0 +1,712 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+//=======================================================================================//
+
+#ifndef GENERICPERSISTENTMANAGER_H
+#define GENERICPERSISTENTMANAGER_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+//----------------------------------------------------------------------------------------
+
+#include "replay/replayhandle.h"
+#include "replay/ienginereplay.h"
+#include "replay/replayutils.h"
+#include "basethinker.h"
+#include "utllinkedlist.h"
+#include "utlstring.h"
+#include "KeyValues.h"
+#include "filesystem.h"
+#include "convar.h"
+#include "replay/ireplayserializeable.h"
+#include "replay/ireplaycontext.h"
+#include "replay/shared_defs.h"
+#include "replay_dbg.h"
+#include "vprof.h"
+#include "fmtstr.h"
+#include "UtlSortVector.h"
+
+// memdbgon must be the last include file in a .cpp file!!!
+#include "tier0/memdbgon.h"
+
+//----------------------------------------------------------------------------------------
+
+extern IEngineReplay *g_pEngine;
+
+//----------------------------------------------------------------------------------------
+
+template< class T >
+class CGenericPersistentManager	: public CBaseThinker
+{
+public:
+						CGenericPersistentManager();
+	virtual				~CGenericPersistentManager();
+
+	virtual bool		Init( bool bLoad = true );
+	virtual void		Shutdown();
+
+	virtual T			*Create() = 0;	// Create an object
+	T					*CreateAndGenerateHandle();	// Creates a new object and generates a unique handle
+
+	void				Add( T *pNewObj );	// Commit the object - NOTE: The Create*() functions don't call Add() for you
+	void				Remove( ReplayHandle_t hObj );	// Remove() will remove the object, remove any .dmx associated with the object on disk, and usually delete attached files (like .dems or movies, etc, depending on what the manager implementation is)
+	void				Remove( T *pObj );
+	void				RemoveFromIndex( int it );
+	void				Clear();	// Remove all objects - NOTE: Doesn't save right away
+	
+	bool				WriteObjToFile( T *pObj, const char *pFilename );	// Write object data to an arbitrary file
+	bool				Save();	// Saves any unsaved data immediately
+
+	void				FlagIndexForFlush();	// Mark index as dirty
+	void				FlagForFlush( T *pObj, bool bForceImmediate );	// Mark an object as dirty
+	void				FlagForUnload( T *pObj );	// Unload as soon as possible
+
+	T					*Find( ReplayHandle_t hHandle );
+	int					FindIteratorFromHandle( ReplayHandle_t hHandle );
+
+	int					Count() const;
+	bool				IsDirty( T *pNewObj );
+
+	virtual void		Think();	// IReplayThinker implementation - NOTE: not meant to be called directly - called from think manager
+	virtual const char	*GetIndexPath() const;	// Should return path where index file lives
+
+private:
+	class CLessFunctor
+	{
+	public:
+		bool Less( const T *pSrc1, const T *pSrc2, void *pContext )
+		{
+			return pSrc1->GetHandle() < pSrc2->GetHandle();
+		}
+	};
+
+public:
+	typedef CUtlSortVector< T *, CLessFunctor >	ObjContainer_t;
+	ObjContainer_t		m_vecObjs;
+
+protected:
+	// For derived classes to implement:
+	virtual IReplayContext	*GetReplayContext() const = 0;
+	virtual const char	*GetRelativeIndexPath() const = 0;	// Should return relative (to replay/client or replay/server) path where index file lives - NOTE: Last char should be a slash
+	virtual const char	*GetIndexFilename() const = 0;	// Should return just the name of the file, e.g. "replays.dmx"
+	virtual const char	*GetDebugName() const = 0;
+	virtual bool		ShouldDeleteObjects() const { return true; }	// TODO: Used by Clear() - I'm not convinced this is needed yet though.
+	virtual int			GetVersion() const = 0;
+	virtual bool		ShouldSerializeToIndividualFiles() const { return true; }
+	virtual bool		ShouldSerializeIndexWithFullPath()  const { return false; }
+	virtual bool		ShouldLoadObj( const T *pObj ) const { return true; }
+	virtual void		OnObjLoaded( T *pObj ) {}
+
+	virtual int			GetHandleBase() const	{ return 0; }	// Subclass can implement this to provide a base/minimum for handles
+	virtual void		PreLoad() {}
+
+	const char			*GetIndexFullFilename() const;		// Should return the full path to the main .dmx file
+	bool				HaveDirtyObjects() const;
+	bool				HaveObjsToUnload() const;
+	bool				ReadObjFromFile( const char *pFile, T *&pOut, bool bForceLoad );
+	bool				Load();
+
+	virtual float		GetNextThinkTime() const;	// IReplayThinker implementation
+	void				FlushThink();
+	void				UnloadThink();
+	void				CreateIndexDir();
+	void				ReadObjFromKeyValues( KeyValues *pObjData );
+	T*					ReadObjFromKeyValues( KeyValues *pObjData, bool bForceLoad );
+	bool				ReadObjFromFile( const char *pFile );
+	void				UpdateHandleSeed( ReplayHandle_t hNewHandle );
+
+	typedef CUtlLinkedList< T *, int > ListContainer_t;
+
+	ReplayHandle_t		m_nHandleSeed;
+	int					m_nVersion;
+	bool				m_bIndexDirty;
+	ListContainer_t		m_lstDirtyObjs;
+	ListContainer_t		m_lstObjsToUnload;
+	float				m_flNextFlushTime;
+	float				m_flNextUnloadTime;
+};
+
+//----------------------------------------------------------------------------------------
+
+template< class T >
+bool CGenericPersistentManager< T >::Init( bool bLoad/*=true*/ )
+{
+	// Make directory structure is in place
+	CreateIndexDir();
+
+	// Initialize handle seed to start at base
+	m_nHandleSeed = GetHandleBase();
+
+	return bLoad ? Load() : true;
+}
+
+template< class T >
+void CGenericPersistentManager< T >::Shutdown()
+{
+	Save();
+}
+
+template< class T >
+CGenericPersistentManager< T >::CGenericPersistentManager()
+:	m_nHandleSeed( 0 ),
+	m_nVersion( -1 ),
+	m_bIndexDirty( false ),
+	m_flNextFlushTime( 0.0f ),
+	m_flNextUnloadTime( 0.0f )
+{
+}
+	
+template< class T >
+CGenericPersistentManager< T >::~CGenericPersistentManager()
+{
+	Clear();
+}
+
+template< class T >
+void CGenericPersistentManager< T >::Clear()
+{
+	if ( ShouldDeleteObjects() )
+	{
+		m_vecObjs.PurgeAndDeleteElements();
+	}
+	else
+	{
+		m_vecObjs.RemoveAll();
+	}
+
+	// NOTE: This list contains pointers to objects in m_vecObjs, so no destruction of elements here
+	m_lstDirtyObjs.RemoveAll();
+	m_lstObjsToUnload.RemoveAll();
+}
+
+template< class T >
+int CGenericPersistentManager< T >::Count() const
+{
+	return m_vecObjs.Count();
+}
+
+template< class T >
+void CGenericPersistentManager< T >::FlagIndexForFlush()
+{
+	m_bIndexDirty = true;
+
+	IF_REPLAY_DBG2( Warning( "%f %s: Index flagged\n", g_pEngine->GetHostTime(), GetDebugName() ) );
+}
+
+template< class T >
+void CGenericPersistentManager< T >::FlagForFlush( T *pObj, bool bForceImmediate )
+{
+	if ( !pObj )
+	{
+		AssertMsg( 0, "Trying to flag a NULL object for flush." );
+		return;
+	}
+
+	// Add to dirty list if it's not already there
+	if ( m_lstDirtyObjs.Find( pObj ) == m_lstDirtyObjs.InvalidIndex() )
+	{
+		m_lstDirtyObjs.AddToTail( pObj );
+	}
+
+	IF_REPLAY_DBG2( Warning( "%f %s: Obj %s flagged for flush\n", g_pEngine->GetHostTime(), GetDebugName(), pObj->GetDebugName() ) );
+
+	// Force write now?
+	if ( bForceImmediate )
+	{
+		Save();
+	}
+}
+
+template< class T >
+void CGenericPersistentManager< T >::FlagForUnload( T *pObj )
+{
+	AssertMsg(
+		ShouldSerializeToIndividualFiles(),
+		"This functionality should only be used for managers that write to individual files, i.e. NOT managers that maintain one monolithic index."
+	);
+
+	if ( !pObj )
+	{
+		AssertMsg( 0, "Trying to flag a NULL object for unload." );
+		return;
+	}
+
+	if ( m_lstObjsToUnload.Find( pObj ) == m_lstObjsToUnload.InvalidIndex() )
+	{
+		m_lstObjsToUnload.AddToTail( pObj );
+	}
+
+	IF_REPLAY_DBG2( Warning( "%f %s: Obj %s flagged for unload\n", g_pEngine->GetHostTime(), GetDebugName(), pObj->GetDebugName() ) );
+}
+
+template< class T >
+bool CGenericPersistentManager< T >::IsDirty( T *pNewObj )
+{
+	return m_lstDirtyObjs.Find( pNewObj ) != m_lstDirtyObjs.InvalidIndex();
+}
+
+template< class T >
+void CGenericPersistentManager< T >::Add( T *pNewObj )
+{
+	IF_REPLAY_DBG2( Warning( "Adding object with handle %i\n", pNewObj->GetHandle() ) );
+	Assert( m_vecObjs.Find( pNewObj ) == m_vecObjs.InvalidIndex() );
+	m_vecObjs.Insert( pNewObj );
+	FlagIndexForFlush();
+	FlagForFlush( pNewObj, false );
+}
+
+template< class T >
+void CGenericPersistentManager< T >::Remove( ReplayHandle_t hObj )
+{
+	int itObj = FindIteratorFromHandle( hObj );
+	if ( itObj == m_vecObjs.InvalidIndex() )
+	{
+		AssertMsg( 0, "Attemting to remove an object which does not exist." );
+		return;
+	}
+
+	RemoveFromIndex( itObj );
+}
+
+template< class T >
+void CGenericPersistentManager< T >::Remove( T *pObj )
+{
+	const int it = m_vecObjs.Find( pObj );
+
+	if ( it != m_vecObjs.InvalidIndex() )
+	{
+		RemoveFromIndex( it );
+	}
+}
+
+template< class T >
+void CGenericPersistentManager< T >::RemoveFromIndex( int it )
+{
+	T *pObj = m_vecObjs[ it ];	// NOTE: Constant speed since the implementation of
+								// CUtlLinkedList indexes into an array
+
+	// Remove file associated w/ this object if necessary
+	if ( ShouldSerializeToIndividualFiles() )
+	{
+		CUtlString strFullFilename = pObj->GetFullFilename();
+		bool bSimulateDelete = false;
+#if _DEBUG
+		extern ConVar replay_fileserver_simulate_delete;
+		bSimulateDelete = replay_fileserver_simulate_delete.GetBool();
+#endif
+		if ( g_pFullFileSystem->FileExists( strFullFilename.Get() ) && !bSimulateDelete )
+		{
+			g_pFullFileSystem->RemoveFile( strFullFilename.Get() );
+		}
+	}
+
+	Assert( !pObj->IsLocked() );
+
+	// Let the object do stuff before it gets deleted
+	pObj->OnDelete();
+
+	// If the object is in the dirty list, remove it - NOTE: this is safe
+	m_lstDirtyObjs.FindAndRemove( pObj );
+
+	// The object should not be in the 'objects-to-unload' list
+	AssertMsg( m_lstObjsToUnload.Find( pObj ) == m_lstObjsToUnload.InvalidIndex(), "The object being removed was also in the unload list - is this OK?  If so, code should be added to remove from that list as well." );
+
+	// Remove the object
+	m_vecObjs.Remove( it );
+
+	// Free the object
+	delete pObj;
+
+	FlagIndexForFlush();
+}
+
+template< class T >
+T *CGenericPersistentManager< T >::Find( ReplayHandle_t hHandle )
+{
+	FOR_EACH_VEC( m_vecObjs, i )
+	{
+		T *pCurObj = m_vecObjs[ i ];
+		if ( hHandle == pCurObj->GetHandle() )
+		{
+			return pCurObj;
+		}
+	}
+
+	return NULL;
+}
+
+template< class T >
+int CGenericPersistentManager< T >::FindIteratorFromHandle( ReplayHandle_t hHandle )
+{
+	FOR_EACH_VEC( m_vecObjs, i )
+	{
+		T *pCurObj = m_vecObjs[ i ];
+		if ( hHandle == pCurObj->GetHandle() )
+		{
+			return i;
+		}
+	}
+
+	return m_vecObjs.InvalidIndex();
+}
+
+template< class T >
+bool CGenericPersistentManager< T >::Load()
+{
+	bool bResult = true;
+
+	Clear();
+	PreLoad();
+
+	const char *pFullFilename = GetIndexFullFilename();
+
+	// Attempt to load from disk
+	KeyValuesAD pRoot( pFullFilename );
+	if ( pRoot->LoadFromFile( g_pFullFileSystem, pFullFilename ) )
+	{
+		// Get file format version
+		m_nVersion = pRoot->GetInt( "version", -1 );
+		if ( m_nVersion != GetVersion() )
+		{
+			Warning( "File (%s) has old format (%i).\n", pFullFilename, m_nVersion );
+		}
+
+		// Read from individual files?
+		if ( ShouldSerializeToIndividualFiles() )
+		{
+			KeyValues *pFileIndex = pRoot->FindKey( "files" );
+			if ( pFileIndex )
+			{
+				FOR_EACH_VALUE( pFileIndex, pValue )
+				{
+					const char *pName = pValue->GetName();
+					if ( !ReadObjFromFile( pName ) )
+					{
+						Warning( "Failed to load data from file, \"%s\"\n", pName );
+					}
+				}
+			}
+			else
+			{
+				// Peek in directory and load files based on what's there
+				CFmtStr fmtPath( "%s*.%s", GetIndexPath(), GENERIC_FILE_EXTENSION );
+				FileFindHandle_t hFind;
+				const char *pFilename = g_pFullFileSystem->FindFirst( fmtPath.Access(), &hFind );
+				while ( pFilename )
+				{
+					// Ignore index file
+					if ( V_stricmp( pFilename, GetIndexFilename() ) )
+					{
+						if ( !ReadObjFromFile( pFilename ) )
+						{
+							Warning( "Failed to load data from file, \"%s\"\n", pFilename );
+						}
+					}
+
+					pFilename = g_pFullFileSystem->FindNext( hFind );
+				}
+			}
+		}
+		else
+		{
+			FOR_EACH_TRUE_SUBKEY( pRoot, pObjSubKey )
+			{
+				// Read data
+				m_vecObjs.Insert( ReadObjFromKeyValues( pObjSubKey, false ) );
+			}
+		}
+
+		// Let derived class do any per-object processing.
+		FOR_EACH_VEC( m_vecObjs, i )
+		{
+			OnObjLoaded( m_vecObjs[ i ] );
+		}
+	}
+
+	return bResult;
+}
+
+template< class T >
+bool CGenericPersistentManager< T >::WriteObjToFile( T *pObj, const char *pFilename )
+{
+	// Create a keyvalues for the object
+	KeyValuesAD pObjData( pObj->GetSubKeyTitle() );
+
+	// Fill the keyvalues w/ data
+	pObj->Write( pObjData );
+
+	// Attempt to save the current object data to a separate file
+	if ( !pObjData->SaveToFile( g_pFullFileSystem, pFilename ) )
+	{
+		Warning( "Failed to write file %s\n", pFilename );
+		return false;
+	}
+
+	return true;
+}
+
+template< class T >
+bool CGenericPersistentManager< T >::Save()
+{
+	IF_REPLAY_DBG2( Warning( "%f %s: Saving now...\n", g_pEngine->GetHostTime(), GetDebugName() ) );
+
+	bool bResult = true;
+
+	// Add subkey for movies
+	KeyValuesAD pRoot( "root" );
+
+	// Write format version
+	pRoot->SetInt( "version", GetVersion() );
+	
+	// Write a file index instead of adding subkeys to the root?
+	if ( ShouldSerializeToIndividualFiles() )
+	{
+		// Go through each object in the dirty list and write to a separate file
+		FOR_EACH_LL( m_lstDirtyObjs, i )
+		{
+			T *pCurObj = m_lstDirtyObjs[ i ];
+
+			// Write to the file
+			bResult = bResult && WriteObjToFile( pCurObj, pCurObj->GetFullFilename() );
+		}
+	}
+
+	// Write all objects to one monolithic file - writes all objects (ignores "dirtyness")
+	else
+	{
+		FOR_EACH_VEC( m_vecObjs, i )
+		{
+			T *pCurObj = m_vecObjs[ i ];
+
+			// Create a keyvalues for the object
+			KeyValues *pCurObjData = new KeyValues( pCurObj->GetSubKeyTitle() );
+
+			// Fill the keyvalues w/ data
+			pCurObj->Write( pCurObjData );
+
+			// Add as a subkey to the root keyvalues
+			pRoot->AddSubKey( pCurObjData );
+		}
+	}
+
+	// Clear the dirty list
+	m_lstDirtyObjs.RemoveAll();
+
+	// Write the index file if dirty
+	if ( m_bIndexDirty )
+	{
+		return bResult && pRoot->SaveToFile( g_pFullFileSystem, GetIndexFullFilename() );
+	}
+
+	return bResult;
+}
+
+template< class T >
+T *CGenericPersistentManager< T >::CreateAndGenerateHandle()
+{
+	T *pNewObj = Create();
+	pNewObj->SetHandle( m_nHandleSeed++ );		Assert( Find( pNewObj->GetHandle() ) == NULL );
+	FlagIndexForFlush();
+	return pNewObj;
+}
+
+template< class T >
+float CGenericPersistentManager< T >::GetNextThinkTime() const
+{
+	// Always think
+	return 0.0f;
+}
+
+template< class T >
+void CGenericPersistentManager< T >::Think()
+{
+	VPROF_BUDGET( "CGenericPersistentManager::Think", VPROF_BUDGETGROUP_REPLAY );
+
+	CBaseThinker::Think();
+
+	FlushThink();
+	UnloadThink();
+}
+
+template< class T >
+void CGenericPersistentManager< T >::FlushThink()
+{
+	const float flHostTime = g_pEngine->GetHostTime();
+	bool bTimeToFlush = flHostTime >= m_flNextFlushTime;
+	if ( !bTimeToFlush || ( !m_bIndexDirty && !HaveDirtyObjects() ) )
+		return;
+
+	// Flush now and clear dirty objects
+	Save();
+
+	// Reset
+	m_bIndexDirty = false;
+
+	// Setup next flush think
+	extern ConVar replay_flushinterval;
+	m_flNextFlushTime = flHostTime + replay_flushinterval.GetInt();
+}
+
+template< class T >
+void CGenericPersistentManager< T >::UnloadThink()
+{
+	const float flHostTime = g_pEngine->GetHostTime();
+	bool bTimeToUnload = flHostTime >= m_flNextUnloadTime;
+	if ( !bTimeToUnload || !HaveObjsToUnload() )
+		return;
+
+	// Unload objects now
+	FOR_EACH_LL( m_lstObjsToUnload, i )
+	{
+		T *pObj = m_lstObjsToUnload[ i ];
+		
+		// If the object has been marked as locked, don't unload it.
+		if ( pObj->IsLocked() )
+			continue;
+
+		// If we're waiting to flush the file, don't unload it yet
+		if ( IsDirty( pObj ) )
+			continue;
+
+		// Let the object do stuff before it gets deleted
+		pObj->OnUnload();
+
+		// Remove the object
+		m_vecObjs.FindAndRemove( pObj );
+
+		IF_REPLAY_DBG( Warning( "Unloading object %s\n", pObj->GetDebugName() ) );
+
+		// Free the object
+		delete pObj;
+	}
+
+	// Clear the list
+	m_lstObjsToUnload.RemoveAll();
+
+	// Think once a second
+	m_flNextUnloadTime = flHostTime + 1.0f;
+}
+
+template< class T >
+const char	*CGenericPersistentManager< T >::GetIndexPath() const
+{
+	return Replay_va( "%s%s", GetReplayContext()->GetBaseDir(), GetRelativeIndexPath() );
+}
+
+template< class T >
+const char *CGenericPersistentManager< T >::GetIndexFullFilename() const		// Should return the full path to the main .dmx file
+{
+	return Replay_va( "%s%s", GetIndexPath(), GetIndexFilename() );
+}
+
+template< class T >
+bool CGenericPersistentManager< T >::HaveDirtyObjects() const
+{
+	return m_lstDirtyObjs.Count() > 0;
+}
+
+template< class T >
+bool CGenericPersistentManager< T >::HaveObjsToUnload() const
+{
+	return m_lstObjsToUnload.Count() > 0;
+}
+
+template< class T >
+void CGenericPersistentManager< T >::CreateIndexDir()
+{
+	g_pFullFileSystem->CreateDirHierarchy( GetIndexPath(), "DEFAULT_WRITE_PATH" );
+}
+
+template< class T >
+bool CGenericPersistentManager< T >::ReadObjFromFile( const char *pFile, T *&pOut, bool bForceLoad )
+{
+	// Use the full path and filename specified, or construct it if necessary
+	CUtlString strFullFilename;
+	if ( ShouldSerializeIndexWithFullPath() )
+	{
+		strFullFilename = pFile;
+	}
+	else
+	{
+		strFullFilename.Format( "%s%s", GetIndexPath(), pFile );
+	}
+
+	// Attempt to load the file
+	KeyValuesAD pObjData( pFile );
+	if ( !pObjData->LoadFromFile( g_pFullFileSystem, strFullFilename.Get() ) )
+	{
+		Warning( "Failed to load from file %s\n", strFullFilename.Get() );
+		AssertMsg( 0, "Manager failed to load something..." );
+		return false;
+	}
+
+	// Create and read a new object
+	pOut = ReadObjFromKeyValues( pObjData, bForceLoad );
+	if ( !pOut )
+		return NULL;
+	
+	// Add the object to the manager
+	m_vecObjs.Insert( pOut );
+
+	return true;
+}
+
+template< class T >
+bool CGenericPersistentManager< T >::ReadObjFromFile( const char *pFile )
+{
+	T *pNewObj;
+	if ( !ReadObjFromFile( pFile, pNewObj, false ) )
+		return false;
+
+	return true;
+}
+
+template< class T >
+T* CGenericPersistentManager< T >::ReadObjFromKeyValues( KeyValues *pObjData, bool bForceLoad )
+{
+	T *pNewObj = Create();			Assert( pNewObj );
+	if ( !pNewObj )
+		return NULL;
+
+	// Attempt to read data for the object, and fail to load this particular object if the reader
+	// says we should.
+	if ( !pNewObj->Read( pObjData ) )
+	{
+		delete pNewObj;
+		return NULL;
+	}
+
+	// This object OK to load?  Only check if bForceLoad is false.
+	if ( !bForceLoad && !ShouldLoadObj( pNewObj ) )
+	{
+		delete pNewObj;
+		return NULL;
+	}
+
+	// Sync up handle seed
+	UpdateHandleSeed( pNewObj->GetHandle() );
+
+	return pNewObj;
+}
+
+template< class T >
+void CGenericPersistentManager< T >::UpdateHandleSeed( ReplayHandle_t hNewHandle )
+{
+	m_nHandleSeed = (ReplayHandle_t)( GetHandleBase() + MAX( (uint32)m_nHandleSeed, (uint32)hNewHandle ) + 1 );
+
+#ifdef _DEBUG
+	FOR_EACH_VEC( m_vecObjs, i )
+	{
+		AssertMsg( m_nHandleSeed != m_vecObjs[ i ]->GetHandle(), "Handle seed collision!" );
+	}
+#endif
+}
+
+//----------------------------------------------------------------------------------------
+
+#define FOR_EACH_OBJ( _manager, _i )	FOR_EACH_VEC( _manager->m_vecObjs, _i )
+
+//----------------------------------------------------------------------------------------
+
+#endif	// GENERICPERSISTENTMANAGER_H