ported datacache, datamodel, dmserializers

datacache/datacache.h

@@ -0,0 +1,384 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose:
+//
+//===========================================================================//
+
+#ifndef DATACACHE_H
+#define DATACACHE_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "datamanager.h"
+#include "utlhash.h"
+#include "mempool.h"
+#include "tier0/tslist.h"
+#include "datacache_common.h"
+#include "tier3/tier3.h"
+
+
+//-----------------------------------------------------------------------------
+//
+// Data Cache class declarations
+//
+//-----------------------------------------------------------------------------
+class CDataCache;
+class CDataCacheSection;
+
+//-----------------------------------------------------------------------------
+
+struct DataCacheItemData_t
+{
+	const void* pItemData;
+	unsigned			size;
+	DataCacheClientID_t	clientId;
+	CDataCacheSection* pSection;
+};
+
+//-------------------------------------
+
+#define DC_NO_NEXT_LOCKED ((DataCacheItem_t *)0xffffffff)
+#define DC_MAX_THREADS_FRAMELOCKED 4
+
+struct DataCacheItem_t : DataCacheItemData_t
+{
+	DataCacheItem_t(const DataCacheItemData_t& data)
+		: DataCacheItemData_t(data),
+		hLRU(INVALID_MEMHANDLE)
+	{
+		memset(pNextFrameLocked, 0xff, sizeof(pNextFrameLocked));
+	}
+
+	static DataCacheItem_t* CreateResource(const DataCacheItemData_t& data) { return new DataCacheItem_t(data); }
+	static unsigned int EstimatedSize(const DataCacheItemData_t& data) { return data.size; }
+	void DestroyResource();
+	DataCacheItem_t* GetData() { return this; }
+	unsigned int Size() { return size; }
+
+	memhandle_t		 hLRU;
+	DataCacheItem_t* pNextFrameLocked[DC_MAX_THREADS_FRAMELOCKED];
+
+	DECLARE_FIXEDSIZE_ALLOCATOR_MT(DataCacheItem_t);
+};
+
+//-------------------------------------
+
+typedef CDataManager<DataCacheItem_t, DataCacheItemData_t, DataCacheItem_t*, CThreadFastMutex> CDataCacheLRU;
+
+//-----------------------------------------------------------------------------
+// CDataCacheSection
+//
+// Purpose: Implements a sub-section of the global cache. Subsections are
+//			areas of the cache with thier own memory constraints and common
+//			management.
+//-----------------------------------------------------------------------------
+class CDataCacheSection : public IDataCacheSection
+{
+public:
+	CDataCacheSection(CDataCache* pSharedCache, IDataCacheClient* pClient, const char* pszName);
+	~CDataCacheSection();
+
+	IDataCache* GetSharedCache();
+	IDataCacheClient* GetClient() { return m_pClient; }
+	const char* GetName() { return szName; }
+
+	//--------------------------------------------------------
+	// IDataCacheSection methods
+	//--------------------------------------------------------
+	virtual void SetLimits(const DataCacheLimits_t& limits);
+	const DataCacheLimits_t& GetLimits();
+	virtual void SetOptions(unsigned options);
+	virtual void GetStatus(DataCacheStatus_t* pStatus, DataCacheLimits_t* pLimits = NULL);
+
+	inline unsigned GetNumBytes() { return m_status.nBytes; }
+	inline unsigned GetNumItems() { return m_status.nItems; }
+
+	inline unsigned GetNumBytesLocked() { return m_status.nBytesLocked; }
+	inline unsigned GetNumItemsLocked() { return m_status.nItemsLocked; }
+
+	inline unsigned GetNumBytesUnlocked() { return m_status.nBytes - m_status.nBytesLocked; }
+	inline unsigned GetNumItemsUnlocked() { return m_status.nItems - m_status.nItemsLocked; }
+
+	virtual void EnsureCapacity(unsigned nBytes, unsigned nItems = 1);
+
+	//--------------------------------------------------------
+
+	virtual bool Add(DataCacheClientID_t clientId, const void* pItemData, unsigned size, DataCacheHandle_t* pHandle);
+	virtual bool AddEx(DataCacheClientID_t clientId, const void* pItemData, unsigned size, unsigned flags, DataCacheHandle_t* pHandle);
+	virtual DataCacheHandle_t Find(DataCacheClientID_t clientId);
+	virtual DataCacheRemoveResult_t Remove(DataCacheHandle_t handle, const void** ppItemData = NULL, unsigned* pItemSize = NULL, bool bNotify = false);
+	virtual bool IsPresent(DataCacheHandle_t handle);
+
+	//--------------------------------------------------------
+
+	virtual void* Lock(DataCacheHandle_t handle);
+	virtual int Unlock(DataCacheHandle_t handle);
+	virtual void* Get(DataCacheHandle_t handle, bool bFrameLock = false);
+	virtual void* GetNoTouch(DataCacheHandle_t handle, bool bFrameLock = false);
+	virtual void LockMutex();
+	virtual void UnlockMutex();
+
+	//--------------------------------------------------------
+
+	virtual int BeginFrameLocking();
+	virtual bool IsFrameLocking();
+	virtual void* FrameLock(DataCacheHandle_t handle);
+	virtual int EndFrameLocking();
+
+	//--------------------------------------------------------
+
+	virtual int GetLockCount(DataCacheHandle_t handle);
+	virtual int BreakLock(DataCacheHandle_t handle);
+
+	//--------------------------------------------------------
+
+	virtual int* GetFrameUnlockCounterPtr();
+	int			m_nFrameUnlockCounter;
+
+	//--------------------------------------------------------
+
+	virtual bool Touch(DataCacheHandle_t handle);
+	virtual bool Age(DataCacheHandle_t handle);
+
+	//--------------------------------------------------------
+
+	virtual unsigned Flush(bool bUnlockedOnly = true, bool bNotify = true);
+	virtual unsigned Purge(unsigned nBytes);
+	unsigned PurgeItems(unsigned nItems);
+
+	//--------------------------------------------------------
+
+	virtual void OutputReport(DataCacheReportType_t reportType = DC_SUMMARY_REPORT);
+
+	virtual void UpdateSize(DataCacheHandle_t handle, unsigned int nNewSize);
+
+private:
+	friend void DataCacheItem_t::DestroyResource();
+
+	virtual void OnAdd(DataCacheClientID_t clientId, DataCacheHandle_t hCacheItem) {}
+	virtual DataCacheHandle_t DoFind(DataCacheClientID_t clientId);
+	virtual void OnRemove(DataCacheClientID_t clientId) {}
+
+	memhandle_t GetFirstUnlockedItem();
+	memhandle_t GetFirstLockedItem();
+	memhandle_t GetNextItem(memhandle_t);
+	DataCacheItem_t* AccessItem(memhandle_t hCurrent);
+	bool DiscardItem(memhandle_t hItem, DataCacheNotificationType_t type);
+	bool DiscardItemData(DataCacheItem_t* pItem, DataCacheNotificationType_t type);
+	void NoteAdd(int size);
+	void NoteRemove(int size);
+	void NoteLock(int size);
+	void NoteUnlock(int size);
+	void NoteSizeChanged(int oldSize, int newSize);
+
+	struct FrameLock_t
+	{
+		//$ WARNING: This needs a TSLNodeBase_t as the first item in here.
+		TSLNodeBase_t	base;
+		int				m_iLock;
+		DataCacheItem_t* m_pFirst;
+		int				m_iThread;
+	};
+	typedef CThreadLocal<FrameLock_t*> CThreadFrameLock;
+
+	CDataCacheLRU& m_LRU;
+	CThreadFrameLock	m_ThreadFrameLock;
+	DataCacheStatus_t	m_status;
+	DataCacheLimits_t	m_limits;
+	IDataCacheClient* m_pClient;
+	unsigned			m_options;
+	CDataCache* m_pSharedCache;
+	char				szName[DC_MAX_CLIENT_NAME + 1];
+	CTSSimpleList<FrameLock_t> m_FreeFrameLocks;
+
+protected:
+	CThreadFastMutex& m_mutex;
+};
+
+
+//-----------------------------------------------------------------------------
+// CDataCacheSectionFastFind
+//
+// Purpose: A section variant that allows clients to have cache support tracking
+//			efficiently (a true cache, not just an LRU)
+//-----------------------------------------------------------------------------
+class CDataCacheSectionFastFind : public CDataCacheSection
+{
+public:
+	CDataCacheSectionFastFind(CDataCache* pSharedCache, IDataCacheClient* pClient, const char* pszName)
+		: CDataCacheSection(pSharedCache, pClient, pszName)
+	{
+		m_Handles.Init(1024);
+	}
+
+private:
+	virtual DataCacheHandle_t DoFind(DataCacheClientID_t clientId);
+	virtual void OnAdd(DataCacheClientID_t clientId, DataCacheHandle_t hCacheItem);
+	virtual void OnRemove(DataCacheClientID_t clientId);
+
+	CUtlHashFast<DataCacheHandle_t> m_Handles;
+};
+
+
+//-----------------------------------------------------------------------------
+// CDataCache
+//
+// Purpose: The global shared cache. Manages sections and overall budgets.
+//
+//-----------------------------------------------------------------------------
+class CDataCache : public CTier3AppSystem< IDataCache >
+{
+	typedef CTier3AppSystem< IDataCache > BaseClass;
+
+public:
+	CDataCache();
+
+	//--------------------------------------------------------
+	// IAppSystem methods
+	//--------------------------------------------------------
+	virtual bool Connect(CreateInterfaceFn factory);
+	virtual void Disconnect();
+	virtual void* QueryInterface(const char* pInterfaceName);
+	virtual InitReturnVal_t Init();
+	virtual void Shutdown();
+
+	//--------------------------------------------------------
+	// IDataCache methods
+	//--------------------------------------------------------
+
+	virtual void SetSize(int nMaxBytes);
+	virtual void SetOptions(unsigned options);
+	virtual void SetSectionLimits(const char* pszSectionName, const DataCacheLimits_t& limits);
+	virtual void GetStatus(DataCacheStatus_t* pStatus, DataCacheLimits_t* pLimits = NULL);
+
+	//--------------------------------------------------------
+
+	virtual IDataCacheSection* AddSection(IDataCacheClient* pClient, const char* pszSectionName, const DataCacheLimits_t& limits = DataCacheLimits_t(), bool bSupportFastFind = false);
+	virtual void RemoveSection(const char* pszClientName, bool bCallFlush = true);
+	virtual IDataCacheSection* FindSection(const char* pszClientName);
+
+	//--------------------------------------------------------
+
+	void EnsureCapacity(unsigned nBytes);
+	virtual unsigned Purge(unsigned nBytes);
+	virtual unsigned Flush(bool bUnlockedOnly = true, bool bNotify = true);
+
+	//--------------------------------------------------------
+
+	virtual void OutputReport(DataCacheReportType_t reportType = DC_SUMMARY_REPORT, const char* pszSection = NULL);
+
+	//--------------------------------------------------------
+
+	inline unsigned GetNumBytes() { return m_status.nBytes; }
+	inline unsigned GetNumItems() { return m_status.nItems; }
+
+	inline unsigned GetNumBytesLocked() { return m_status.nBytesLocked; }
+	inline unsigned GetNumItemsLocked() { return m_status.nItemsLocked; }
+
+	inline unsigned GetNumBytesUnlocked() { return m_status.nBytes - m_status.nBytesLocked; }
+	inline unsigned GetNumItemsUnlocked() { return m_status.nItems - m_status.nItemsLocked; }
+
+private:
+	//-----------------------------------------------------
+
+	friend class CDataCacheSection;
+
+	//-----------------------------------------------------
+
+	DataCacheItem_t* AccessItem(memhandle_t hCurrent);
+
+	bool IsInFlush() { return m_bInFlush; }
+	int FindSectionIndex(const char* pszSection);
+
+	// Utilities used by the data cache report
+	void OutputItemReport(memhandle_t hItem);
+	static bool SortMemhandlesBySizeLessFunc(const memhandle_t& lhs, const memhandle_t& rhs);
+
+	//-----------------------------------------------------
+
+	CDataCacheLRU					m_LRU;
+	DataCacheStatus_t				m_status;
+	CUtlVector<CDataCacheSection*>	m_Sections;
+	bool							m_bInFlush;
+	CThreadFastMutex& m_mutex;
+};
+
+//---------------------------------------------------------
+
+extern CDataCache g_DataCache;
+
+//-----------------------------------------------------------------------------
+
+inline DataCacheItem_t* CDataCache::AccessItem(memhandle_t hCurrent)
+{
+	return m_LRU.GetResource_NoLockNoLRUTouch(hCurrent);
+}
+
+//-----------------------------------------------------------------------------
+
+inline IDataCache* CDataCacheSection::GetSharedCache()
+{
+	return m_pSharedCache;
+}
+
+inline DataCacheItem_t* CDataCacheSection::AccessItem(memhandle_t hCurrent)
+{
+	return m_pSharedCache->AccessItem(hCurrent);
+}
+
+// Note: if status updates are moved out of a mutexed section, will need to change these to use interlocked instructions
+
+inline void CDataCacheSection::NoteSizeChanged(int oldSize, int newSize)
+{
+	int nBytes = (newSize - oldSize);
+
+	m_status.nBytes += nBytes;
+	m_status.nBytesLocked += nBytes;
+	ThreadInterlockedExchangeAdd(&m_pSharedCache->m_status.nBytes, nBytes);
+	ThreadInterlockedExchangeAdd(&m_pSharedCache->m_status.nBytesLocked, nBytes);
+}
+
+inline void CDataCacheSection::NoteAdd(int size)
+{
+	m_status.nBytes += size;
+	m_status.nItems++;
+
+	ThreadInterlockedExchangeAdd(&m_pSharedCache->m_status.nBytes, size);
+	ThreadInterlockedIncrement(&m_pSharedCache->m_status.nItems);
+}
+
+inline void CDataCacheSection::NoteRemove(int size)
+{
+	m_status.nBytes -= size;
+	m_status.nItems--;
+
+	ThreadInterlockedExchangeAdd(&m_pSharedCache->m_status.nBytes, -size);
+	ThreadInterlockedDecrement(&m_pSharedCache->m_status.nItems);
+}
+
+inline void CDataCacheSection::NoteLock(int size)
+{
+	m_status.nBytesLocked += size;
+	m_status.nItemsLocked++;
+
+	ThreadInterlockedExchangeAdd(&m_pSharedCache->m_status.nBytesLocked, size);
+	ThreadInterlockedIncrement(&m_pSharedCache->m_status.nItemsLocked);
+}
+
+inline void CDataCacheSection::NoteUnlock(int size)
+{
+	m_status.nBytesLocked -= size;
+	m_status.nItemsLocked--;
+
+	ThreadInterlockedExchangeAdd(&m_pSharedCache->m_status.nBytesLocked, -size);
+	ThreadInterlockedDecrement(&m_pSharedCache->m_status.nItemsLocked);
+
+	// something has been unlocked, assume cached pointers are now invalid
+	m_nFrameUnlockCounter++;
+}
+
+//-----------------------------------------------------------------------------
+
+#endif // DATACACHE_H

datacache/datacache.vpc

@@ -0,0 +1,55 @@
+//-----------------------------------------------------------------------------
+//	DATACACHE.VPC
+//
+//	Project Script
+//-----------------------------------------------------------------------------
+
+$macro SRCDIR		".."
+$Macro OUTBINDIR	"$SRCDIR\..\game\bin"
+
+$include "$SRCDIR\vpc_scripts\source_dll_base.vpc"
+
+$Configuration
+{
+	$Linker
+	{
+			$SystemLibraries                                                        "iconv" [$OSXALL]
+	}
+	$Compiler
+	{
+		$PreprocessorDefinitions	"$BASE;MDLCACHE_DLL_EXPORT"
+	}
+}
+
+
+$Project "datacache"
+{
+	$Folder	"Source Files"
+	{
+		$File	"datacache.cpp"
+		$File	"mdlcache.cpp"
+		$File	"$SRCDIR\public\studio.cpp"
+		$File	"$SRCDIR\public\studio_virtualmodel.cpp"
+		$File	"..\common\studiobyteswap.cpp"
+	}
+
+	$Folder	"Header Files"
+	{
+		$File	"datacache.h"
+		$File	"datacache_common.h"
+		$File	"$SRCDIR\public\studio.h"
+		$File	"..\common\studiobyteswap.h"
+	}
+
+	$Folder	"Interface"
+	{
+		$File	"$SRCDIR\public\datacache\idatacache.h"
+		$File	"$SRCDIR\public\datacache\imdlcache.h"
+	}
+
+	$folder	"Link Libraries"
+	{
+		$Lib	tier2
+		$Lib	tier3
+	}
+}

datacache/datacache_common.h

@@ -0,0 +1,31 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose:
+//
+//===========================================================================//
+
+#ifndef DATACACHE_COMMON_H
+#define DATACACHE_COMMON_H
+
+#if defined( _WIN32 )
+#pragma once
+#endif
+
+#include "tier3/tier3.h"
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class IDataCacheSection;
+FORWARD_DECLARE_HANDLE( memhandle_t );
+typedef memhandle_t DataCacheHandle_t;
+
+
+//-----------------------------------------------------------------------------
+// Console commands
+//-----------------------------------------------------------------------------
+extern ConVar developer;
+
+
+#endif // DATACACHE_COMMON_H

datacache/xbox/xbox.def

@@ -0,0 +1,3 @@
+LIBRARY	datacache_360.dll
+EXPORTS
+	CreateInterface		@1

datamodel/DmElementFramework.cpp

@@ -0,0 +1,209 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "DmElementFramework.h"
+#include "datamodel.h"
+
+// memdbgon must be the last include file in a .cpp file!!!
+#include "tier0/memdbgon.h"
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CDmElementFramework g_DmElementFramework;
+CDmElementFramework *g_pDmElementFrameworkImp = &g_DmElementFramework;
+IDmElementFramework *g_pDmElementFramework = &g_DmElementFramework;
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CDmElementFramework::CDmElementFramework() : m_phase( PH_EDIT ), m_dirtyElements( 128, 256 ) 
+{
+}
+
+	
+//-----------------------------------------------------------------------------
+// Methods of IAppSystem
+//-----------------------------------------------------------------------------
+bool CDmElementFramework::Connect( CreateInterfaceFn factory )
+{
+	return true;
+}
+
+void CDmElementFramework::Disconnect()
+{
+}
+
+void *CDmElementFramework::QueryInterface( const char *pInterfaceName )
+{
+	if ( !V_strcmp( pInterfaceName, VDMELEMENTFRAMEWORK_VERSION ) )
+		return (IDmElementFramework*)this;
+
+	return NULL;
+}
+
+InitReturnVal_t CDmElementFramework::Init( )
+{
+	return INIT_OK;
+}
+
+void CDmElementFramework::Shutdown()
+{
+	m_dependencyGraph.Cleanup();
+}
+
+
+//-----------------------------------------------------------------------------
+// element framework phase transition methods
+//-----------------------------------------------------------------------------
+void CDmElementFramework::EditApply()
+{
+	g_pDataModelImp->RemoveUnreferencedElements();
+}
+
+void CDmElementFramework::Resolve( bool clearDirtyFlags )
+{
+	int nCount = m_dirtyElements.Count();
+	for ( int ei = 0; ei < nCount; ++ei )
+	{
+		DmElementHandle_t h = m_dirtyElements[ ei ];
+		CDmElement *pElement = g_pDataModel->GetElement( h );
+		if ( !pElement )
+			continue;
+
+		pElement->Resolve();
+
+		if ( clearDirtyFlags )
+		{
+			CDmeElementAccessor::MarkDirty( pElement, false ); // marks element clean
+			CDmeElementAccessor::MarkAttributesClean( pElement ); // marks all attributes clean
+		}
+	}
+
+	if ( clearDirtyFlags )
+	{
+		m_dirtyElements.RemoveAll();
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Returns the current phase
+//-----------------------------------------------------------------------------
+DmPhase_t CDmElementFramework::GetPhase()
+{
+	return FastGetPhase();
+}
+
+void CDmElementFramework::SetOperators( const CUtlVector< IDmeOperator* > &operators )
+{
+	VPROF( "CDmElementFramework::SetOperators()" );
+	m_dependencyGraph.Reset( operators );
+}
+
+void CDmElementFramework::BeginEdit()
+{
+	Assert( m_phase == PH_EDIT || m_phase == PH_OUTPUT );
+
+	if ( m_phase == PH_EDIT )
+	{
+		m_phase = PH_EDIT_APPLY;
+		EditApply();
+
+		m_phase = PH_EDIT_RESOLVE;
+		Resolve( false );
+	}
+
+	m_phase = PH_EDIT;
+}
+
+void CDmElementFramework::Operate( bool bResolve )
+{
+	VPROF( "CDmElementFramework::Operate" );
+
+	Assert( m_phase == PH_EDIT || m_phase == PH_OUTPUT );
+
+	if ( m_phase == PH_EDIT )
+	{
+		{
+			VPROF( "CDmElementFramework::PH_EDIT_APPLY" );
+			m_phase = PH_EDIT_APPLY;
+			EditApply();
+		}
+
+		{
+			VPROF( "CDmElementFramework::PH_EDIT_RESOLVE" );
+			m_phase = PH_EDIT_RESOLVE;
+			Resolve( false );
+		}
+	}
+
+	{
+		VPROF( "CDmElementFramework::PH_DEPENDENCY" );
+		m_phase = PH_DEPENDENCY;
+		bool cycle = m_dependencyGraph.CullAndSortOperators();
+		if ( cycle )
+		{
+			Warning( "Operator cycle found during dependency graph traversal!\n" );
+		}
+	}
+
+	{
+		VPROF( "CDmElementFramework::PH_OPERATE" );
+		m_phase = PH_OPERATE;
+		const CUtlVector< IDmeOperator* > &operatorsToRun = m_dependencyGraph.GetSortedOperators();
+		uint on = operatorsToRun.Count();
+		for ( uint oi = 0; oi < on; ++oi )
+		{
+			operatorsToRun[ oi ]->Operate();
+		}
+	}
+
+	if ( bResolve )
+	{
+		VPROF( "CDmElementFramework::PH_OPERATE_RESOLVE" );
+		m_phase = PH_OPERATE_RESOLVE;
+		Resolve( true );
+
+		m_phase = PH_OUTPUT;
+	}
+}
+
+void CDmElementFramework::Resolve()
+{
+	VPROF( "CDmElementFramework::Resolve" );
+
+	Assert( m_phase == PH_OPERATE );
+
+	m_phase = PH_OPERATE_RESOLVE;
+	Resolve( true );
+
+	m_phase = PH_OUTPUT;
+}
+
+void CDmElementFramework::AddElementToDirtyList( DmElementHandle_t hElement )
+{
+	m_dirtyElements.AddToTail( hElement );
+}
+
+void CDmElementFramework::RemoveCleanElementsFromDirtyList()
+{
+	int nCount = m_dirtyElements.Count();
+	while ( --nCount >= 0 )
+	{
+		DmElementHandle_t h = m_dirtyElements[ nCount ];
+		CDmElement *pElement = g_pDataModel->GetElement( h );
+		if ( !pElement )
+			continue;
+
+		if ( !CDmeElementAccessor::IsDirty( pElement ) )
+		{
+			m_dirtyElements.FastRemove( nCount );
+		}
+	}
+}

datamodel/DmElementFramework.h

@@ -0,0 +1,78 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef DMELEMENTFRAMEWORK_H
+#define DMELEMENTFRAMEWORK_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "datamodel/idatamodel.h"
+#include "tier1/utlvector.h"
+#include "dependencygraph.h"
+
+
+//-----------------------------------------------------------------------------
+// element framework implementation
+//-----------------------------------------------------------------------------
+class CDmElementFramework : public IDmElementFramework
+{
+public:
+	CDmElementFramework();
+
+public:
+	// Methods of IAppSystem
+	virtual bool Connect( CreateInterfaceFn factory );
+	virtual void Disconnect();
+	virtual void *QueryInterface( const char *pInterfaceName );
+	virtual InitReturnVal_t Init();
+	virtual void Shutdown();
+
+	// Methods of IDmElementFramework
+	virtual DmPhase_t GetPhase();
+	virtual void SetOperators( const CUtlVector< IDmeOperator* > &operators );
+	virtual void BeginEdit(); // ends in edit phase, forces apply/resolve if from edit phase
+	virtual void Operate( bool bResolve ); // ends in output phase
+	virtual void Resolve();
+
+public:
+	// Other public methods
+	void AddElementToDirtyList( DmElementHandle_t hElement );
+	void RemoveCleanElementsFromDirtyList();
+
+	// Non-virtual methods of identical virtual functions
+	DmPhase_t FastGetPhase();
+
+
+private:
+	void EditApply();
+
+	// Invoke the resolve method
+	void Resolve( bool clearDirtyFlags );
+
+	CDependencyGraph m_dependencyGraph;
+	CUtlVector< DmElementHandle_t > m_dirtyElements;
+	DmPhase_t m_phase;
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton
+//-----------------------------------------------------------------------------
+extern CDmElementFramework *g_pDmElementFrameworkImp;
+
+
+//-----------------------------------------------------------------------------
+// Inline methods
+//-----------------------------------------------------------------------------
+inline DmPhase_t CDmElementFramework::FastGetPhase()
+{
+	return m_phase;
+}
+
+
+#endif // DMELEMENTFRAMEWORK_H

datamodel/clipboardmanager.cpp

@@ -0,0 +1,169 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "clipboardmanager.h"
+#include "datamodel.h"
+#include "tier1/KeyValues.h"
+
+#ifndef _LINUX
+#define USE_WINDOWS_CLIPBOARD
+#endif
+
+#if defined( USE_WINDOWS_CLIPBOARD )
+#include <windows.h>
+#endif
+
+CClipboardManager::CClipboardManager( ) :
+	m_pfnCleanup( NULL )
+{
+}
+
+CClipboardManager::~CClipboardManager()
+{
+	EmptyClipboard( false );
+}
+
+void CClipboardManager::EmptyClipboard( bool bClearWindowsClipboard )
+{
+	// Call optional cleanup function if there is one...
+	if ( m_pfnCleanup )
+	{
+		m_pfnCleanup->ReleaseClipboardData( m_Data );
+	}
+	int c = m_Data.Count();
+	for ( int i = 0; i < c; ++i )
+	{
+		m_Data[ i ]->deleteThis();
+	}
+	m_Data.RemoveAll();
+	m_pfnCleanup = NULL;
+
+#if defined( USE_WINDOWS_CLIPBOARD )
+	if ( bClearWindowsClipboard )
+	{
+		if ( ::OpenClipboard( ::GetDesktopWindow() ) )
+		{
+			::EmptyClipboard();
+			::CloseClipboard();
+		}
+	}
+#endif
+}
+
+void CClipboardManager::SetClipboardData( CUtlVector< KeyValues * >& data, IClipboardCleanup *pfnOptionalCleanuFunction )
+{
+	EmptyClipboard( true );
+	m_Data = data;
+	m_pfnCleanup = pfnOptionalCleanuFunction;
+
+#if defined( USE_WINDOWS_CLIPBOARD )
+	if ( m_Data.Count() >= 0 )
+	{
+		// Only stick the first item's data into the clipboard
+		char const *text = m_Data[ 0 ]->GetString( "text", "" );
+		if ( text && text[ 0 ] )
+		{
+			int textLen = Q_strlen( text );
+
+			if ( ::OpenClipboard( ::GetDesktopWindow() ) )
+			{
+				HANDLE hmem = ::GlobalAlloc(GMEM_MOVEABLE, textLen + 1);
+				if (hmem)
+				{
+					void *ptr = ::GlobalLock( hmem );
+					if ( ptr  )
+					{
+						Q_memset( ptr, 0, textLen + 1 );
+						Q_memcpy( ptr, text, textLen );
+						::GlobalUnlock( hmem );
+
+						::SetClipboardData( CF_TEXT, hmem );
+					}
+				}
+				::CloseClipboard();
+			}
+		}
+	}
+#endif
+}
+
+void CClipboardManager::AddToClipboardData( KeyValues *add )
+{
+	m_Data.AddToTail( add );
+#if defined( USE_WINDOWS_CLIPBOARD )
+	if ( m_Data.Count() >= 0 )
+	{
+		// Only stick the first item's data into the clipboard
+		char const *text = m_Data[ 0 ]->GetString( "text", "" );
+		if ( text && text[ 0 ] )
+		{
+			int textLen = Q_strlen( text );
+
+
+			if ( ::OpenClipboard( ::GetDesktopWindow() ) )
+			{
+				::EmptyClipboard();
+
+				HANDLE hmem = ::GlobalAlloc(GMEM_MOVEABLE, textLen + 1);
+				if (hmem)
+				{
+					void *ptr = ::GlobalLock( hmem );
+					if ( ptr  )
+					{
+						Q_memset( ptr, 0, textLen + 1 );
+						Q_memcpy( ptr, text, textLen );
+						::GlobalUnlock( hmem );
+
+						::SetClipboardData( CF_TEXT, hmem );
+					}
+				}
+				::CloseClipboard();
+			}
+		}
+	}
+#endif
+}
+
+void CClipboardManager::GetClipboardData( CUtlVector< KeyValues * >& data )
+{
+	data.RemoveAll();
+	data = m_Data;
+#if defined( USE_WINDOWS_CLIPBOARD )
+	if ( data.Count() == 0 )
+	{
+		// See if windows has some text since we didn't have any internally
+		if ( ::OpenClipboard( ::GetDesktopWindow() ) )
+		{
+			HANDLE hmem = ::GetClipboardData( CF_TEXT );
+			if ( hmem )
+			{
+				int len = GlobalSize( hmem );
+				if ( len > 0 )
+				{
+					void *ptr = GlobalLock(hmem);
+					if ( ptr )
+					{
+						char buf[ 8192 ];
+						len = min( len, 8191 );
+						Q_memcpy( buf, ( char * )ptr, len );
+						buf[ 8191 ] = 0;
+						GlobalUnlock(hmem);
+
+						KeyValues *newData = new KeyValues( "ClipBoard", "text", buf );
+						data.AddToTail( newData );
+					}
+				}
+			}
+			::CloseClipboard();
+		}
+	}
+#endif
+}
+
+bool CClipboardManager::HasClipboardData() const
+{
+	return m_Data.Count() > 0 ? true : false;
+}

datamodel/clipboardmanager.h

@@ -0,0 +1,36 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef CLIPBOARDMANAGER_H
+#define CLIPBOARDMANAGER_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "tier1/utlvector.h"
+
+class KeyValues;
+class IClipboardCleanup;
+
+// Clipboard:
+//
+class CClipboardManager
+{
+public:
+	CClipboardManager();
+	~CClipboardManager();
+
+	void			EmptyClipboard( bool bClearWindowsClipboard );
+	void			SetClipboardData( CUtlVector< KeyValues * >& data, IClipboardCleanup *pfnOptionalCleanuFunction );
+	void			AddToClipboardData( KeyValues *add );
+	void			GetClipboardData( CUtlVector< KeyValues * >& data );
+	bool			HasClipboardData() const;
+private:
+	CUtlVector< KeyValues * >	m_Data;
+	IClipboardCleanup			*m_pfnCleanup;
+};
+
+#endif // CLIPBOARDMANAGER_H

datamodel/datamodel.h

@@ -0,0 +1,527 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef DATAMODEL_H
+#define DATAMODEL_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "datamodel/dmattribute.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "datamodel/dmehandle.h"
+#include "tier1/uniqueid.h"
+#include "tier1/utlsymbol.h"
+#include "tier1/utllinkedlist.h"
+#include "tier1/utldict.h"
+#include "tier1/utlstring.h"
+#include "tier1/utlhandletable.h"
+#include "tier1/utlhash.h"
+#include "tier2/tier2.h"
+#include "clipboardmanager.h"
+#include "undomanager.h"
+#include "tier1/convar.h"
+#include "tier0/vprof.h" 
+
+
+//-----------------------------------------------------------------------------
+// forward declarations
+//-----------------------------------------------------------------------------
+class IDmElementFramework;
+class IUndoElement;
+class CDmElement;
+
+enum DmHandleReleasePolicy
+{
+	HR_ALWAYS,
+	HR_NEVER,
+	HR_IF_NOT_REFERENCED,
+};
+
+
+//-----------------------------------------------------------------------------
+// memory categories
+//-----------------------------------------------------------------------------
+enum
+{
+	MEMORY_CATEGORY_OUTER,
+	MEMORY_CATEGORY_ELEMENT_INTERNAL,
+	MEMORY_CATEGORY_DATAMODEL,
+	MEMORY_CATEGORY_REFERENCES,
+	MEMORY_CATEGORY_ATTRIBUTE_TREE,
+	MEMORY_CATEGORY_ATTRIBUTE_OVERHEAD,
+	MEMORY_CATEGORY_ATTRIBUTE_DATA,
+	MEMORY_CATEGORY_ATTRIBUTE_COUNT,
+
+	MEMORY_CATEGORY_COUNT,
+};
+
+
+//-----------------------------------------------------------------------------
+// hash map of id->element, with the id storage optimized out
+//-----------------------------------------------------------------------------
+class CElementIdHash : public CUtlHash< DmElementHandle_t >
+{
+public:
+	CElementIdHash( int nBucketCount = 0, int nGrowCount = 0, int nInitCount = 0 )
+		: CUtlHash< DmElementHandle_t >( nBucketCount, nGrowCount, nInitCount, CompareFunc, KeyFunc )
+	{
+	}
+
+protected:
+	typedef CUtlHash< DmElementHandle_t > BaseClass;
+
+	static bool CompareFunc( DmElementHandle_t const& a, DmElementHandle_t const& b ) { return a == b; }
+	static bool IdCompareFunc( DmElementHandle_t const& hElement, DmObjectId_t const& id )
+	{
+		CDmElement *pElement = g_pDataModel->GetElement( hElement );
+		Assert( pElement );
+		if ( !pElement )
+			return false;
+
+		return IsUniqueIdEqual( id, pElement->GetId() );
+	}
+
+	static unsigned int KeyFunc( DmElementHandle_t const& hElement )
+	{
+		CDmElement *pElement = g_pDataModel->GetElement( hElement );
+		Assert( pElement );
+		if ( !pElement )
+			return 0;
+
+		return *( unsigned int* )&pElement->GetId();
+	}
+	static unsigned int IdKeyFunc( DmObjectId_t const &src )
+	{
+		return *(unsigned int*)&src;
+	}
+
+protected:
+	bool DoFind( DmObjectId_t const &src, unsigned int *pBucket, int *pIndex )
+	{
+		// generate the data "key"
+		unsigned int key = IdKeyFunc( src );
+
+		// hash the "key" - get the correct hash table "bucket"
+		unsigned int ndxBucket;
+		if( m_bPowerOfTwo )
+		{
+			*pBucket = ndxBucket = ( key & m_ModMask );
+		}
+		else
+		{
+			int bucketCount = m_Buckets.Count();
+			*pBucket = ndxBucket = key % bucketCount;
+		}
+
+		int ndxKeyData;
+		CUtlVector< DmElementHandle_t > &bucket = m_Buckets[ndxBucket];
+		int keyDataCount = bucket.Count();
+		for( ndxKeyData = 0; ndxKeyData < keyDataCount; ndxKeyData++ )
+		{
+			if( IdCompareFunc( bucket.Element( ndxKeyData ), src ) )
+				break;
+		}
+
+		if( ndxKeyData == keyDataCount )
+			return false;
+
+		*pIndex = ndxKeyData;
+		return true;
+	}
+
+public:
+	UtlHashHandle_t Find( DmElementHandle_t const &src ) { return BaseClass::Find( src ); }
+	UtlHashHandle_t Find( DmObjectId_t const &src )
+	{
+		unsigned int ndxBucket;
+		int ndxKeyData;
+
+		if ( DoFind( src, &ndxBucket, &ndxKeyData ) )
+			return BuildHandle( ndxBucket, ndxKeyData );
+
+		return InvalidHandle();
+	}
+};
+
+//-----------------------------------------------------------------------------
+// struct to hold the set of elements in any given file
+//-----------------------------------------------------------------------------
+
+struct FileElementSet_t
+{
+	FileElementSet_t( UtlSymId_t filename = UTL_INVAL_SYMBOL, UtlSymId_t format = UTL_INVAL_SYMBOL ) :
+		m_filename( filename ), m_format( format ),
+		m_hRoot( DMELEMENT_HANDLE_INVALID ),
+		m_bLoaded( true ),
+		m_nElements( 0 )
+	{
+	}
+	FileElementSet_t( const FileElementSet_t& that ) : m_filename( that.m_filename ), m_format( that.m_format ), m_hRoot( DMELEMENT_HANDLE_INVALID ), m_bLoaded( that.m_bLoaded ), m_nElements( that.m_nElements )
+	{
+		// the only time this should be copy constructed is when passing in an empty set to the parent array
+		// otherwise it could get prohibitively expensive time and memory wise
+		Assert( that.m_nElements == 0 );
+	}
+
+	UtlSymId_t m_filename;
+	UtlSymId_t m_format;
+	CDmeCountedHandle m_hRoot;
+	bool m_bLoaded;
+	int m_nElements;
+};
+
+
+//-----------------------------------------------------------------------------
+// Purpose: Versionable factor for element types
+//-----------------------------------------------------------------------------
+class CDataModel : public CBaseAppSystem< IDataModel >
+{
+	typedef CBaseAppSystem< IDataModel > BaseClass;
+
+public:
+	CDataModel();
+	virtual ~CDataModel();
+
+// External interface
+public:
+	// Methods of IAppSystem
+	virtual bool Connect( CreateInterfaceFn factory );
+	virtual void *QueryInterface( const char *pInterfaceName );
+	virtual InitReturnVal_t Init();
+	virtual void Shutdown();
+
+	// Methods of IDataModel
+	virtual void				AddElementFactory( const char *pClassName, IDmElementFactory *pFactory );
+	virtual bool				HasElementFactory( const char *pElementType ) const;
+	virtual void				SetDefaultElementFactory( IDmElementFactory *pFactory );
+	virtual int					GetFirstFactory() const;
+	virtual int					GetNextFactory( int index ) const;
+	virtual bool				IsValidFactory( int index ) const;
+	virtual char const			*GetFactoryName( int index ) const;
+	virtual DmElementHandle_t	CreateElement( UtlSymId_t typeSymbol, const char *pElementName, DmFileId_t fileid, const DmObjectId_t *pObjectID = NULL );
+	virtual DmElementHandle_t	CreateElement( const char *pTypeName, const char *pElementName, DmFileId_t fileid, const DmObjectId_t *pObjectID = NULL );
+	virtual void				DestroyElement( DmElementHandle_t hElement );
+	virtual	CDmElement*		GetElement( DmElementHandle_t hElement ) const;
+	virtual UtlSymId_t			GetElementType( DmElementHandle_t hElement ) const;
+	virtual const char*			GetElementName( DmElementHandle_t hElement ) const;
+	virtual const DmObjectId_t&	GetElementId( DmElementHandle_t hElement ) const;
+	virtual const char			*GetAttributeNameForType( DmAttributeType_t attType ) const;
+	virtual DmAttributeType_t	GetAttributeTypeForName( const char *name ) const;
+
+	virtual void				AddSerializer( IDmSerializer *pSerializer );
+	virtual void				AddLegacyUpdater( IDmLegacyUpdater *pUpdater );
+	virtual void				AddFormatUpdater( IDmFormatUpdater *pUpdater );
+	virtual const char*			GetFormatExtension( const char *pFormatName );
+	virtual const char*			GetFormatDescription( const char *pFormatName );
+	virtual int					GetFormatCount() const;
+	virtual const char *		GetFormatName( int i ) const;
+	virtual const char *		GetDefaultEncoding( const char *pFormatName );
+	virtual int					GetEncodingCount() const;
+	virtual const char *		GetEncodingName( int i ) const;
+	virtual bool				IsEncodingBinary( const char *pEncodingName ) const;
+	virtual bool				DoesEncodingStoreVersionInFile( const char *pEncodingName ) const;
+
+	virtual void				SetSerializationDelimiter( CUtlCharConversion *pConv );
+	virtual void				SetSerializationArrayDelimiter( const char *pDelimiter );
+	virtual bool				IsUnserializing();
+	virtual bool				Serialize( CUtlBuffer &outBuf, const char *pEncodingName, const char *pFormatName, DmElementHandle_t hRoot );
+	virtual bool				Unserialize( CUtlBuffer &buf, const char *pEncodingName, const char *pSourceFormatName, const char *pFormatHint,
+											 const char *pFileName, DmConflictResolution_t idConflictResolution, DmElementHandle_t &hRoot );
+	virtual bool				UpdateUnserializedElements( const char *pSourceFormatName, int nSourceFormatVersion,
+															DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot );
+	virtual IDmSerializer*		FindSerializer( const char *pEncodingName ) const;
+	virtual IDmLegacyUpdater*	FindLegacyUpdater( const char *pLegacyFormatName ) const;
+	virtual IDmFormatUpdater*	FindFormatUpdater( const char *pFormatName ) const;
+	virtual bool				SaveToFile( char const *pFileName, char const *pPathID, const char *pEncodingName, const char *pFormatName, CDmElement *pRoot );
+	virtual DmFileId_t			RestoreFromFile( char const *pFileName, char const *pPathID, const char *pFormatHint, CDmElement **ppRoot, DmConflictResolution_t idConflictResolution = CR_DELETE_NEW, DmxHeader_t *pHeaderOut = NULL );
+
+	virtual void				SetKeyValuesElementCallback( IElementForKeyValueCallback *pCallbackInterface );
+	virtual const char *		GetKeyValuesElementName( const char *pszKeyName, int iNestingLevel );
+	virtual UtlSymId_t			GetSymbol( const char *pString );
+	virtual const char *		GetString( UtlSymId_t sym ) const;
+	virtual int					GetElementsAllocatedSoFar();
+	virtual int					GetMaxNumberOfElements();
+	virtual int					GetAllocatedAttributeCount();
+	virtual int					GetAllocatedElementCount();
+	virtual DmElementHandle_t	FirstAllocatedElement();
+	virtual DmElementHandle_t	NextAllocatedElement( DmElementHandle_t hElement );
+	virtual int					EstimateMemoryUsage( DmElementHandle_t hElement, TraversalDepth_t depth = TD_DEEP );
+	virtual void				SetUndoEnabled( bool enable );
+	virtual bool				IsUndoEnabled() const;
+	virtual bool				UndoEnabledForElement( const CDmElement *pElement ) const;
+	virtual bool				IsDirty() const;
+	virtual bool				CanUndo() const;
+	virtual bool				CanRedo() const;
+	virtual void				StartUndo( const char *undodesc, const char *redodesc, int nChainingID = 0 );
+	virtual void				FinishUndo();
+	virtual void				AbortUndoableOperation();
+	virtual void				ClearRedo();
+	virtual const char			*GetUndoDesc();
+	virtual const char			*GetRedoDesc();
+	virtual void				Undo();
+	virtual void				Redo();
+	virtual void				TraceUndo( bool state ); // if true, undo records spew as they are added
+	virtual void				ClearUndo();
+	virtual void				GetUndoInfo( CUtlVector< UndoInfo_t >& list );
+	virtual const char *		GetUndoString( UtlSymId_t sym );
+	virtual void				AddUndoElement( IUndoElement *pElement );
+	virtual UtlSymId_t			GetUndoDescInternal( const char *context );
+	virtual UtlSymId_t			GetRedoDescInternal( const char *context );
+	virtual void				EmptyClipboard();
+	virtual void				SetClipboardData( CUtlVector< KeyValues * >& data, IClipboardCleanup *pfnOptionalCleanuFunction = 0 );
+	virtual void				AddToClipboardData( KeyValues *add );
+	virtual void				GetClipboardData( CUtlVector< KeyValues * >& data );
+	virtual bool				HasClipboardData() const;
+
+ 	virtual CDmAttribute *		GetAttribute( DmAttributeHandle_t h );
+	virtual bool				IsAttributeHandleValid( DmAttributeHandle_t h ) const;
+	virtual void				OnlyCreateUntypedElements( bool bEnable );
+	virtual int					NumFileIds();
+	virtual DmFileId_t			GetFileId( int i );
+	virtual DmFileId_t			FindOrCreateFileId( const char *pFilename );
+	virtual void				RemoveFileId( DmFileId_t fileid );
+	virtual DmFileId_t			GetFileId( const char *pFilename );
+	virtual const char *		GetFileName( DmFileId_t fileid );
+	virtual void				SetFileName( DmFileId_t fileid, const char *pFileName );
+	virtual const char *		GetFileFormat( DmFileId_t fileid );
+	virtual void				SetFileFormat( DmFileId_t fileid, const char *pFormat );
+	virtual DmElementHandle_t	GetFileRoot( DmFileId_t fileid );
+	virtual void				SetFileRoot( DmFileId_t fileid, DmElementHandle_t hRoot );
+	virtual bool				IsFileLoaded( DmFileId_t fileid );
+	virtual void				MarkFileLoaded( DmFileId_t fileid );
+	virtual void				UnloadFile( DmFileId_t fileid );
+	virtual int					NumElementsInFile( DmFileId_t fileid );
+	virtual void				DontAutoDelete( DmElementHandle_t hElement );
+	virtual void				MarkHandleInvalid( DmElementHandle_t hElement );
+	virtual void				MarkHandleValid( DmElementHandle_t hElement );
+	virtual DmElementHandle_t	FindElement( const DmObjectId_t &id );
+	virtual	DmAttributeReferenceIterator_t	FirstAttributeReferencingElement( DmElementHandle_t hElement );
+	virtual DmAttributeReferenceIterator_t	NextAttributeReferencingElement( DmAttributeReferenceIterator_t hAttrIter );
+	virtual CDmAttribute *					GetAttribute( DmAttributeReferenceIterator_t hAttrIter );
+	virtual bool				InstallNotificationCallback( IDmNotify *pNotify );
+	virtual void				RemoveNotificationCallback( IDmNotify *pNotify );
+	virtual bool				IsSuppressingNotify( ) const;
+	virtual void				SetSuppressingNotify( bool bSuppress );
+	virtual void				PushNotificationScope( const char *pReason, int nNotifySource, int nNotifyFlags );
+	virtual void				PopNotificationScope( bool bAbort );
+	virtual void				SetUndoDepth( int nSize );
+	virtual void				DisplayMemoryStats();
+
+public:
+	// Internal public methods
+	int GetCurrentFormatVersion( const char *pFormatName );
+
+	// CreateElement references the attribute list passed in via ref, so don't edit or purge ref's attribute list afterwards
+	CDmElement* CreateElement( const DmElementReference_t &ref, const char *pElementType, const char *pElementName, DmFileId_t fileid, const DmObjectId_t *pObjectID );
+	void DeleteElement( DmElementHandle_t hElement, DmHandleReleasePolicy hrp = HR_ALWAYS );
+
+	// element handle related methods
+	DmElementHandle_t AcquireElementHandle();
+	void ReleaseElementHandle( DmElementHandle_t hElement );
+
+	// Handles to attributes
+	DmAttributeHandle_t AcquireAttributeHandle( CDmAttribute *pAttribute );
+	void ReleaseAttributeHandle( DmAttributeHandle_t hAttribute );
+
+	// remove orphaned element subtrees
+	void FindAndDeleteOrphanedElements();
+
+	// Event "mailing list"
+	DmMailingList_t CreateMailingList();
+	void DestroyMailingList( DmMailingList_t list );
+	void AddElementToMailingList( DmMailingList_t list, DmElementHandle_t h );
+
+	// Returns false if the mailing list is empty now
+	bool RemoveElementFromMailingList( DmMailingList_t list, DmElementHandle_t h );
+
+	// Returns false if the mailing list is empty now (can happen owing to stale attributes)
+	bool PostAttributeChanged( DmMailingList_t list, CDmAttribute *pAttribute );
+
+	void GetInvalidHandles( CUtlVector< DmElementHandle_t > &handles );
+	void MarkHandlesValid( CUtlVector< DmElementHandle_t > &handles );
+	void MarkHandlesInvalid( CUtlVector< DmElementHandle_t > &handles );
+
+	// search id->handle table (both loaded and unloaded) for id, and if not found, create a new handle, map it to the id and return it
+	DmElementHandle_t FindOrCreateElementHandle( const DmObjectId_t &id );
+
+	// changes an element's id and associated mappings - generally during unserialization
+	DmElementHandle_t ChangeElementId( DmElementHandle_t hElement, const DmObjectId_t &oldId, const DmObjectId_t &newId );
+
+	DmElementReference_t *FindElementReference( DmElementHandle_t hElement, DmObjectId_t **ppId = NULL );
+
+	void RemoveUnreferencedElements();
+
+	void RemoveElementFromFile( DmElementHandle_t hElement, DmFileId_t fileid );
+	void AddElementToFile( DmElementHandle_t hElement, DmFileId_t fileid );
+
+	void NotifyState( int nNotifyFlags );
+
+	int EstimateMemoryOverhead() const;
+
+	bool IsCreatingUntypedElements() const { return m_bOnlyCreateUntypedElements; }
+
+	unsigned short GetSymbolCount() const;
+
+private:
+	struct MailingList_t
+	{
+		CUtlVector<DmElementHandle_t> m_Elements;
+	};
+
+	struct ElementIdHandlePair_t
+	{
+		DmObjectId_t m_id;
+		DmElementReference_t m_ref;
+		ElementIdHandlePair_t() {}
+		explicit ElementIdHandlePair_t( const DmObjectId_t &id ) : m_ref()
+		{
+			CopyUniqueId( id, &m_id );
+		}
+		ElementIdHandlePair_t( const DmObjectId_t &id, const DmElementReference_t &ref ) : m_ref( ref )
+		{
+			CopyUniqueId( id, &m_id );
+		}
+		ElementIdHandlePair_t( const ElementIdHandlePair_t& that ) : m_ref( that.m_ref )
+		{
+			CopyUniqueId( that.m_id, &m_id );
+		}
+		ElementIdHandlePair_t &operator=( const ElementIdHandlePair_t &that )
+		{
+			CopyUniqueId( that.m_id, &m_id );
+			m_ref = that.m_ref;
+			return *this;
+		}
+		static unsigned int HashKey( const ElementIdHandlePair_t& that )
+		{
+			return *( unsigned int* )&that.m_id.m_Value;
+		}
+		static bool Compare( const ElementIdHandlePair_t& a, const ElementIdHandlePair_t& b )
+		{
+			return IsUniqueIdEqual( a.m_id, b.m_id );
+		}
+	};
+
+private:
+	CDmElement *Unserialize( CUtlBuffer& buf );
+	void Serialize( CDmElement *element, CUtlBuffer& buf );
+
+	// Read the header, return the version (or false if it's not a DMX file)
+	bool ReadDMXHeader( CUtlBuffer &inBuf, DmxHeader_t *pHeader ) const;
+	const char *GetEncodingFromLegacyFormat( const char *pLegacyFormatName ) const;
+	bool IsValidNonDMXFormat( const char *pFormatName ) const;
+	bool IsLegacyFormat( const char *pFormatName ) const;
+
+	// Returns the current undo manager
+	CUndoManager* GetUndoMgr();
+	const CUndoManager* GetUndoMgr() const;
+	CClipboardManager *GetClipboardMgr();
+	const CClipboardManager *GetClipboardMgr() const;
+
+	void UnloadFile( DmFileId_t fileid, bool bDeleteElements );
+
+	friend class CDmeElementRefHelper;
+	friend class CDmAttribute;
+	template< class T > friend class CDmArrayAttributeOp;
+
+	void OnElementReferenceAdded  ( DmElementHandle_t hElement, CDmAttribute *pAttribute );
+	void OnElementReferenceRemoved( DmElementHandle_t hElement, CDmAttribute *pAttribute );
+	void OnElementReferenceAdded  ( DmElementHandle_t hElement, bool bRefCount );
+	void OnElementReferenceRemoved( DmElementHandle_t hElement, bool bRefCount );
+						
+private:
+	CUtlVector< IDmSerializer* >		m_Serializers;
+	CUtlVector< IDmLegacyUpdater* >		m_LegacyUpdaters;
+	CUtlVector< IDmFormatUpdater* >		m_FormatUpdaters;
+
+	IDmElementFactory *m_pDefaultFactory;
+	CUtlDict< IDmElementFactory*, int >	m_Factories;
+	CUtlSymbolTable m_SymbolTable;
+	CUtlHandleTable< CDmElement, 20 > m_Handles;
+	CUtlHandleTable< CDmAttribute, 20 > m_AttributeHandles;
+	CUndoManager m_UndoMgr;
+	CUtlLinkedList< MailingList_t, DmMailingList_t > m_MailingLists;
+
+	bool m_bIsUnserializing : 1;
+	bool m_bUnableToSetDefaultFactory : 1;
+	bool m_bOnlyCreateUntypedElements : 1;
+	bool m_bUnableToCreateOnlyUntypedElements : 1;
+	bool m_bDeleteOrphanedElements : 1;
+
+	CUtlHandleTable< FileElementSet_t, 20 > m_openFiles;
+
+	CElementIdHash m_elementIds;
+	CUtlHash< ElementIdHandlePair_t > m_unloadedIdElementMap;
+	CUtlVector< DmObjectId_t > m_unreferencedElementIds;
+	CUtlVector< DmElementHandle_t > m_unreferencedElementHandles;
+
+	CClipboardManager m_ClipboardMgr;
+	IElementForKeyValueCallback *m_pKeyvaluesCallbackInterface;
+
+	int m_nElementsAllocatedSoFar;
+	int m_nMaxNumberOfElements;
+};
+
+//-----------------------------------------------------------------------------
+// Singleton
+//-----------------------------------------------------------------------------
+extern CDataModel *g_pDataModelImp;
+
+
+//-----------------------------------------------------------------------------
+// Inline methods
+//-----------------------------------------------------------------------------
+inline CUndoManager* CDataModel::GetUndoMgr()
+{
+	return &m_UndoMgr;
+}
+
+inline const CUndoManager* CDataModel::GetUndoMgr() const
+{
+	return &m_UndoMgr;
+}
+
+inline void CDataModel::NotifyState( int nNotifyFlags )
+{
+	GetUndoMgr()->NotifyState( nNotifyFlags );
+}
+
+inline CClipboardManager *CDataModel::GetClipboardMgr()
+{
+	return &m_ClipboardMgr;
+}
+
+inline const CClipboardManager *CDataModel::GetClipboardMgr() const
+{
+	return &m_ClipboardMgr;
+}
+
+
+//-----------------------------------------------------------------------------
+// Methods of DmElement which are public to datamodel
+//-----------------------------------------------------------------------------
+class CDmeElementAccessor
+{
+public:
+	static void Purge( CDmElement *pElement )													{ pElement->Purge(); }
+	static void SetId( CDmElement *pElement, const DmObjectId_t &id )							{ pElement->SetId( id ); }
+	static bool IsDirty( const CDmElement *pElement )											{ return pElement->IsDirty(); }
+	static void MarkDirty( CDmElement *pElement, bool dirty = true )							{ pElement->MarkDirty( dirty ); }
+	static void MarkAttributesClean( CDmElement *pElement )									{ pElement->MarkAttributesClean(); }
+	static void MarkBeingUnserialized( CDmElement *pElement, bool beingUnserialized = true )	{ pElement->MarkBeingUnserialized( beingUnserialized ); }
+	static bool IsBeingUnserialized( const CDmElement *pElement ) 								{ return pElement->IsBeingUnserialized(); }
+	static void AddAttributeByPtr( CDmElement *pElement, CDmAttribute *ptr )					{ pElement->AddAttributeByPtr( ptr ); }
+	static void RemoveAttributeByPtrNoDelete( CDmElement *pElement, CDmAttribute *ptr )		{ pElement->RemoveAttributeByPtrNoDelete( ptr); }
+	static void ChangeHandle( CDmElement *pElement, DmElementHandle_t handle )					{ pElement->ChangeHandle( handle ); }
+	static DmElementReference_t	*GetReference( CDmElement *pElement )							{ return pElement->GetReference(); }
+	static void SetReference( CDmElement *pElement, const DmElementReference_t &ref )			{ pElement->SetReference( ref ); }
+	static int EstimateMemoryUsage( CDmElement *pElement, CUtlHash< DmElementHandle_t > &visited, TraversalDepth_t depth, int *pCategories ) { return pElement->EstimateMemoryUsage( visited, depth, pCategories ); }
+	static void PerformConstruction( CDmElement *pElement )									{ pElement->PerformConstruction(); }
+	static void PerformDestruction( CDmElement *pElement )										{ pElement->PerformDestruction(); }
+};
+
+#endif // DATAMODEL_H

datamodel/datamodel.vpc

@@ -0,0 +1,76 @@
+//-----------------------------------------------------------------------------
+//	DATAMODEL.VPC
+//
+//	Project Script
+//-----------------------------------------------------------------------------
+
+$Macro SRCDIR		".."
+
+$include "$SRCDIR\vpc_scripts\source_lib_base.vpc"
+
+$Configuration
+{
+	$Compiler
+	{
+		$PreprocessorDefinitions			"$BASE;DATAMODEL_LIB"
+		$PrecompiledHeaderFile				"$(IntDir)/datamodel.pch"
+	}
+}
+
+$Project "Datamodel"
+{
+	$Folder	"Source Files"
+	{
+		$File	"clipboardmanager.cpp"
+		$File	"datamodel.cpp"
+		$File	"dependencygraph.cpp"
+		$File	"dmattribute.cpp"
+		$File	"dmelement.cpp"
+		$File	"dmelementdictionary.cpp"
+		$File	"dmelementfactoryhelper.cpp"
+		$File	"DmElementFramework.cpp"
+		$File	"dmserializerbinary.cpp"
+		$File	"dmserializerkeyvalues.cpp"
+		$File	"dmserializerkeyvalues2.cpp"
+		$File	"undomanager.cpp"
+	}
+
+	$Folder	"Header Files"
+	{
+		$File	"clipboardmanager.h"
+		$File	"datamodel.h"
+		$File	"dependencygraph.h"
+		$File	"dmattributeinternal.h"
+		$File	"dmelementdictionary.h"
+		$File	"$SRCDIR\public\datamodel\dmelementfactoryhelper.h"
+		$File	"DmElementFramework.h"
+		$File	"$SRCDIR\public\datamodel\dmelementhandle.h"
+		$File	"dmserializerbinary.h"
+		$File	"dmserializerkeyvalues.h"
+		$File	"dmserializerkeyvalues2.h"
+		$File	"undomanager.h"
+	}
+
+	$Folder	"external"
+	{
+		$File	"$SRCDIR\public\tier0\basetypes.h"
+		$File	"$SRCDIR\public\tier0\commonmacros.h"
+		$File	"$SRCDIR\public\tier0\dbg.h"
+		$File	"$SRCDIR\public\tier0\fasttimer.h"
+		$File	"$SRCDIR\public\appframework\IAppSystem.h"
+		$File	"$SRCDIR\public\tier1\interface.h"
+		$File	"$SRCDIR\public\tier0\platform.h"
+		$File	"$SRCDIR\public\tier0\protected_things.h"
+		$File	"$SRCDIR\public\string_t.h"
+	}
+
+	$Folder	"Interface"
+	{
+		$File	"$SRCDIR\public\datamodel\attributeflags.h"
+		$File	"$SRCDIR\public\datamodel\dmattributetypes.h"
+		$File	"$SRCDIR\public\datamodel\dmattributevar.h"
+		$File	"$SRCDIR\public\datamodel\dmelement.h"
+		$File	"$SRCDIR\public\datamodel\dmehandle.h"
+		$File	"$SRCDIR\public\datamodel\idatamodel.h"
+	}
+}

datamodel/dependencygraph.cpp

@@ -0,0 +1,331 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dependencygraph.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "mathlib/mathlib.h" // for swap
+
+#include "datamodel/dmattribute.h"
+#include "datamodel/dmattributevar.h"
+
+#include "tier1/mempool.h"
+
+#include "tier0/vprof.h"
+
+// memdbgon must be the last include file in a .cpp file!!!
+#include "tier0/memdbgon.h"
+//-----------------------------------------------------------------------------
+// Misc helper enums and classes for CDependencyGraph class
+//-----------------------------------------------------------------------------
+enum TraversalState_t
+{
+	TS_NOT_VISITED,
+	TS_VISITING,
+	TS_VISITED,
+};
+
+struct COperatorNode
+{
+	COperatorNode( IDmeOperator *pOp = NULL ) :
+		m_state( TS_NOT_VISITED ), 
+		m_operator( pOp ),
+		m_bInList( false )
+	{
+	}
+
+	TraversalState_t m_state;
+	IDmeOperator *m_operator;
+	CUtlVector< CAttributeNode * > m_OutputAttributes;
+	bool			m_bInList;
+};
+
+class CAttributeNode
+{
+public:
+	CAttributeNode( CDmAttribute *attribute = NULL ) : 
+		m_attribute( attribute ),
+		m_bIsOutputToOperator( false )
+	{
+	}
+
+	CDmAttribute *m_attribute;
+	CUtlVector< COperatorNode * > m_InputDependentOperators;
+	bool		m_bIsOutputToOperator;
+};
+
+CClassMemoryPool< CAttributeNode >	g_AttrNodePool( 1000 );
+CClassMemoryPool< COperatorNode >	g_OperatorNodePool( 1000 );
+
+bool HashEntryCompareFunc( CAttributeNode *const& lhs, CAttributeNode *const& rhs )
+{
+	return lhs->m_attribute == rhs->m_attribute;
+}
+
+uint HashEntryKeyFunc( CAttributeNode *const& keyinfo )
+{
+	uint i = (uint)keyinfo->m_attribute;
+	return i >> 2; // since memory is allocated on a 4-byte (at least!) boundary
+}
+
+//-----------------------------------------------------------------------------
+// CDependencyGraph constructor - builds dependency graph from operators
+//-----------------------------------------------------------------------------
+CDependencyGraph::CDependencyGraph() :
+	m_attrNodes( 4096, 0, 0, HashEntryCompareFunc, HashEntryKeyFunc )
+{
+}
+
+void CDependencyGraph::Reset( const CUtlVector< IDmeOperator * > &operators )
+{
+	VPROF_BUDGET( "CDependencyGraph::Reset", VPROF_BUDGETGROUP_TOOLS );
+
+	Cleanup();
+
+	CUtlVector< CDmAttribute * > attrs; // moved outside the loop to function as a temporary memory pool for performance
+	int on = operators.Count();
+	CUtlRBTree< IDmeOperator * > operatorDict( 0, on * 2, DefLessFunc(IDmeOperator *) );
+	for ( int i = 0; i < on; ++i )
+	{
+		operatorDict.Insert( operators[i] );
+	}
+
+	m_opNodes.EnsureCapacity( on );
+	for ( int oi = 0; oi < on; ++oi )
+	{
+		IDmeOperator *pOp = operators[ oi ];
+		Assert( pOp );
+		if ( pOp == NULL )
+			continue;
+
+		COperatorNode *pOpNode = g_OperatorNodePool.Alloc();
+		pOpNode->m_operator = pOp;
+
+		attrs.RemoveAll();
+		pOp->GetInputAttributes( attrs );
+		int an = attrs.Count();
+		for ( int ai = 0; ai < an; ++ai )
+		{
+			CAttributeNode *pAttrNode = FindAttrNode( attrs[ ai ] );
+			pAttrNode->m_InputDependentOperators.AddToTail( pOpNode );
+		}
+
+		attrs.RemoveAll();
+		pOp->GetOutputAttributes( attrs );
+		an = attrs.Count();
+		for ( int ai = 0; ai < an; ++ai )
+		{
+            CAttributeNode *pAttrNode = FindAttrNode( attrs[ ai ] );
+			pAttrNode->m_bIsOutputToOperator = true;
+			pOpNode->m_OutputAttributes.AddToTail( pAttrNode );
+
+#ifdef _DEBUG
+			// Look for dependent operators, add them if they are not in the array
+			// FIXME: Should this happen for input attributes too?
+			CDmElement* pElement = pAttrNode->m_attribute->GetOwner();
+			IDmeOperator *pOperator = dynamic_cast< IDmeOperator* >( pElement );
+			if ( pOperator )
+			{
+				// Look for the operator in the existing list
+				if ( operatorDict.Find( pOperator ) == operatorDict.InvalidIndex() )
+				{
+					CDmElement *pOp1 = dynamic_cast< CDmElement* >( pOperator );
+					CDmElement *pOp2 = dynamic_cast< CDmElement* >( pOp );
+					Warning( "Found dependent operator '%s' referenced by operator '%s' that wasn't in the scene or trackgroups!\n", pOp1->GetName(), pOp2->GetName() );
+				}
+			}
+#endif
+		}
+
+		m_opNodes.AddToTail( pOpNode );
+	}
+}
+
+//-----------------------------------------------------------------------------
+// CDependencyGraph destructor - releases temporary opNodes and attrNodes
+//-----------------------------------------------------------------------------
+CDependencyGraph::~CDependencyGraph()
+{
+	Cleanup();
+}
+
+void CDependencyGraph::Cleanup()
+{
+	VPROF_BUDGET( "CDependencyGraph::Cleanup", VPROF_BUDGETGROUP_TOOLS );
+
+	int on = m_opNodes.Count();
+	for ( int oi = 0; oi < on; ++oi )
+	{
+		g_OperatorNodePool.Free( m_opNodes[ oi ] );
+	}
+
+	UtlHashHandle_t h = m_attrNodes.GetFirstHandle();
+	for ( ; h != m_attrNodes.InvalidHandle(); h = m_attrNodes.GetNextHandle( h ) )
+	{
+		g_AttrNodePool.Free( m_attrNodes[ h ] );
+	}
+
+	m_opRoots.RemoveAll();
+	m_opNodes.RemoveAll();
+	m_attrNodes.RemoveAll();
+	m_operators.RemoveAll();
+}
+
+
+//-----------------------------------------------------------------------------
+// caches changed operators as roots - typically once per frame, every frame
+//-----------------------------------------------------------------------------
+void CDependencyGraph::FindRoots()
+{
+	m_opRoots.RemoveAll();
+
+	uint oi;
+	uint on = m_opNodes.Count();
+
+	for ( oi = 0; oi < on; ++oi )
+	{
+		COperatorNode *pOpNode = m_opNodes[ oi ];
+		pOpNode->m_bInList = false;
+		pOpNode->m_state = TS_NOT_VISITED;
+
+		IDmeOperator *pOp = pOpNode->m_operator;
+		if ( !pOp->IsDirty() )
+			continue;
+
+		m_opRoots.AddToTail( pOpNode );
+		pOpNode->m_bInList = true;
+	}
+
+	
+
+	// Do we have an attribute which is an input to us which is not an output to some other op?
+	UtlHashHandle_t h = m_attrNodes.GetFirstHandle();
+	for ( ; h != m_attrNodes.InvalidHandle(); h = m_attrNodes.GetNextHandle( h ) )
+	{
+		CAttributeNode *pAttrNode = m_attrNodes[ h ];
+		//Msg( "attrib %s %p\n", pAttrNode->m_attribute->GetName(), pAttrNode->m_attribute );
+		if ( !pAttrNode->m_bIsOutputToOperator &&
+			pAttrNode->m_attribute->IsFlagSet( FATTRIB_OPERATOR_DIRTY ) )
+		{
+			on = pAttrNode->m_InputDependentOperators.Count();
+			for ( oi = 0; oi < on; ++oi )
+			{
+				COperatorNode *pOpNode = pAttrNode->m_InputDependentOperators[ oi ];
+				if ( !pOpNode->m_bInList )
+				{
+					m_opRoots.AddToTail( pOpNode );
+					pOpNode->m_bInList = true;
+				}
+			}
+		}
+
+		pAttrNode->m_attribute->RemoveFlag( FATTRIB_OPERATOR_DIRTY );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// returns only the operators that need to be evaluated, sorted by dependencies
+//-----------------------------------------------------------------------------
+bool CDependencyGraph::CullAndSortOperators()
+{
+	FindRoots();
+
+	m_operators.RemoveAll();
+
+	bool cycle = GetOperatorOrdering( m_opRoots, m_operators ); // leaves to roots (outputs to inputs)
+
+	int on = m_operators.Count();
+	int oh = on / 2;
+	for ( int oi = 0; oi < oh; ++oi )
+	{
+		V_swap( m_operators[ oi ], m_operators[ on - oi - 1 ] );
+	}
+	return cycle;
+
+//	return GetOperatorOrdering( m_opLeaves, operators ); // roots to leaves (inputs to outputs)
+}
+
+//-----------------------------------------------------------------------------
+// GetOperatorOrdering is just a recursive post-order depth-first-search
+//   since we have two types of nodes, we manually traverse to the opnodes grandchildren, which are opnodes
+//   (skipping children, which are attrnodes)
+//   returns true if a cycle found - in this case, an arbitrary link of the cycle will be ignored
+//-----------------------------------------------------------------------------
+bool CDependencyGraph::GetOperatorOrdering( CUtlVector< COperatorNode * > &pOpNodes, CUtlVector< IDmeOperator * > &operators )
+{
+	bool cycle = false;
+
+	uint on = pOpNodes.Count();
+	for ( uint oi = 0; oi < on; ++oi )
+	{
+		COperatorNode *pOpNode = pOpNodes[ oi ];
+		if ( pOpNode->m_state != TS_NOT_VISITED )
+		{
+			if ( pOpNode->m_state == TS_VISITING )
+			{
+				cycle = true;
+			}
+			continue;
+		}
+		pOpNode->m_state = TS_VISITING; // mark as in being visited
+
+		// DBG_PrintOperator( pIndent, pOpNode->m_operator );
+
+		// leaves to roots (outputs to inputs)
+		uint an = pOpNode->m_OutputAttributes.Count();
+		for ( uint ai = 0; ai < an; ++ai )
+		{
+			CAttributeNode *pAttrNode = pOpNode->m_OutputAttributes[ ai ];
+			if ( GetOperatorOrdering( pAttrNode->m_InputDependentOperators, operators ) )
+			{
+				cycle = true;
+			}
+		}
+
+		operators.AddToTail( pOpNode->m_operator );
+		pOpNode->m_state = TS_VISITED; // mark as done visiting
+	}
+	return cycle;
+}
+
+//-----------------------------------------------------------------------------
+// internal helper method - finds attrNode corresponding to pAttr
+//-----------------------------------------------------------------------------
+CAttributeNode *CDependencyGraph::FindAttrNode( CDmAttribute *pAttr )
+{
+	VPROF_BUDGET( "CDependencyGraph::FindAttrNode", VPROF_BUDGETGROUP_TOOLS );
+
+	Assert( pAttr );
+
+	CAttributeNode search( pAttr );
+	UtlHashHandle_t idx = m_attrNodes.Find( &search );
+	if ( idx != m_attrNodes.InvalidHandle() )
+	{
+		return m_attrNodes.Element( idx );
+	}
+
+	CAttributeNode *pAttrNode = 0;
+	{
+		VPROF( "CDependencyGraph::FindAttrNode_Alloc" );
+		pAttrNode = g_AttrNodePool.Alloc();
+		pAttrNode->m_attribute = pAttr;
+	}
+	{
+		VPROF( "CDependencyGraph::FindAttrNode_Alloc2" );
+		m_attrNodes.Insert( pAttrNode );
+	}
+	return pAttrNode;
+}
+
+//-----------------------------------------------------------------------------
+// temporary internal debugging function
+//-----------------------------------------------------------------------------
+void CDependencyGraph::DBG_PrintOperator( const char *pIndent, IDmeOperator *pOp )
+{
+	CDmElement *pElement = dynamic_cast< CDmElement* >( pOp );
+	Msg( "%s%s <%s> {\n", pIndent, pElement->GetName(), pElement->GetTypeString() );
+}

datamodel/dependencygraph.h

@@ -0,0 +1,62 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef DEPENDENCYGRAPH_H
+#define DEPENDENCYGRAPH_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "tier1/utlvector.h"
+#include "tier1/utlhash.h"
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class CDmAttribute;
+class IDmeOperator;
+struct COperatorNode;
+class CAttributeNode;
+
+
+//-----------------------------------------------------------------------------
+// CDependencyGraph class - sorts operators based upon the input/output graph
+//-----------------------------------------------------------------------------
+class CDependencyGraph
+{
+public:
+	CDependencyGraph();
+	~CDependencyGraph();
+
+	void Reset( const CUtlVector< IDmeOperator * > &operators );
+
+	// caches only the operators that need to be evaluated, sorted by dependencies
+	// returns true if a cycle found - in this case, an arbitrary link of the cycle will be ignored
+	bool CullAndSortOperators();
+
+	const CUtlVector< IDmeOperator* > &GetSortedOperators() const { return m_operators; }
+
+private:
+	static bool GetOperatorOrdering( CUtlVector< COperatorNode* > &pOpNodes, CUtlVector< IDmeOperator * > &operators );
+	static void DBG_PrintOperator( const char *pIndent, IDmeOperator *pOp );
+
+	friend class CDmElementFramework;
+
+	void Cleanup();
+	void FindRoots();
+	CAttributeNode *FindAttrNode( CDmAttribute *pAttr );
+
+	CUtlVector< COperatorNode* > m_opRoots;
+//	CUtlVector< COperatorNode* > m_opLeaves;
+
+	CUtlVector< COperatorNode* > m_opNodes;
+
+	CUtlHash< CAttributeNode* > m_attrNodes;
+
+	CUtlVector< IDmeOperator* > m_operators;
+};
+
+#endif // DEPENDENCYGRAPH_H

datamodel/dmattributeinternal.h

@@ -0,0 +1,75 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef DMATTRIBUTEINTERNAL_H
+#define DMATTRIBUTEINTERNAL_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "datamodel/dmattribute.h"
+#include "wchar.h"
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations: 
+//-----------------------------------------------------------------------------
+class IDataModelFactory;
+class CUtlBuffer;
+class Vector;
+class Color;
+class CUtlCharConversion;
+class CDmElement;
+
+
+//-----------------------------------------------------------------------------
+// Utility class to allow datamodel objects to access private members of CDmAttribute
+//-----------------------------------------------------------------------------
+class CDmAttributeAccessor
+{
+public:
+	static void OnChanged( CDmAttribute *pAttribute, bool bArrayCountChanged = false, bool bIsTopological = false )
+	{
+		pAttribute->OnChanged( bArrayCountChanged, bIsTopological );
+	}
+
+	static void DestroyAttribute( CDmAttribute *pOldAttribute )
+	{
+		CDmAttribute::DestroyAttribute( pOldAttribute );
+	}
+
+	static bool MarkDirty( CDmAttribute *pAttribute )
+	{
+		return pAttribute->MarkDirty();
+	}
+};
+
+//-----------------------------------------------------------------------------
+// For serialization, set the delimiter rules
+//-----------------------------------------------------------------------------
+void SetSerializationDelimiter( CUtlCharConversion *pConv );
+void SetSerializationArrayDelimiter( const char *pDelimiter );
+
+
+//-----------------------------------------------------------------------------
+// Skip unserialization for an attribute type (unserialize into a dummy variable)
+//-----------------------------------------------------------------------------
+bool SkipUnserialize( CUtlBuffer &buf, DmAttributeType_t type );
+
+
+//-----------------------------------------------------------------------------
+// Attribute names/types
+//-----------------------------------------------------------------------------
+const char *AttributeTypeName( DmAttributeType_t type );
+DmAttributeType_t AttributeType( const char *pAttributeType );
+
+
+//-----------------------------------------------------------------------------
+// returns the number of attributes currently allocated
+//-----------------------------------------------------------------------------
+int GetAllocatedAttributeCount();
+
+#endif // DMATTRIBUTEINTERNAL_H

datamodel/dmelementdictionary.cpp

@@ -0,0 +1,468 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmelementdictionary.h"
+#include "datamodel/dmelement.h"
+#include "datamodel/dmattribute.h"
+#include "datamodel/dmattributevar.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel.h"
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CDmElementDictionary::CDmElementDictionary()
+	: m_idmap( 1024, 0, 0, DmIdPair_t::Compare, DmIdPair_t::HashKey )
+{
+}
+
+
+//-----------------------------------------------------------------------------
+// Clears the dictionary
+//-----------------------------------------------------------------------------
+void CDmElementDictionary::Clear()
+{
+	m_Dict.Purge();
+	m_Attributes.Purge();
+	m_ArrayAttributes.Purge();
+	m_elementsToDelete.Purge();
+}
+
+	
+//-----------------------------------------------------------------------------
+// Inserts an element into the table
+//-----------------------------------------------------------------------------
+DmElementDictHandle_t CDmElementDictionary::InsertElement( CDmElement *pElement )
+{
+	// Insert it into the reconnection table
+	return m_Dict.AddToTail( pElement ? pElement->GetHandle() : DMELEMENT_HANDLE_INVALID );
+}
+
+
+//-----------------------------------------------------------------------------
+// Returns a particular element
+//-----------------------------------------------------------------------------
+CDmElement *CDmElementDictionary::GetElement( DmElementDictHandle_t handle )
+{
+	if ( handle == ELEMENT_DICT_HANDLE_INVALID )
+		return NULL;
+
+	return g_pDataModel->GetElement( m_Dict[ handle ] );
+}
+
+
+//-----------------------------------------------------------------------------
+// Adds an attribute to the fixup list
+//-----------------------------------------------------------------------------
+void CDmElementDictionary::AddAttribute( CDmAttribute *pAttribute, const DmObjectId_t &objectId )
+{
+	if ( m_elementsToDelete.Find( pAttribute->GetOwner()->GetHandle() ) != m_elementsToDelete.InvalidIndex() )
+		return; // don't add attributes if their element is being deleted
+
+	int i = m_Attributes.AddToTail();
+	m_Attributes[i].m_nType = AT_OBJECTID;
+	m_Attributes[i].m_pAttribute = pAttribute;
+	CopyUniqueId( objectId, &m_Attributes[i].m_ObjectId );
+}
+
+
+//-----------------------------------------------------------------------------
+// Adds an element of an attribute array to the fixup list
+//-----------------------------------------------------------------------------
+void CDmElementDictionary::AddArrayAttribute( CDmAttribute *pAttribute, DmElementDictHandle_t hElement )
+{
+	if ( m_elementsToDelete.Find( pAttribute->GetOwner()->GetHandle() ) != m_elementsToDelete.InvalidIndex() )
+		return; // don't add attributes if their element is being deleted
+
+	int i = m_ArrayAttributes.AddToTail();
+	m_ArrayAttributes[i].m_nType = AT_ELEMENT;
+	m_ArrayAttributes[i].m_pAttribute = pAttribute;
+	m_ArrayAttributes[i].m_hElement = hElement;
+}
+
+void CDmElementDictionary::AddArrayAttribute( CDmAttribute *pAttribute, const DmObjectId_t &objectId )
+{
+	if ( m_elementsToDelete.Find( pAttribute->GetOwner()->GetHandle() ) != m_elementsToDelete.InvalidIndex() )
+		return; // don't add attributes if their element is being deleted
+
+	int i = m_ArrayAttributes.AddToTail();
+	m_ArrayAttributes[i].m_nType = AT_OBJECTID;
+	m_ArrayAttributes[i].m_pAttribute = pAttribute;
+	CopyUniqueId( objectId, &m_ArrayAttributes[i].m_ObjectId );
+}
+
+//-----------------------------------------------------------------------------
+// 
+//-----------------------------------------------------------------------------
+void CDmElementDictionary::RemoveAttributeInfosOfElement( AttributeList_t &attributes, DmElementHandle_t hElement )
+{
+	while ( attributes.Count() > 0 && attributes.Tail().m_pAttribute->GetOwner()->GetHandle() == hElement )
+	{
+		attributes.Remove( attributes.Count() - 1 );
+	}
+}
+
+DmElementHandle_t CDmElementDictionary::SetElementId( DmElementDictHandle_t hDictHandle, const DmObjectId_t &newId, DmConflictResolution_t idConflictResolution )
+{
+	DmElementHandle_t hElement = m_Dict[ hDictHandle ];
+	CDmElement *pElement = g_pDataModel->GetElement( hElement );
+	Assert( pElement );
+	if ( !pElement )
+		return DMELEMENT_HANDLE_INVALID;
+
+	const DmObjectId_t &oldId = pElement->GetId();
+
+	if ( idConflictResolution == CR_FORCE_COPY )
+	{
+		m_idmap.Insert( DmIdPair_t( newId, oldId ) ); // map the newId back to the old id, and keep the old id
+		return hElement;
+	}
+
+	DmElementHandle_t newHandle = g_pDataModelImp->ChangeElementId( hElement, oldId, newId );
+	if ( newHandle != DMELEMENT_HANDLE_INVALID )
+	{
+		// if ChangeElementId returns a handle, the id has been changed
+		if ( newHandle != hElement )
+		{
+			int i = m_Dict.Find( hElement );
+			if ( i != m_Dict.InvalidIndex() )
+			{
+				m_Dict[ i ] = newHandle;
+			}
+		}
+		return newHandle; // either keeping the old handle, with the new id, or found a new handle associated with that new id
+	}
+
+	// id not changed because that id is already in use
+	if ( idConflictResolution == CR_DELETE_NEW )
+	{
+		DmElementHandle_t hExistingElement = g_pDataModel->FindElement( newId );
+
+		int i = m_elementsToDelete.AddToTail( );
+		m_elementsToDelete[i].m_hDictHandle = hDictHandle;
+		m_elementsToDelete[i].m_hElementToDelete = hElement;
+		m_elementsToDelete[i].m_hReplacementElement = hExistingElement;
+
+		// remove all element ref attributes read in before the id (typically none)
+		RemoveAttributeInfosOfElement( m_Attributes, hElement );
+		RemoveAttributeInfosOfElement( m_ArrayAttributes, hElement );
+
+		return DMELEMENT_HANDLE_INVALID;
+	}
+
+	if ( idConflictResolution == CR_DELETE_OLD )
+	{
+		DmElementHandle_t hExistingElement = g_pDataModel->FindElement( newId );
+		Assert( hExistingElement != DMELEMENT_HANDLE_INVALID );
+		if ( hExistingElement == DMELEMENT_HANDLE_INVALID )
+			return DMELEMENT_HANDLE_INVALID; // unexpected error in ChangeElementId (failed due to something other than a conflict)
+
+		g_pDataModelImp->DeleteElement( hExistingElement, HR_NEVER ); // need to keep the handle around until ChangeElemendId
+		newHandle = g_pDataModelImp->ChangeElementId( hElement, oldId, newId );
+		Assert( newHandle == hExistingElement );
+
+		int i = m_Dict.Find( hElement );
+		if ( i != m_Dict.InvalidIndex() )
+		{
+			m_Dict[ i ] = newHandle;
+		}
+
+		return newHandle;
+	}
+
+	if ( idConflictResolution == CR_COPY_NEW )
+	{
+		m_idmap.Insert( DmIdPair_t( newId, oldId ) ); // map the newId back to the old id, and keep the old id
+		return hElement;
+	}
+
+	Assert( 0 );
+	return DMELEMENT_HANDLE_INVALID;
+}
+
+
+//-----------------------------------------------------------------------------
+// Finds an element into the table
+//-----------------------------------------------------------------------------
+DmElementDictHandle_t CDmElementDictionary::FindElement( CDmElement *pElement )
+{
+	return m_Dict.Find( pElement ? pElement->GetHandle() : DMELEMENT_HANDLE_INVALID );
+}
+
+
+//-----------------------------------------------------------------------------
+// Hook up all element references (which were unserialized as object ids)
+//-----------------------------------------------------------------------------
+void CDmElementDictionary::HookUpElementAttributes()
+{
+	int n = m_Attributes.Count();
+	for ( int i = 0; i < n; ++i )
+	{
+		Assert( m_Attributes[i].m_pAttribute->GetType() == AT_ELEMENT );
+		Assert( m_Attributes[i].m_nType == AT_OBJECTID );
+
+		UtlHashHandle_t h = m_idmap.Find( DmIdPair_t( m_Attributes[i].m_ObjectId ) );
+		DmObjectId_t &id = h == m_idmap.InvalidHandle() ? m_Attributes[i].m_ObjectId : m_idmap[ h ].m_newId;
+
+		// search id->handle table (both loaded and unloaded) for id, and if not found, create a new handle, map it to the id and return it
+		DmElementHandle_t hElement = g_pDataModelImp->FindOrCreateElementHandle( id );
+		m_Attributes[i].m_pAttribute->SetValue<DmElementHandle_t>( hElement );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Hook up all element array references
+//-----------------------------------------------------------------------------
+void CDmElementDictionary::HookUpElementArrayAttributes()
+{
+	// Find unique array attributes; we need to clear them all before adding stuff.
+	// This clears them of stuff added during their construction phase.
+	int n = m_ArrayAttributes.Count();
+	CUtlRBTree< CDmAttribute*, unsigned short >	lookup( 0, n, DefLessFunc(CDmAttribute*) );
+	for ( int i = 0; i < n; ++i )
+	{
+		Assert( m_ArrayAttributes[i].m_pAttribute->GetType() == AT_ELEMENT_ARRAY );
+		CDmAttribute *pElementArray = m_ArrayAttributes[i].m_pAttribute;
+		CDmrElementArray<> array( pElementArray );
+		if ( lookup.Find( pElementArray ) == lookup.InvalidIndex() )
+		{
+			array.RemoveAll();
+			lookup.Insert( pElementArray );
+		}
+	}
+
+	for ( int i = 0; i < n; ++i )
+	{
+		Assert( m_ArrayAttributes[i].m_pAttribute->GetType() == AT_ELEMENT_ARRAY );
+
+		CDmrElementArray<> array( m_ArrayAttributes[i].m_pAttribute );
+
+		if ( m_ArrayAttributes[i].m_nType == AT_ELEMENT )
+		{
+			CDmElement *pElement = GetElement( m_ArrayAttributes[i].m_hElement );
+			array.AddToTail( pElement );
+		}
+		else
+		{
+			UtlHashHandle_t h = m_idmap.Find( DmIdPair_t( m_ArrayAttributes[i].m_ObjectId ) );
+			DmObjectId_t &id = ( h == m_idmap.InvalidHandle() ) ? m_ArrayAttributes[i].m_ObjectId : m_idmap[ h ].m_newId;
+
+			// search id->handle table (both loaded and unloaded) for id, and if not found, create a new handle, map it to the id and return it
+			DmElementHandle_t hElement = g_pDataModelImp->FindOrCreateElementHandle( id );
+			int nIndex = array.AddToTail();
+			array.SetHandle( nIndex, hElement );
+		}
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Hook up all element references (which were unserialized as object ids)
+//-----------------------------------------------------------------------------
+void CDmElementDictionary::HookUpElementReferences()
+{
+	int nElementsToDelete = m_elementsToDelete.Count();
+	for ( int i = 0; i < nElementsToDelete; ++i )
+	{
+		DmElementDictHandle_t hDictIndex = m_elementsToDelete[i].m_hDictHandle;
+		DmElementHandle_t hElement = m_Dict[ hDictIndex ];
+		g_pDataModelImp->DeleteElement( hElement );
+		m_Dict[ hDictIndex ] = m_elementsToDelete[i].m_hReplacementElement;
+	}
+
+	HookUpElementArrayAttributes();
+	HookUpElementAttributes();
+}
+
+
+
+//-----------------------------------------------------------------------------
+//
+// Element dictionary used in serialization
+//
+//-----------------------------------------------------------------------------
+CDmElementSerializationDictionary::CDmElementSerializationDictionary() :
+	m_Dict( 1024, 0, CDmElementSerializationDictionary::LessFunc )
+{
+}
+
+
+//-----------------------------------------------------------------------------
+// Used to sort the list of elements
+//-----------------------------------------------------------------------------
+bool CDmElementSerializationDictionary::LessFunc( const ElementInfo_t &lhs, const ElementInfo_t &rhs )
+{
+	return lhs.m_pElement < rhs.m_pElement;
+}
+
+
+//-----------------------------------------------------------------------------
+// Finds the handle of the element
+//-----------------------------------------------------------------------------
+DmElementDictHandle_t CDmElementSerializationDictionary::Find( CDmElement *pElement )
+{
+	ElementInfo_t find;
+	find.m_pElement = pElement;
+	return m_Dict.Find( find );
+}
+
+	
+//-----------------------------------------------------------------------------
+// Creates the list of all things to serialize
+//-----------------------------------------------------------------------------
+void CDmElementSerializationDictionary::BuildElementList_R( CDmElement *pElement, bool bFlatMode, bool bIsRoot )
+{
+	if ( !pElement )
+		return;
+
+	// FIXME: Right here we should ask the element if it's an external
+	// file reference and exit immediately if so.
+
+	// This means we've already encountered this guy.
+	// Therefore, he can never be a root element
+	DmElementDictHandle_t h = Find( pElement );
+	if ( h != m_Dict.InvalidIndex() )
+	{
+		m_Dict[h].m_bRoot = true;
+		return;
+	}
+
+	ElementInfo_t info;
+	info.m_bRoot = bFlatMode || bIsRoot;
+	info.m_pElement = pElement;
+	m_Dict.Insert( info );
+
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->IsFlagSet( FATTRIB_DONTSAVE ) )
+			continue;
+
+		switch( pAttribute->GetType() )
+		{
+		case AT_ELEMENT:
+			{
+				CDmElement *pChild = pAttribute->GetValueElement<CDmElement>();
+				if ( !pChild || pChild->GetFileId() != pElement->GetFileId() )
+					break;
+
+				BuildElementList_R( pChild, bFlatMode, false );
+			}
+			break;
+
+		case AT_ELEMENT_ARRAY:
+			{
+				CDmrElementArray<> array( pAttribute );
+				int nCount = array.Count();
+				for ( int i = 0; i < nCount; ++i )
+				{
+					CDmElement *pChild = array[i];
+					if ( !pChild || pChild->GetFileId() != pElement->GetFileId() )
+						break;
+
+					BuildElementList_R( pChild, bFlatMode, false );
+				}
+			}
+			break;
+		}
+	}
+}
+
+void CDmElementSerializationDictionary::BuildElementList( CDmElement *pElement, bool bFlatMode )
+{
+	BuildElementList_R( pElement, bFlatMode, true );
+}
+
+
+//-----------------------------------------------------------------------------
+// Should I inline the serialization of this element?
+//-----------------------------------------------------------------------------
+bool CDmElementSerializationDictionary::ShouldInlineElement( CDmElement *pElement )
+{
+	// This means we've already encountered this guy.
+	// Therefore, he can never be a root element
+	DmElementDictHandle_t h = Find( pElement );
+	if ( h != m_Dict.InvalidIndex() )
+		return !m_Dict[h].m_bRoot;
+
+	// If we didn't find the element, it means it's a reference to an external
+	// element (or it's NULL), so don't inline ie.
+	return false;
+}
+
+
+//-----------------------------------------------------------------------------
+// Clears the dictionary
+//-----------------------------------------------------------------------------
+void CDmElementSerializationDictionary::Clear()
+{
+	m_Dict.RemoveAll();
+}
+
+
+//-----------------------------------------------------------------------------
+// How many root elements do we have?
+//-----------------------------------------------------------------------------
+int CDmElementSerializationDictionary::RootElementCount() const
+{
+	int nCount = 0;
+	DmElementDictHandle_t h = m_Dict.FirstInorder();
+	while( h != m_Dict.InvalidIndex() )
+	{
+		if ( m_Dict[h].m_bRoot )
+		{
+			++nCount;
+		}
+		h = m_Dict.NextInorder( h );
+	}
+	return nCount;
+}
+
+	
+//-----------------------------------------------------------------------------
+// Iterates over all root elements to serialize
+//-----------------------------------------------------------------------------
+DmElementDictHandle_t CDmElementSerializationDictionary::FirstRootElement() const
+{
+	// NOTE - this code only works with BlockMemory or Memory (NOT FixedMemory)
+
+	// NOTE: I don't have to use First/NextInorder here because there
+	// are guaranteed to be no removals from the dictionary.
+	// Also, using inorder traversal won't get my actual root element to be first in the file
+	int nCount = m_Dict.Count();
+	for ( DmElementDictHandle_t h = 0; h < nCount; ++h )
+	{
+		if ( m_Dict[h].m_bRoot )
+			return h;
+	}
+	return ELEMENT_DICT_HANDLE_INVALID;
+}
+
+DmElementDictHandle_t CDmElementSerializationDictionary::NextRootElement( DmElementDictHandle_t h ) const
+{
+	// NOTE - this code only works with BlockMemory or Memory (NOT FixedMemory)
+
+	// NOTE: I don't have to use First/NextInorder here because there
+	// are guaranteed to be no removals from the dictionary.
+	// Also, using inorder traversal won't get my actual root element to be first in the file
+	++h;
+	int nCount = m_Dict.Count();
+	for ( ; h < nCount; ++h )
+	{
+		if ( m_Dict[h].m_bRoot )
+			return h;
+	}
+	return ELEMENT_DICT_HANDLE_INVALID;
+}
+
+CDmElement *CDmElementSerializationDictionary::GetRootElement( DmElementDictHandle_t h )
+{
+	Assert( m_Dict[h].m_bRoot );
+	return m_Dict[h].m_pElement;
+}
+
+

datamodel/dmelementdictionary.h

@@ -0,0 +1,184 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef DMELEMENTDICTIONARY_H
+#define DMELEMENTDICTIONARY_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+
+#include "tier1/utlvector.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmattribute.h"
+#include "tier1/utlrbtree.h"
+#include "tier1/utlhash.h"
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class CDmElement;
+class CDmAttribute;
+
+
+//-----------------------------------------------------------------------------
+// Element dictionary used in unserialization
+//-----------------------------------------------------------------------------
+typedef int DmElementDictHandle_t;
+enum
+{
+	ELEMENT_DICT_HANDLE_INVALID = (DmElementDictHandle_t)~0
+};
+
+
+class CDmElementDictionary
+{
+public:
+	CDmElementDictionary();
+
+	DmElementDictHandle_t InsertElement( CDmElement *pElement );
+	CDmElement *GetElement( DmElementDictHandle_t handle );
+	void AddAttribute( CDmAttribute *pAttribute, const DmObjectId_t &pElementId );
+	void AddArrayAttribute( CDmAttribute *pAttribute, DmElementDictHandle_t hChild );
+	void AddArrayAttribute( CDmAttribute *pAttribute, const DmObjectId_t &pElementId );
+
+	DmElementHandle_t SetElementId( DmElementDictHandle_t hDictHandle,
+									const DmObjectId_t &newId,
+									DmConflictResolution_t idConflictResolution );
+
+	// Finds an element into the table
+	DmElementDictHandle_t FindElement( CDmElement *pElement );
+
+ 	// Hook up all element references (which were unserialized as object ids)
+	void HookUpElementReferences();
+
+	// Clears the dictionary
+	void Clear();
+
+	// iteration through elements
+	DmElementDictHandle_t FirstElement() { return 0; }
+	DmElementDictHandle_t NextElement( DmElementDictHandle_t h )
+	{
+		return m_Dict.IsValidIndex( h+1 ) ? h+1 : ELEMENT_DICT_HANDLE_INVALID;
+	}
+
+
+private:
+	struct AttributeInfo_t
+	{
+		CDmAttribute *m_pAttribute;
+		int m_nType;	// AT_ELEMENT or AT_OBJECTID
+		union
+		{
+			DmElementDictHandle_t m_hElement;
+			DmObjectId_t m_ObjectId;
+		};
+	};
+	typedef CUtlVector<AttributeInfo_t> AttributeList_t;
+
+
+	struct DmIdPair_t
+	{
+		DmObjectId_t m_oldId;
+		DmObjectId_t m_newId;
+		DmIdPair_t() {}
+		DmIdPair_t( const DmObjectId_t &id )
+		{
+			CopyUniqueId( id, &m_oldId );
+		}
+		DmIdPair_t( const DmObjectId_t &oldId, const DmObjectId_t &newId )
+		{
+			CopyUniqueId( oldId, &m_oldId );
+			CopyUniqueId( newId, &m_newId );
+		}
+		DmIdPair_t &operator=( const DmIdPair_t &that )
+		{
+			CopyUniqueId( that.m_oldId, &m_oldId );
+			CopyUniqueId( that.m_newId, &m_newId );
+			return *this;
+		}
+		static unsigned int HashKey( const DmIdPair_t& that )
+		{
+			return *( unsigned int* )&that.m_oldId.m_Value;
+		}
+		static bool Compare( const DmIdPair_t& a, const DmIdPair_t& b )
+		{
+			return IsUniqueIdEqual( a.m_oldId, b.m_oldId );
+		}
+	};
+
+	struct DeletionInfo_t
+	{
+		DeletionInfo_t() {}
+		DeletionInfo_t( DmElementHandle_t hElement ) : m_hElementToDelete( hElement ) {}
+		bool operator==( const DeletionInfo_t& src ) const { return m_hElementToDelete == src.m_hElementToDelete; }
+
+		DmElementDictHandle_t m_hDictHandle;
+		DmElementHandle_t m_hElementToDelete;
+		DmElementHandle_t m_hReplacementElement;
+	};
+
+	// Hook up all element references (which were unserialized as object ids)
+	void HookUpElementAttributes();
+	void HookUpElementArrayAttributes();
+
+	void RemoveAttributeInfosOfElement( AttributeList_t &attributes, DmElementHandle_t hElement );
+
+	CUtlVector< DmElementHandle_t > m_Dict;
+	AttributeList_t m_Attributes;
+	AttributeList_t m_ArrayAttributes;
+
+	CUtlVector< DeletionInfo_t > m_elementsToDelete;
+	CUtlHash< DmIdPair_t > m_idmap;
+};
+
+
+//-----------------------------------------------------------------------------
+// Element dictionary used in serialization
+//-----------------------------------------------------------------------------
+class CDmElementSerializationDictionary
+{
+public:
+	CDmElementSerializationDictionary();
+
+	// Creates the list of all things to serialize
+	void BuildElementList( CDmElement *pRoot, bool bFlatMode );
+
+	// Should I inline the serialization of this element?
+	bool ShouldInlineElement( CDmElement *pElement );
+
+	// Clears the dictionary
+	void Clear();
+
+	// Iterates over all root elements to serialize
+	DmElementDictHandle_t FirstRootElement() const;
+	DmElementDictHandle_t NextRootElement( DmElementDictHandle_t h ) const;
+ 	CDmElement* GetRootElement( DmElementDictHandle_t h );
+
+	// Finds the handle of the element
+	DmElementDictHandle_t Find( CDmElement *pElement );
+
+	// How many root elements do we have?
+	int RootElementCount() const;
+
+private:
+	struct ElementInfo_t
+	{
+		bool m_bRoot;
+		CDmElement* m_pElement;
+	};
+
+	// Creates the list of all things to serialize
+	void BuildElementList_R( CDmElement *pRoot, bool bFlatMode, bool bIsRoot );
+	static bool LessFunc( const ElementInfo_t &lhs, const ElementInfo_t &rhs );
+
+	CUtlBlockRBTree< ElementInfo_t, DmElementDictHandle_t > m_Dict;
+};
+
+
+#endif // DMELEMENTDICTIONARY_H

datamodel/dmelementfactoryhelper.cpp

@@ -0,0 +1,100 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "datamodel/dmelementfactoryhelper.h"
+#include "tier0/dbg.h"
+
+// memdbgon must be the last include file in a .cpp file!!!
+#include "tier0/memdbgon.h"
+
+CDmElementFactoryHelper *CDmElementFactoryHelper::s_pHelpers[2] = { NULL, NULL };
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CDmElementFactoryHelper::CDmElementFactoryHelper( const char *classname, IDmElementFactoryInternal *pFactory, bool bIsStandardFactory )
+{
+	m_pNext = s_pHelpers[bIsStandardFactory];
+	s_pHelpers[bIsStandardFactory] = this;
+
+	// Set attributes
+	Assert( pFactory );
+	m_pFactory = pFactory;
+	Assert( classname );
+	m_pszClassname	= classname;
+}
+
+//-----------------------------------------------------------------------------
+// Purpose: Returns next object in list
+// Output : CDmElementFactoryHelper
+//-----------------------------------------------------------------------------
+CDmElementFactoryHelper *CDmElementFactoryHelper::GetNext( void )
+{ 
+	return m_pNext;
+}
+
+
+//-----------------------------------------------------------------------------
+// Installs all factories into the datamodel system
+//-----------------------------------------------------------------------------
+// NOTE: The name of this extern is defined by the macro IMPLEMENT_ELEMENT_FACTORY 
+extern CDmElementFactoryHelper g_CDmElement_Helper;
+
+void CDmElementFactoryHelper::InstallFactories( )
+{
+	// Just set up the type symbols of the other factories
+	CDmElementFactoryHelper *p = s_pHelpers[0];
+	while ( p )
+	{
+		// Add factories to database
+		if ( !p->GetFactory()->IsAbstract() )
+		{
+			g_pDataModel->AddElementFactory( p->GetClassname(), p->GetFactory() );
+		}
+
+		// Set up the type symbol. Note this can't be done at
+		// constructor time since we don't have a DataModel pointer then
+		p->GetFactory()->SetElementTypeSymbol( g_pDataModel->GetSymbol( p->GetClassname() ) );
+		p = p->GetNext();
+	}
+
+	p = s_pHelpers[1];
+	while ( p )
+	{
+		// Add factories to database, but not if they've been overridden
+		if ( !g_pDataModel->HasElementFactory( p->GetClassname() ) )
+		{
+			if ( !p->GetFactory()->IsAbstract() )
+			{
+				g_pDataModel->AddElementFactory( p->GetClassname(), p->GetFactory() );
+			}
+
+			// Set up the type symbol. Note this can't be done at
+			// constructor time since we don't have a DataModel pointer then
+
+			// Backward compat--don't let the type symbol be 'DmeElement'
+			if ( Q_stricmp( p->GetClassname(), "DmeElement" ) )
+			{
+				p->GetFactory()->SetElementTypeSymbol( g_pDataModel->GetSymbol( p->GetClassname() ) );
+			}
+		}
+
+		p = p->GetNext();
+	}
+
+	// Also install the DmElement factory as the default factory
+	g_pDataModel->SetDefaultElementFactory( g_CDmElement_Helper.GetFactory() );
+}
+
+
+//-----------------------------------------------------------------------------
+// Installs all DmElement factories
+//-----------------------------------------------------------------------------
+void InstallDmElementFactories( )
+{
+	CDmElementFactoryHelper::InstallFactories( );
+}

datamodel/dmserializerbinary.cpp

@@ -0,0 +1,587 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializerbinary.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel.h"
+#include "datamodel/dmelement.h"
+#include "datamodel/dmattributevar.h"
+#include "dmattributeinternal.h"
+#include "dmelementdictionary.h"
+#include "tier1/utlbuffer.h"
+#include "DmElementFramework.h"
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class CUtlBuffer;
+class CBaseSceneObject;
+
+
+//-----------------------------------------------------------------------------
+// special element indices
+//-----------------------------------------------------------------------------
+enum
+{
+	ELEMENT_INDEX_NULL = -1,
+	ELEMENT_INDEX_EXTERNAL = -2,
+};
+
+
+//-----------------------------------------------------------------------------
+// Serialization class for Binary output
+//-----------------------------------------------------------------------------
+class CDmSerializerBinary : public IDmSerializer
+{
+public:
+	CDmSerializerBinary() {}
+	// Inherited from IDMSerializer
+	virtual const char *GetName() const { return "binary"; }
+	virtual const char *GetDescription() const { return "Binary"; }
+	virtual bool StoresVersionInFile() const { return true; }
+	virtual bool IsBinaryFormat() const { return true; }
+	virtual int GetCurrentVersion() const { return 2; }
+	virtual bool Serialize( CUtlBuffer &buf, CDmElement *pRoot );
+	virtual bool Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion,
+							  const char *pSourceFormatName, int nFormatVersion,
+							  DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot );
+    
+private:
+	// Methods related to serialization
+	void SerializeElementIndex( CUtlBuffer& buf, CDmElementSerializationDictionary& list, DmElementHandle_t hElement, DmFileId_t fileid );
+	void SerializeElementAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute );
+	void SerializeElementArrayAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute );
+	bool SerializeAttributes( CUtlBuffer& buf, CDmElementSerializationDictionary& list, unsigned short *symbolToIndexMap, CDmElement *pElement );
+	bool SaveElementDict( CUtlBuffer& buf, unsigned short *symbolToIndexMap, CDmElement *pElement );
+	bool SaveElement( CUtlBuffer& buf, CDmElementSerializationDictionary& dict, unsigned short *symbolToIndexMap, CDmElement *pElement);
+
+	// Methods related to unserialization
+	DmElementHandle_t UnserializeElementIndex( CUtlBuffer &buf, CUtlVector<CDmElement*> &elementList );
+	void UnserializeElementAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList );
+	void UnserializeElementArrayAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList );
+	bool UnserializeAttributes( CUtlBuffer &buf, CDmElement *pElement, CUtlVector<CDmElement*> &elementList, UtlSymId_t *symbolTable );
+	bool UnserializeElements( CUtlBuffer &buf, DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot, UtlSymId_t *symbolTable );
+};
+   
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CDmSerializerBinary s_DMSerializerBinary;
+
+void InstallBinarySerializer( IDataModel *pFactory )
+{
+	pFactory->AddSerializer( &s_DMSerializerBinary );
+}
+
+
+//-----------------------------------------------------------------------------
+// Write out the index of the element to avoid looks at read time
+//-----------------------------------------------------------------------------
+void CDmSerializerBinary::SerializeElementIndex( CUtlBuffer& buf, CDmElementSerializationDictionary& list, DmElementHandle_t hElement, DmFileId_t fileid )
+{
+	if ( hElement == DMELEMENT_HANDLE_INVALID )
+	{
+		buf.PutInt( ELEMENT_INDEX_NULL ); // invalid handle
+	}
+	else
+	{
+		CDmElement *pElement = g_pDataModel->GetElement( hElement );
+		if ( pElement )
+		{
+			if ( pElement->GetFileId() == fileid )
+			{
+				buf.PutInt( list.Find( pElement ) );
+			}
+			else
+			{
+				buf.PutInt( ELEMENT_INDEX_EXTERNAL );
+				char idstr[ 40 ];
+				UniqueIdToString( pElement->GetId(), idstr, sizeof( idstr ) );
+				buf.PutString( idstr );
+			}
+		}
+		else
+		{
+			DmObjectId_t *pId = NULL;
+			DmElementReference_t *pRef = g_pDataModelImp->FindElementReference( hElement, &pId );
+			if ( pRef && pId )
+			{
+				buf.PutInt( ELEMENT_INDEX_EXTERNAL );
+				char idstr[ 40 ];
+				UniqueIdToString( *pId, idstr, sizeof( idstr ) );
+				buf.PutString( idstr );
+			}
+			else
+			{
+				buf.PutInt( ELEMENT_INDEX_NULL ); // invalid handle
+			}
+		}
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out element attributes
+//-----------------------------------------------------------------------------
+void CDmSerializerBinary::SerializeElementAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute )
+{
+	SerializeElementIndex( buf, list, pAttribute->GetValue< DmElementHandle_t >(), pAttribute->GetOwner()->GetFileId() );
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out element array attributes
+//-----------------------------------------------------------------------------
+void CDmSerializerBinary::SerializeElementArrayAttribute( CUtlBuffer& buf, CDmElementSerializationDictionary& list, CDmAttribute *pAttribute )
+{
+	DmFileId_t fileid = pAttribute->GetOwner()->GetFileId();
+	CDmrElementArray<> vec( pAttribute );
+
+	int nCount = vec.Count();
+	buf.PutInt( nCount );
+	for ( int i = 0; i < nCount; ++i )
+	{
+		SerializeElementIndex( buf, list, vec.GetHandle(i), fileid );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out all attributes
+//-----------------------------------------------------------------------------
+bool CDmSerializerBinary::SerializeAttributes( CUtlBuffer& buf, CDmElementSerializationDictionary& list, unsigned short *symbolToIndexMap, CDmElement *pElement )
+{
+	// Collect the attributes to be written
+	CDmAttribute **ppAttributes = ( CDmAttribute** )_alloca( pElement->AttributeCount() * sizeof( CDmAttribute* ) );
+	int nAttributes = 0;
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->IsFlagSet( FATTRIB_DONTSAVE | FATTRIB_STANDARD ) )
+			continue;
+
+		ppAttributes[ nAttributes++ ] = pAttribute;
+	}
+
+	// Now write them all out in reverse order, since FirstAttribute is actually the *last* attribute for perf reasons
+	buf.PutInt( nAttributes );
+	for ( int i = nAttributes - 1; i >= 0; --i )
+	{
+		CDmAttribute *pAttribute = ppAttributes[ i ];
+		Assert( pAttribute );
+
+		buf.PutShort( symbolToIndexMap[ pAttribute->GetNameSymbol() ] );
+		buf.PutChar( pAttribute->GetType() );
+		switch( pAttribute->GetType() )
+		{
+		default:
+ 			pAttribute->Serialize( buf );
+			break;
+
+		case AT_ELEMENT:
+			SerializeElementAttribute( buf, list, pAttribute );
+			break;
+
+		case AT_ELEMENT_ARRAY:
+			SerializeElementArrayAttribute( buf, list, pAttribute );
+			break;
+		}
+	}
+
+	return buf.IsValid();
+}
+
+
+bool CDmSerializerBinary::SaveElement( CUtlBuffer& buf, CDmElementSerializationDictionary& list, unsigned short *symbolToIndexMap, CDmElement *pElement )
+{
+	SerializeAttributes( buf, list, symbolToIndexMap, pElement );
+	return buf.IsValid();
+}
+
+bool CDmSerializerBinary::SaveElementDict( CUtlBuffer& buf, unsigned short *symbolToIndexMap, CDmElement *pElement )
+{
+	buf.PutShort( symbolToIndexMap[ pElement->GetType() ] );
+	buf.PutString( pElement->GetName() );
+	buf.Put( &pElement->GetId(), sizeof(DmObjectId_t) );
+	return buf.IsValid();
+}
+
+void MarkSymbol( UtlSymId_t *indexToSymbolMap, unsigned short *symbolToIndexMap, unsigned short &nSymbols, UtlSymId_t sym )
+{
+	if ( symbolToIndexMap[ sym ] != 0xffff )
+		return;
+
+	symbolToIndexMap[ sym ] = nSymbols;
+	indexToSymbolMap[ nSymbols ] = sym;
+	nSymbols++;
+}
+
+void MarkSymbols( UtlSymId_t *indexToSymbolMap, unsigned short *symbolToIndexMap, unsigned short &nSymbols, CDmElement *pElement )
+{
+	MarkSymbol( indexToSymbolMap, symbolToIndexMap, nSymbols, pElement->GetType() );
+	for ( CDmAttribute *pAttr = pElement->FirstAttribute(); pAttr; pAttr = pAttr->NextAttribute() )
+	{
+		MarkSymbol( indexToSymbolMap, symbolToIndexMap, nSymbols, pAttr->GetNameSymbol() );
+	}
+}
+
+bool CDmSerializerBinary::Serialize( CUtlBuffer &outBuf, CDmElement *pRoot )
+{
+	// Save elements, attribute links
+	CDmElementSerializationDictionary dict;
+	dict.BuildElementList( pRoot, true );
+
+	// TODO - consider allowing dmxconvert to skip the collection step, since the only datamodel symbols will be the ones from the file
+
+	unsigned short nTotalSymbols = g_pDataModelImp->GetSymbolCount();
+	UtlSymId_t     *indexToSymbolMap = ( UtlSymId_t    * )stackalloc( nTotalSymbols * sizeof( UtlSymId_t ) );
+	unsigned short *symbolToIndexMap = ( unsigned short* )stackalloc( nTotalSymbols * sizeof( unsigned short ) );
+	V_memset( indexToSymbolMap, 0xff, nTotalSymbols * sizeof( UtlSymId_t ) );
+	V_memset( symbolToIndexMap, 0xff, nTotalSymbols * sizeof( unsigned short ) );
+
+	// collect list of attribute names and element types into string table
+	unsigned short nUsedSymbols = 0;
+	DmElementDictHandle_t i;
+	for ( i = dict.FirstRootElement(); i != ELEMENT_DICT_HANDLE_INVALID; i = dict.NextRootElement(i) )
+	{
+		MarkSymbols( indexToSymbolMap, symbolToIndexMap, nUsedSymbols, dict.GetRootElement( i ) );
+	}
+	Assert( nUsedSymbols <= nTotalSymbols );
+
+	// write out the symbol table for this file (may be significantly smaller than datamodel's full symbol table)
+	outBuf.PutShort( nUsedSymbols );
+	for ( int si = 0; si < nUsedSymbols; ++si )
+	{
+		UtlSymId_t sym = indexToSymbolMap[ si ];
+		const char *pStr = g_pDataModel->GetString( sym );
+		outBuf.PutString( pStr );
+	}
+
+	// First write out the dictionary of all elements (to avoid later stitching up in unserialize)
+	outBuf.PutInt( dict.RootElementCount() );
+	for ( i = dict.FirstRootElement(); i != ELEMENT_DICT_HANDLE_INVALID; i = dict.NextRootElement(i) )
+	{
+		SaveElementDict( outBuf, symbolToIndexMap, dict.GetRootElement( i ) );
+	}
+
+	// Now write out the attributes of each of those elements
+	for ( i = dict.FirstRootElement(); i != ELEMENT_DICT_HANDLE_INVALID; i = dict.NextRootElement(i) )
+	{
+		SaveElement( outBuf, dict, symbolToIndexMap, dict.GetRootElement( i ) );
+	}
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Reads an element index and converts it to a handle (local or external)
+//-----------------------------------------------------------------------------
+DmElementHandle_t CDmSerializerBinary::UnserializeElementIndex( CUtlBuffer &buf, CUtlVector<CDmElement*> &elementList )
+{
+	int nElementIndex = buf.GetInt();
+	Assert( nElementIndex < elementList.Count() );
+	if ( nElementIndex == ELEMENT_INDEX_EXTERNAL )
+	{
+		char idstr[ 40 ];
+		buf.GetString( idstr );
+		DmObjectId_t id;
+		UniqueIdFromString( &id, idstr, sizeof( idstr ) );
+		return g_pDataModelImp->FindOrCreateElementHandle( id );
+	}
+
+	Assert( nElementIndex >= 0 || nElementIndex == ELEMENT_INDEX_NULL );
+	if ( nElementIndex < 0 || !elementList[ nElementIndex ] )
+		return DMELEMENT_HANDLE_INVALID;
+
+	return elementList[ nElementIndex ]->GetHandle();
+}
+
+
+//-----------------------------------------------------------------------------
+// Reads an element attribute
+//-----------------------------------------------------------------------------
+void CDmSerializerBinary::UnserializeElementAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList )
+{
+	DmElementHandle_t hElement = UnserializeElementIndex( buf, elementList );
+	if ( !pAttribute )
+		return;
+
+	pAttribute->SetValue( hElement );
+}
+
+
+//-----------------------------------------------------------------------------
+// Reads an element array attribute
+//-----------------------------------------------------------------------------
+void CDmSerializerBinary::UnserializeElementArrayAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, CUtlVector<CDmElement*> &elementList )
+{
+	int nElementCount = buf.GetInt();
+
+	if ( !pAttribute || pAttribute->GetType() != AT_ELEMENT_ARRAY )
+	{
+		// Parse past the data
+		for ( int i = 0; i < nElementCount; ++i )
+		{
+			UnserializeElementIndex( buf, elementList );
+		}
+		return;
+	}
+
+	CDmrElementArray<> array( pAttribute );
+	array.RemoveAll();
+	array.EnsureCapacity( nElementCount );
+	for ( int i = 0; i < nElementCount; ++i )
+	{
+		DmElementHandle_t hElement = UnserializeElementIndex( buf, elementList );
+		array.AddToTail( hElement );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Reads a single element
+//-----------------------------------------------------------------------------
+bool CDmSerializerBinary::UnserializeAttributes( CUtlBuffer &buf, CDmElement *pElement, CUtlVector<CDmElement*> &elementList, UtlSymId_t *symbolTable )
+{
+	char nameBuf[ 1024 ];
+
+	int nAttributeCount = buf.GetInt();
+	for ( int i = 0; i < nAttributeCount; ++i )
+	{
+		const char *pName = NULL;
+		if ( symbolTable )
+		{
+			unsigned short nName = buf.GetShort();
+			pName = g_pDataModel->GetString( symbolTable[ nName ] );
+		}
+		else
+		{
+			buf.GetString( nameBuf );
+			pName = nameBuf;
+		}
+		DmAttributeType_t nAttributeType = (DmAttributeType_t)buf.GetChar();
+
+		Assert( pName != NULL && pName[ 0 ] != '\0' );
+		Assert( nAttributeType != AT_UNKNOWN );
+
+		CDmAttribute *pAttribute = pElement ? pElement->AddAttribute( pName, nAttributeType ) : NULL;
+		if ( pElement && !pAttribute )
+		{
+			Warning("Dm: Attempted to read an attribute (\"%s\") of an inappropriate type!\n", pName );
+			return false;
+		}
+
+		switch( nAttributeType )
+		{
+		default:
+			if ( !pAttribute )
+			{
+				SkipUnserialize( buf, nAttributeType );
+			}
+			else
+			{
+				pAttribute->Unserialize( buf );
+			}
+			break;
+
+		case AT_ELEMENT:
+			UnserializeElementAttribute( buf, pAttribute, elementList );
+			break;
+
+		case AT_ELEMENT_ARRAY:
+			UnserializeElementArrayAttribute( buf, pAttribute, elementList );
+			break;
+		}
+	}
+
+	return buf.IsValid();
+}
+
+struct DmIdPair_t
+{
+	DmObjectId_t m_oldId;
+	DmObjectId_t m_newId;
+	DmIdPair_t &operator=( const DmIdPair_t &that )
+	{
+		CopyUniqueId( that.m_oldId, &m_oldId );
+		CopyUniqueId( that.m_newId, &m_newId );
+		return *this;
+	}
+	static unsigned int HashKey( const DmIdPair_t& that )
+	{
+		return *( unsigned int* )&that.m_oldId.m_Value;
+	}
+	static bool Compare( const DmIdPair_t& a, const DmIdPair_t& b )
+	{
+		return IsUniqueIdEqual( a.m_oldId, b.m_oldId );
+	}
+};
+
+DmElementHandle_t CreateElementWithFallback( const char *pType, const char *pName, DmFileId_t fileid, const DmObjectId_t &id )
+{
+	DmElementHandle_t hElement = g_pDataModel->CreateElement( pType, pName, fileid, &id );
+	if ( hElement == DMELEMENT_HANDLE_INVALID )
+	{
+		Warning("Binary: Element uses unknown element type %s\n", pType );
+		hElement = g_pDataModel->CreateElement( "DmElement", pName, fileid, &id );
+		Assert( hElement != DMELEMENT_HANDLE_INVALID );
+	}
+	return hElement;
+}
+
+//-----------------------------------------------------------------------------
+// Main entry point for the unserialization
+//-----------------------------------------------------------------------------
+bool CDmSerializerBinary::Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion,
+									   const char *pSourceFormatName, int nSourceFormatVersion,
+									   DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot )
+{
+	Assert( !V_stricmp( pEncodingName, GetName() ) );
+	if ( V_stricmp( pEncodingName, GetName() ) != 0 )
+		return false;
+
+	Assert( nEncodingVersion >= 0 && nEncodingVersion <= 2 );
+	if ( nEncodingVersion < 0 || nEncodingVersion > 2 )
+		return false;
+
+	bool bReadSymbolTable = nEncodingVersion >= 2;
+
+	// Read string table
+	unsigned short nStrings = 0;
+	UtlSymId_t *symbolTable = NULL;
+	if ( bReadSymbolTable )
+	{
+		char stringBuf[ 256 ];
+
+		nStrings = buf.GetShort();
+		symbolTable = ( UtlSymId_t* )stackalloc( nStrings * sizeof( UtlSymId_t ) );
+		for ( int i = 0; i < nStrings; ++i )
+		{
+			buf.GetString( stringBuf );
+			symbolTable[ i ] = g_pDataModel->GetSymbol( stringBuf );
+		}
+	}
+
+	bool bSuccess = UnserializeElements( buf, fileid, idConflictResolution, ppRoot, symbolTable );
+	if ( !bSuccess )
+		return false;
+
+	return g_pDataModel->UpdateUnserializedElements( pSourceFormatName, nSourceFormatVersion, fileid, idConflictResolution, ppRoot );
+}
+
+bool CDmSerializerBinary::UnserializeElements( CUtlBuffer &buf, DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot, UtlSymId_t *symbolTable )
+{
+	*ppRoot = NULL;
+
+	// Read in the element count.
+	int nElementCount = buf.GetInt();
+	if ( !nElementCount )
+		return true;
+
+	int nMaxIdConflicts = min( nElementCount, g_pDataModel->GetAllocatedElementCount() );
+	int nExpectedIdCopyConflicts = ( idConflictResolution == CR_FORCE_COPY || idConflictResolution == CR_COPY_NEW ) ? nMaxIdConflicts : 0;
+	int nBuckets = min( 0x20000, max( 64, nExpectedIdCopyConflicts / 64 ) ); // CUtlHash can only address up to 65k buckets
+	CUtlHash< DmIdPair_t > idmap( nBuckets, 0, 0, DmIdPair_t::Compare, DmIdPair_t::HashKey );
+
+	// Read + create all elements
+	CUtlVector<CDmElement*> elementList( 0, nElementCount );
+	for ( int i = 0; i < nElementCount; ++i )
+	{
+		char pName[2048];
+		DmObjectId_t id;
+
+		char typeBuf[ 256 ];
+		const char *pType = NULL;
+		if ( symbolTable )
+		{
+			unsigned short nType = buf.GetShort();
+			pType = g_pDataModel->GetString( symbolTable[ nType ] );
+		}
+		else
+		{
+			buf.GetString( typeBuf );
+			pType = typeBuf;
+		}
+
+		buf.GetString( pName );
+		buf.Get( &id, sizeof(DmObjectId_t) );
+
+		if ( idConflictResolution == CR_FORCE_COPY )
+		{
+			DmIdPair_t idpair;
+			CopyUniqueId( id, &idpair.m_oldId );
+			CreateUniqueId( &idpair.m_newId );
+			idmap.Insert( idpair );
+
+			CopyUniqueId( idpair.m_newId, &id );
+		}
+
+		DmElementHandle_t hElement = DMELEMENT_HANDLE_INVALID;
+		DmElementHandle_t hExistingElement = g_pDataModel->FindElement( id );
+		if ( hExistingElement != DMELEMENT_HANDLE_INVALID )
+		{
+			// id is already in use - need to resolve conflict
+
+			if ( idConflictResolution == CR_DELETE_NEW )
+			{
+				elementList.AddToTail( g_pDataModel->GetElement( hExistingElement ) );
+				continue; // just don't create this element
+			}
+			else if ( idConflictResolution == CR_DELETE_OLD )
+			{
+				g_pDataModelImp->DeleteElement( hExistingElement, HR_NEVER ); // keep the handle around until CreateElementWithFallback
+				hElement = CreateElementWithFallback( pType, pName, fileid, id );
+				Assert( hElement == hExistingElement );
+			}
+			else if ( idConflictResolution == CR_COPY_NEW )
+			{
+				DmIdPair_t idpair;
+				CopyUniqueId( id, &idpair.m_oldId );
+				CreateUniqueId( &idpair.m_newId );
+				idmap.Insert( idpair );
+
+				hElement = CreateElementWithFallback( pType, pName, fileid, idpair.m_newId );
+			}
+			else
+				Assert( 0 );
+		}
+
+		// if not found, then create it
+		if ( hElement == DMELEMENT_HANDLE_INVALID )
+		{
+			hElement = CreateElementWithFallback( pType, pName, fileid, id );
+		}
+
+		CDmElement *pElement = g_pDataModel->GetElement( hElement );
+		CDmeElementAccessor::MarkBeingUnserialized( pElement, true );
+		elementList.AddToTail( pElement );
+	}
+
+	// The root is the 0th element
+	*ppRoot = elementList[ 0 ];
+
+	// Now read all attributes
+	for ( int i = 0; i < nElementCount; ++i )
+	{
+		CDmElement *pInternal = elementList[ i ];
+		UnserializeAttributes( buf, pInternal->GetFileId() == fileid ? pInternal : NULL, elementList, symbolTable );
+	}
+
+	for ( int i = 0; i < nElementCount; ++i )
+	{
+		CDmElement *pElement = elementList[ i ];
+		if ( pElement->GetFileId() == fileid )
+		{
+			// mark all unserialized elements as done unserializing, and call Resolve()
+			CDmeElementAccessor::MarkBeingUnserialized( pElement, false );
+		}
+	}
+
+	g_pDmElementFrameworkImp->RemoveCleanElementsFromDirtyList( );
+	return buf.IsValid();
+}

datamodel/dmserializerbinary.h

@@ -0,0 +1,27 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: Outputs directly into a binary format
+//
+//=============================================================================
+
+#ifndef DMSERIALIZERBINARY_H
+#define DMSERIALIZERBINARY_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class IDataModel;
+
+
+//-----------------------------------------------------------------------------
+// Installation methods for standard serializers
+//-----------------------------------------------------------------------------
+void InstallBinarySerializer( IDataModel *pFactory );
+
+
+#endif // DMSERIALIZERBINARY_H

datamodel/dmserializerkeyvalues.cpp

@@ -0,0 +1,466 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializerkeyvalues.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel.h"
+#include "datamodel/dmelement.h"
+#include "datamodel/dmattributevar.h"
+#include "dmattributeinternal.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/utlvector.h"
+#include <limits.h>
+#include "DmElementFramework.h"
+
+// memdbgon must be the last include file in a .cpp file!!!
+#include "tier0/memdbgon.h"
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class CUtlBuffer;
+class CBaseSceneObject;
+
+
+//-----------------------------------------------------------------------------
+// Used to remap keyvalues names
+//-----------------------------------------------------------------------------
+struct AttributeRemap_t
+{
+	const char *m_pKeyValuesName;
+	const char *m_pDmeName;
+};
+
+static AttributeRemap_t s_pAttributeRemap[] = 
+{
+	{ "type", "_type" }, // FIXME - remove this once we've made type no longer be an attribute
+	{ "name", "_name" },
+	{ "id", "_id" }, // FIXME - remove this once we've made id no longer be an attribute
+	{ NULL, NULL }
+};
+
+
+//-----------------------------------------------------------------------------
+// Serialization class for Key Values
+//-----------------------------------------------------------------------------
+class CDmSerializerKeyValues : public IDmSerializer
+{
+public:
+	// Inherited from IDMSerializer
+	virtual const char *GetName() const { return "keyvalues"; }
+	virtual const char *GetDescription() const { return "KeyValues"; }
+	virtual bool StoresVersionInFile() const { return false; }
+	virtual bool IsBinaryFormat() const { return false; }
+	virtual int GetCurrentVersion() const { return 0; } // doesn't store a version
+	virtual bool Serialize( CUtlBuffer &buf, CDmElement *pRoot );
+	virtual bool Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion,
+							  const char *pSourceFormatName, int nSourceFormatVersion,
+							  DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot );
+
+private:
+	// Methods related to serialization
+	void SerializeSubKeys( CUtlBuffer& buf, CDmAttribute *pSubKeys );
+	bool SerializeAttributes( CUtlBuffer& buf, CDmElement *pElement );
+	bool SerializeElement( CUtlBuffer& buf, CDmElement *pElement );
+
+	// Methods related to unserialization
+	DmElementHandle_t UnserializeElement( KeyValues *pKeyValues, int iNestingLevel );
+	void UnserializeAttribute( CDmElement *pElement, KeyValues *pKeyValues );
+	DmElementHandle_t CreateDmElement( const char *pElementType, const char *pElementName );
+	CDmElement* UnserializeFromKeyValues( KeyValues *pKeyValues );
+
+	// Deterimines the attribute type of a keyvalue
+	DmAttributeType_t DetermineAttributeType( KeyValues *pKeyValues );
+
+	// For unserialization
+	CUtlVector<DmElementHandle_t> m_ElementList;
+	DmElementHandle_t m_hRoot;
+	DmFileId_t m_fileid;
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CDmSerializerKeyValues s_DMSerializerKeyValues;
+
+void InstallKeyValuesSerializer( IDataModel *pFactory )
+{
+	pFactory->AddSerializer( &s_DMSerializerKeyValues );
+}
+
+
+//-----------------------------------------------------------------------------
+// Serializes a single element attribute
+//-----------------------------------------------------------------------------
+void CDmSerializerKeyValues::SerializeSubKeys( CUtlBuffer& buf, CDmAttribute *pSubKeys )
+{
+	CDmrElementArray<> array( pSubKeys );
+	int c = array.Count();
+	for ( int i = 0; i < c; ++i )
+	{
+		CDmElement *pChild = array[i];
+		if ( pChild )
+		{
+			SerializeElement( buf, pChild );
+		}
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Serializes all attributes in an element
+//-----------------------------------------------------------------------------
+bool CDmSerializerKeyValues::SerializeAttributes( CUtlBuffer& buf, CDmElement *pElement )
+{
+	// Collect the attributes to be written
+	CDmAttribute **ppAttributes = ( CDmAttribute** )_alloca( pElement->AttributeCount() * sizeof( CDmAttribute* ) );
+	int nAttributes = 0;
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->IsFlagSet( FATTRIB_DONTSAVE | FATTRIB_STANDARD ) )
+			continue;
+
+		ppAttributes[ nAttributes++ ] = pAttribute;
+	}
+
+	// Now write them all out in reverse order, since FirstAttribute is actually the *last* attribute for perf reasons
+	for ( int i = nAttributes - 1; i >= 0; --i )
+	{
+		CDmAttribute *pAttribute = ppAttributes[ i ];
+		Assert( pAttribute );
+
+		const char *pName = pAttribute->GetName();
+
+		// Rename "_type", "_name", or "_id" fields, since they are special fields
+		for ( int iAttr = 0; s_pAttributeRemap[i].m_pKeyValuesName; ++i )
+		{
+			if ( !Q_stricmp( pName, s_pAttributeRemap[iAttr].m_pDmeName ) )
+			{
+				pName = s_pAttributeRemap[iAttr].m_pKeyValuesName;
+				break;
+			}
+		}
+
+  		DmAttributeType_t nAttrType = pAttribute->GetType();
+		if ( ( nAttrType ==  AT_ELEMENT_ARRAY ) && !Q_stricmp( pName, "subkeys" ) )
+		{
+			SerializeSubKeys( buf, pAttribute );
+			continue;
+		}
+
+		buf.Printf( "\"%s\" ", pName );
+
+		switch( nAttrType )
+		{
+		case AT_VOID:
+		case AT_STRING_ARRAY:
+		case AT_VOID_ARRAY:
+		case AT_ELEMENT:
+		case AT_ELEMENT_ARRAY:
+			Warning("KeyValues: Can't serialize attribute of type %s into KeyValues files!\n", 
+				g_pDataModel->GetAttributeNameForType( nAttrType ) );
+			buf.PutChar( '\"' );
+			buf.PutChar( '\"' );
+			break;
+
+		case AT_FLOAT:
+		case AT_INT:
+		case AT_BOOL:
+			pAttribute->Serialize( buf );
+			break;
+
+		case AT_VECTOR4:
+		case AT_VECTOR3:
+		case AT_VECTOR2:
+		case AT_STRING:
+		default:
+			buf.PutChar( '\"' );
+			buf.PushTab();
+			pAttribute->Serialize( buf );
+			buf.PopTab();
+			buf.PutChar( '\"' );
+			break;
+		}
+
+ 		buf.PutChar( '\n' );
+	}
+
+	return true;
+}
+
+bool CDmSerializerKeyValues::SerializeElement( CUtlBuffer& buf, CDmElement *pElement )
+{
+	buf.Printf( "\"%s\"\n{\n", pElement->GetName() );
+	buf.PushTab();
+	SerializeAttributes( buf, pElement );
+	buf.PopTab();
+	buf.Printf( "}\n" );
+	return true;
+}
+
+bool CDmSerializerKeyValues::Serialize( CUtlBuffer &outBuf, CDmElement *pRoot )
+{
+	if ( !pRoot )
+		return true;
+
+	CDmAttribute* pSubKeys = pRoot->GetAttribute( "subkeys" );
+	if ( !pSubKeys )
+		return true;
+
+	//SetSerializationDelimiter( GetCStringCharConversion() );
+	SerializeSubKeys( outBuf, pSubKeys );
+	//SetSerializationDelimiter( NULL );
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Creates a scene object, adds it to the element dictionary
+//-----------------------------------------------------------------------------
+DmElementHandle_t CDmSerializerKeyValues::CreateDmElement( const char *pElementType, const char *pElementName )
+{
+	// See if we can create an element of that type
+	DmElementHandle_t hElement = g_pDataModel->CreateElement( pElementType, pElementName, m_fileid );
+	if ( hElement == DMELEMENT_HANDLE_INVALID )
+	{
+		Warning("KeyValues: Element uses unknown element type %s\n", pElementType );
+		return DMELEMENT_HANDLE_INVALID;
+	}
+
+	m_ElementList.AddToTail( hElement );
+
+	CDmElement *pElement = g_pDataModel->GetElement( hElement );
+	CDmeElementAccessor::MarkBeingUnserialized( pElement, true );
+	return hElement;
+}
+
+
+//-----------------------------------------------------------------------------
+// Deterimines the attribute type of a keyvalue
+//-----------------------------------------------------------------------------
+DmAttributeType_t CDmSerializerKeyValues::DetermineAttributeType( KeyValues *pKeyValues )
+{
+	// FIXME: Add detection of vectors/matrices?
+	switch( pKeyValues->GetDataType() )
+	{
+	default:
+	case KeyValues::TYPE_NONE:
+		Assert( 0 );
+		return AT_UNKNOWN;
+
+	case KeyValues::TYPE_STRING:
+		{
+			float f1, f2, f3, f4;
+			if ( sscanf( pKeyValues->GetString(), "%f %f %f %f", &f1, &f2, &f3, &f4 ) == 4 )
+				return AT_VECTOR4;
+			if ( sscanf( pKeyValues->GetString(), "%f %f %f",&f1, &f2, &f3 ) == 3 )
+				return AT_VECTOR3;
+			if ( sscanf( pKeyValues->GetString(), "%f %f", &f1, &f2 ) == 2 )
+				return AT_VECTOR2;
+
+			int i = pKeyValues->GetInt( NULL, INT_MAX );
+			if ( ( sscanf( pKeyValues->GetString(), "%d", &i ) == 1 ) && 
+				 ( !strchr( pKeyValues->GetString(), '.' ) ) )
+				return AT_INT;
+
+			if ( sscanf( pKeyValues->GetString(), "%f", &f1 ) == 1 )
+				return AT_FLOAT;
+
+			return AT_STRING;
+		}
+
+	case KeyValues::TYPE_INT:
+		return AT_INT;
+
+	case KeyValues::TYPE_FLOAT:
+		return AT_FLOAT;
+
+	case KeyValues::TYPE_PTR:
+		return AT_VOID;
+
+	case KeyValues::TYPE_COLOR:
+		return AT_COLOR;
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Reads an attribute for an element
+//-----------------------------------------------------------------------------
+void CDmSerializerKeyValues::UnserializeAttribute( CDmElement *pElement, KeyValues *pKeyValues )
+{
+	// It's an attribute
+	const char *pAttributeName = pKeyValues->GetName();
+	const char *pAttributeValue = pKeyValues->GetString();
+
+	// Convert to lower case
+	CUtlString pLowerName = pAttributeName;
+	pLowerName.ToLower();
+
+	// Rename "type", "name", or "id" fields, since they are special fields
+	for ( int i = 0; s_pAttributeRemap[i].m_pKeyValuesName; ++i )
+	{
+		if ( !Q_stricmp( pLowerName, s_pAttributeRemap[i].m_pKeyValuesName ) )
+		{
+			pLowerName = s_pAttributeRemap[i].m_pDmeName;
+			break;
+		}
+	}
+
+	// Element types are stored out by GUID, we need to hang onto the guid and 
+	// link it back up once all elements have been loaded from the file
+	DmAttributeType_t type = DetermineAttributeType( pKeyValues );
+
+	// In this case, we have an inlined element or element array attribute
+	if ( type == AT_UNKNOWN )
+	{
+		// Assume this is an empty attribute or attribute array element
+		Warning("Dm Unserialize: Attempted to read an attribute (\"%s\") of an inappropriate type!\n", pLowerName.Get() );
+		return;
+	}
+
+	CDmAttribute *pAttribute = pElement->AddAttribute( pLowerName, type );
+	if ( !pAttribute )
+	{
+		Warning("Dm Unserialize: Attempted to read an attribute (\"%s\") of an inappropriate type!\n", pLowerName.Get() );
+		return;
+	}
+
+	switch( type )
+	{
+	case AT_STRING:
+		{
+			// Strings have different delimiter rules for KeyValues, 
+			// so let's just directly copy the string instead of going through unserialize
+			pAttribute->SetValue( pAttributeValue );
+		}
+		break;
+	
+	default:
+		{
+			int nLen = Q_strlen( pAttributeValue );
+			CUtlBuffer buf( pAttributeValue, nLen, CUtlBuffer::TEXT_BUFFER | CUtlBuffer::READ_ONLY ); 
+			pAttribute->Unserialize( buf );
+		}
+		break;
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Reads a single element
+//-----------------------------------------------------------------------------
+DmElementHandle_t CDmSerializerKeyValues::UnserializeElement( KeyValues *pKeyValues, int iNestingLevel )
+{
+	const char *pElementName = pKeyValues->GetName( );
+	const char *pszKeyValuesElement = g_pDataModel->GetKeyValuesElementName( pElementName, iNestingLevel );
+	if ( !pszKeyValuesElement )
+	{
+		pszKeyValuesElement = "DmElement";
+	}
+
+	DmElementHandle_t handle = CreateDmElement( pszKeyValuesElement, pElementName );
+	Assert( handle != DMELEMENT_HANDLE_INVALID );
+
+	iNestingLevel++;
+
+	CDmElement *pElement = g_pDataModel->GetElement( handle );
+	CDmrElementArray<> subKeys;
+	for ( KeyValues *pSub = pKeyValues->GetFirstSubKey(); pSub != NULL ; pSub = pSub->GetNextKey() )
+	{
+		// Read in a subkey
+		if ( pSub->GetDataType() == KeyValues::TYPE_NONE )
+		{
+			if ( !subKeys.IsValid() )
+			{
+				subKeys.Init( pElement->AddAttribute( "subkeys", AT_ELEMENT_ARRAY ) );
+			}
+
+			DmElementHandle_t hChild = UnserializeElement( pSub, iNestingLevel );
+			if ( hChild != DMELEMENT_HANDLE_INVALID )
+			{
+				subKeys.AddToTail( hChild );
+			}
+		}
+		else
+		{
+			UnserializeAttribute( pElement, pSub );
+		}
+	}
+
+	return handle;
+}
+
+
+//-----------------------------------------------------------------------------
+// Main entry point for the unserialization
+//-----------------------------------------------------------------------------
+CDmElement* CDmSerializerKeyValues::UnserializeFromKeyValues( KeyValues *pKeyValues )
+{
+	m_ElementList.RemoveAll();
+
+	m_hRoot = CreateDmElement( "DmElement", "root" );
+ 	CDmElement *pRoot = g_pDataModel->GetElement( m_hRoot );
+	CDmrElementArray<> subkeys( pRoot->AddAttribute( "subkeys", AT_ELEMENT_ARRAY ) );
+
+	int iNestingLevel = 0;
+
+	for ( KeyValues *pElementKey = pKeyValues; pElementKey != NULL; pElementKey = pElementKey->GetNextKey() )
+	{
+		DmElementHandle_t hChild = UnserializeElement( pElementKey, iNestingLevel );
+		if ( hChild != DMELEMENT_HANDLE_INVALID )
+		{
+			subkeys.AddToTail( hChild );
+		}
+	}
+
+	// mark all unserialized elements as done unserializing, and call Resolve()
+	int c = m_ElementList.Count();
+	for ( int i = 0; i < c; ++i )
+	{
+		CDmElement *pElement = g_pDataModel->GetElement( m_ElementList[i] );
+		CDmeElementAccessor::MarkBeingUnserialized( pElement, false );
+	}
+
+	g_pDmElementFrameworkImp->RemoveCleanElementsFromDirtyList( );
+	m_ElementList.RemoveAll();
+    return pRoot;
+}
+
+	
+//-----------------------------------------------------------------------------
+// Main entry point for the unserialization
+//-----------------------------------------------------------------------------
+bool CDmSerializerKeyValues::Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion,
+										  const char *pSourceFormatName, int nSourceFormatVersion,
+										  DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot )
+{
+	Assert( !V_stricmp( pEncodingName, "keyvalues" ) );
+
+	*ppRoot = NULL;
+
+	KeyValues *kv = new KeyValues( "keyvalues file" );
+	if ( !kv )
+		return false;
+
+	m_fileid = fileid;
+
+	bool bOk = kv->LoadFromBuffer( "keyvalues file", buf );
+	if ( bOk )
+	{
+		//SetSerializationDelimiter( GetCStringCharConversion() );
+		*ppRoot = UnserializeFromKeyValues( kv );
+		//SetSerializationDelimiter( NULL );
+	}
+
+	m_fileid = DMFILEID_INVALID;
+
+	kv->deleteThis();
+	return bOk;
+}

datamodel/dmserializerkeyvalues.h

@@ -0,0 +1,27 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef DMSERIALIZERKEYVALUES_H
+#define DMSERIALIZERKEYVALUES_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class IDataModel;
+
+
+//-----------------------------------------------------------------------------
+// Installation methods for standard serializers
+//-----------------------------------------------------------------------------
+void InstallKeyValuesSerializer( IDataModel *pFactory );
+
+
+#endif // DMSERIALIZER_H

datamodel/dmserializerkeyvalues2.h

@@ -0,0 +1,27 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef DMSERIALIZERKEYVALUES2_H
+#define DMSERIALIZERKEYVALUES2_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class IDataModel;
+
+
+//-----------------------------------------------------------------------------
+// Installation methods for standard serializers
+//-----------------------------------------------------------------------------
+void InstallKeyValues2Serializer( IDataModel *pFactory );
+
+
+#endif // DMSERIALIZERKEYVALUES2_H

datamodel/undomanager.cpp

@@ -0,0 +1,444 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "undomanager.h"
+#include "datamodel.h"
+
+extern CDataModel *g_pDataModelImp;
+
+CUtlSymbolTableMT CUndoManager::s_UndoSymbolTable;
+
+
+CUndoManager::CUndoManager( ) :
+	m_bEnabled( true ),
+	m_bDiscarded( false ),
+	m_nMaxUndoDepth( 4096 ),
+	m_nNesting( 0 ),
+	m_nNotifyNesting( 0 ),
+	m_bStreamStart( false ),
+	m_bTrace( false ),
+	m_bSuppressingNotify( false ),
+	m_nItemsAddedSinceStartOfStream( 0 ),
+	m_nNotifySource( 0 ),
+	m_nNotifyFlags( 0 ),
+	m_nChainingID( 0 ),
+	m_PreviousChainingID( 0 )
+{
+}
+
+CUndoManager::~CUndoManager()
+{
+}
+
+void CUndoManager::Shutdown()
+{
+	WipeUndo();
+	WipeRedo();
+}
+
+bool CUndoManager::InstallNotificationCallback( IDmNotify *pNotify )
+{
+	if ( m_Notifiers.Find( pNotify ) >= 0 )
+		return false;
+	m_Notifiers.AddToTail( pNotify );
+	return true;
+}
+
+void CUndoManager::RemoveNotificationCallback( IDmNotify *pNotify )
+{
+	m_Notifiers.FindAndRemove( pNotify );
+}
+
+bool CUndoManager::IsSuppressingNotify( ) const
+{
+	return m_bSuppressingNotify;
+}
+
+void CUndoManager::SetSuppressingNotify( bool bSuppress )
+{
+	m_bSuppressingNotify = bSuppress;
+}
+
+void CUndoManager::Trace( const char *fmt, ... )
+{
+	if ( !m_bTrace )
+		return;
+
+	char str[ 2048 ];
+	va_list argptr;
+	va_start( argptr, fmt );
+	_vsnprintf( str, sizeof( str ) - 1, fmt, argptr );
+	va_end( argptr );
+	str[ sizeof( str ) - 1 ] = 0;
+
+	char spaces[ 128 ];
+	Q_memset( spaces, 0, sizeof( spaces ) );
+	for ( int i = 0; i < ( m_nNesting * 3 ); ++i )
+	{
+		if ( i >= 127 )
+			break;
+		spaces[ i ] = ' ';
+	}
+
+	Msg( "%s%s", spaces, str );
+}
+
+void CUndoManager::SetUndoDepth( int nMaxUndoDepth )
+{
+	Assert( !HasUndoData() );
+	m_nMaxUndoDepth = nMaxUndoDepth;
+}
+
+void CUndoManager::EnableUndo()
+{
+	m_bEnabled = true;
+}
+
+void CUndoManager::DisableUndo()
+{
+	m_bEnabled = false;
+}
+
+bool CUndoManager::HasUndoData() const
+{
+	return m_UndoList.Count() != 0;
+}
+
+bool CUndoManager::UndoDataDiscarded() const
+{
+	return m_bDiscarded;
+}
+
+bool CUndoManager::HasRedoData() const
+{
+	return m_RedoStack.Count() > 0;
+}
+
+void CUndoManager::PushNotificationScope( const char *pReason, int nNotifySource, int nNotifyFlags )
+{
+	if ( m_nNotifyNesting++ == 0 )
+	{
+		m_pNotifyReason = pReason;
+		m_nNotifySource = nNotifySource;
+		m_nNotifyFlags = nNotifyFlags;
+	}
+}
+
+void CUndoManager::PopNotificationScope( bool bAbort )
+{
+	--m_nNotifyNesting;
+	Assert( m_nNotifyNesting >= 0 );
+	if ( m_nNotifyNesting == 0 )
+	{
+		if ( !m_bSuppressingNotify && ( ( m_nNotifyFlags & NOTIFY_CHANGE_MASK ) != 0 ) )
+		{
+			int nNotifyCount = m_Notifiers.Count();
+			for( int i = 0; i < nNotifyCount; ++i )
+			{
+				m_Notifiers[i]->NotifyDataChanged( m_pNotifyReason, m_nNotifySource, m_nNotifyFlags );
+			}
+		}
+		m_nNotifySource = 0;
+		m_nNotifyFlags = 0;
+	}
+}
+
+
+
+void CUndoManager::PushUndo( const char *udesc, const char *rdesc, int nChainingID )
+{
+	if ( !IsEnabled() )
+		return;
+
+	Trace( "[%d] Pushing undo '%s'\n", m_nNesting + 1, udesc );
+
+	if ( m_nNesting++ == 0 )
+	{
+		m_PreviousChainingID = m_nChainingID;
+		m_nChainingID = nChainingID;
+		m_UndoDesc = s_UndoSymbolTable.AddString( udesc );
+		m_RedoDesc = ( udesc == rdesc ) ? m_UndoDesc : s_UndoSymbolTable.AddString( rdesc );
+		m_bStreamStart = true;
+		m_nItemsAddedSinceStartOfStream = 0;
+	}
+}
+
+void CUndoManager::PushRedo()
+{
+	if ( !IsEnabled() )
+		return;
+
+	Trace( "[%d] Popping undo '%s'\n", m_nNesting, s_UndoSymbolTable.String( m_UndoDesc ) );
+
+	--m_nNesting;
+	Assert( m_nNesting >= 0 );
+	if ( m_nNesting == 0 )
+	{
+		if ( m_nItemsAddedSinceStartOfStream > 0 )
+		{
+			WipeRedo();
+
+			// Accumulate this operation into the previous "undo" operation if there is one
+			if ( m_nChainingID != 0 &&
+				m_PreviousChainingID == m_nChainingID )
+			{
+				// Walk undo list backward looking for previous end of stream and unmark that indicator
+				int i = m_UndoList.Tail();
+				while ( i != m_UndoList.InvalidIndex() )
+				{
+					IUndoElement *e = m_UndoList[ i ];
+					if ( e && e->IsEndOfStream() )
+					{
+						e->SetEndOfStream( false );
+						break;
+					}
+					i = m_UndoList.Previous( i );
+				}
+			}
+		}
+
+		m_nItemsAddedSinceStartOfStream = 0;
+	}
+}
+
+void CUndoManager::AbortUndoableOperation()
+{
+	if ( !IsEnabled() )
+		return;
+
+	bool hasItems = m_nItemsAddedSinceStartOfStream > 0 ? true : false;
+	
+	Trace( "[%d] Aborting undo '%s'\n", m_nNesting, s_UndoSymbolTable.String( m_UndoDesc )  );
+
+	// Close off context
+	PushRedo();
+
+	if ( m_nNesting == 0 && hasItems )
+	{
+		Undo();
+		WipeRedo();
+	}
+}
+
+void CUndoManager::WipeUndo()
+{
+	CDisableUndoScopeGuard sg;
+
+	FOR_EACH_LL( m_UndoList, elem )
+	{
+		Trace( "WipeUndo '%s'\n", m_UndoList[ elem ]->GetDesc() );
+
+		m_UndoList[ elem ]->Release();
+	}
+	m_UndoList.RemoveAll();
+	m_PreviousChainingID = 0;
+}
+
+void CUndoManager::WipeRedo()
+{
+	int c = m_RedoStack.Count();
+	if ( c == 0 )
+		return;
+
+	CUtlVector< DmElementHandle_t > handles;
+	g_pDataModelImp->GetInvalidHandles( handles );
+	g_pDataModelImp->MarkHandlesValid( handles );
+
+	CDisableUndoScopeGuard sg;
+
+	for ( int i = 0; i < c ; ++i )
+	{
+		IUndoElement *elem;
+		elem = m_RedoStack[ i ];
+		
+		Trace( "WipeRedo '%s'\n", elem->GetDesc() );
+
+		elem->Release();
+	}
+
+	m_RedoStack.Clear();
+
+	g_pDataModelImp->MarkHandlesInvalid( handles );
+}
+
+void CUndoManager::AddUndoElement( IUndoElement *pElement )
+{
+	Assert( IsEnabled() );
+
+	if ( !pElement )
+		return;
+
+	++m_nItemsAddedSinceStartOfStream;
+
+	WipeRedo();
+
+	/*
+	// For later
+	if ( m_UndoList.Count() >= m_nMaxUndos )
+	{
+		m_bDiscarded = true;
+	}
+	*/
+	
+	Trace( "AddUndoElement '%s'\n", pElement->GetDesc() );
+
+	m_UndoList.AddToTail( pElement );
+
+	if ( m_bStreamStart )
+	{
+		pElement->SetEndOfStream( true );
+		m_bStreamStart = false;
+	}
+}
+
+void CUndoManager::Undo()
+{
+	CNotifyScopeGuard notify( "CUndoManager::Undo", NOTIFY_SOURCE_UNDO, NOTIFY_SETDIRTYFLAG );
+
+	Trace( "Undo\n======\n" );
+
+	bool saveEnabled = m_bEnabled;
+	m_bEnabled = false;
+	bool bEndOfStream = false;
+	while ( !bEndOfStream && m_UndoList.Count() > 0 )
+	{
+		int i = m_UndoList.Tail();
+		IUndoElement *action = m_UndoList[ i ];
+		Assert( action );
+
+		Trace( "  %s\n", action->GetDesc() );
+
+		action->Undo();
+		m_RedoStack.Push( action );
+		bEndOfStream = action->IsEndOfStream();
+		m_UndoList.Remove( i );
+	}
+
+	Trace( "======\n\n" );
+
+	m_bEnabled = saveEnabled;
+	m_PreviousChainingID = 0;
+}
+
+void CUndoManager::Redo()
+{
+	CNotifyScopeGuard notify( "CUndoManager::Redo", NOTIFY_SOURCE_UNDO, NOTIFY_SETDIRTYFLAG );
+
+	Trace( "Redo\n======\n" );
+
+	bool saveEnabled = m_bEnabled;
+	m_bEnabled = false;
+	bool bEndOfStream = false;
+	while ( !bEndOfStream && m_RedoStack.Count() > 0 )
+	{
+		IUndoElement *action = NULL;
+		m_RedoStack.Pop( action );
+		Assert( action );
+
+		Trace( "  %s\n", action->GetDesc() );
+
+		action->Redo();
+		m_UndoList.AddToTail( action );
+		if ( m_RedoStack.Count() > 0 )
+		{
+			action = m_RedoStack.Top();
+			bEndOfStream = action->IsEndOfStream();
+		}
+	}
+
+	Trace( "======\n\n" );
+
+	m_bEnabled = saveEnabled;
+	m_PreviousChainingID = 0;
+}
+
+const char *CUndoManager::UndoDesc() const
+{
+	if ( m_UndoList.Count() <= 0 )
+		return "";
+
+	int i = m_UndoList.Tail();
+	IUndoElement *action = m_UndoList[ i ];
+	return action->UndoDesc();
+}
+
+const char *CUndoManager::RedoDesc() const
+{
+	if ( m_RedoStack.Count() <= 0 )
+	{
+		return "";
+	}
+
+	IUndoElement *action = m_RedoStack.Top();
+	return action->RedoDesc();
+}
+
+UtlSymId_t CUndoManager::GetUndoDescInternal( const char *context )
+{
+	if ( m_nNesting <= 0 )
+	{
+		static CUtlSymbolTable s_DescErrorsTable;
+		static CUtlVector< CUtlSymbol >	s_DescErrors;
+		CUtlSymbol sym = s_DescErrorsTable.AddString( context );
+		if ( s_DescErrors.Find( sym ) == s_DescErrors.InvalidIndex() )
+		{
+			Warning( "CUndoManager::GetUndoDescInternal:  undoable operation missing CUndoScopeGuard in application\nContext( %s )\n", context );
+			s_DescErrors.AddToTail( sym );
+		}
+		return s_UndoSymbolTable.AddString( context );
+	}
+	return m_UndoDesc;
+}
+
+UtlSymId_t CUndoManager::GetRedoDescInternal( const char *context )
+{
+	if ( m_nNesting <= 0 )
+	{
+		// Warning( "CUndoManager::GetRedoDescInternal:  undoable operation missing CUndoScopeGuard in application\nContext( %s )", context );
+		return s_UndoSymbolTable.AddString( context );
+	}
+	return m_RedoDesc;
+}
+
+void CUndoManager::GetUndoInfo( CUtlVector< UndoInfo_t >& list )
+{
+	// Needs to persist after function returns...
+	static CUtlSymbolTable table;
+
+	int ops = 0;
+	for ( int i = m_UndoList.Tail(); i != m_UndoList.InvalidIndex(); i = m_UndoList.Previous( i ) )
+	{
+		++ops;
+		IUndoElement *action = m_UndoList[ i ];
+		Assert( action );
+		bool bEndOfStream = action->IsEndOfStream();
+
+		UndoInfo_t info;
+		info.undo = action->UndoDesc();
+		info.redo = action->RedoDesc();
+
+		// This is a hack because GetDesc() returns a static char buf[] and so the last one will clobber them all
+		// So we have the requester pass in a temporary string table so we can get a ptr to a CUtlSymbol in the table
+		// and use that.  Sigh.
+		const char *desc = action->GetDesc();
+		CUtlSymbol sym = table.AddString( desc );
+		info.desc = table.String( sym );
+		info.terminator = bEndOfStream;
+		info.numoperations = bEndOfStream ? ops : 1;
+
+		list.AddToTail( info );
+
+		if ( bEndOfStream )
+		{
+			ops = 0;
+		}
+	}
+}
+
+void CUndoManager::TraceUndo( bool state )
+{
+	m_bTrace = state;
+}

datamodel/undomanager.h

@@ -0,0 +1,124 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef UNDOMANAGER_H
+#define UNDOMANAGER_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "tier1/utlsymbol.h"
+#include "tier1/utllinkedlist.h"
+#include "tier1/utlstack.h"
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class IUndoElement;
+struct UndoInfo_t;
+class IDmNotify;
+
+
+//-----------------------------------------------------------------------------
+// Undo/Redo stuff:
+//-----------------------------------------------------------------------------
+class CUndoManager
+{
+public:
+	CUndoManager();
+	~CUndoManager();
+
+	void			Shutdown();
+	void			SetUndoDepth( int nMaxUndoDepth );
+	void			AddUndoElement( IUndoElement *pElement );
+	void			Undo();
+	void			Redo();
+	void			TraceUndo( bool state );
+
+	void			EnableUndo();
+	void			DisableUndo();
+	bool			IsEnabled() const;
+	bool			HasUndoData() const;
+	bool			UndoDataDiscarded() const;
+	bool			HasRedoData() const;
+
+	void			WipeUndo();
+	void			WipeRedo();
+
+	const char		*UndoDesc() const;
+	const char		*RedoDesc() const;
+
+	void			PushUndo( char const *udesc, char const *rdesc, int nChainingID );
+	void			PushRedo();
+	void			AbortUndoableOperation();
+
+	UtlSymId_t		GetUndoDescInternal( const char *context );
+	UtlSymId_t		GetRedoDescInternal( const char *context );
+
+	void			GetUndoInfo( CUtlVector< UndoInfo_t >& list );
+
+	bool			InstallNotificationCallback( IDmNotify *pNotify );
+	void			RemoveNotificationCallback( IDmNotify *pNotify );
+	bool			IsSuppressingNotify( ) const;
+	void			SetSuppressingNotify( bool bSuppress );
+	void			PushNotificationScope( const char *pReason, int nNotifySource, int nNotifyFlags );
+	void			PopNotificationScope( bool bAbort );
+
+	void			NotifyState( int nNotifyFlags );
+
+	static const char *GetUndoString( CUtlSymbol sym );
+
+private:
+	void			Trace( PRINTF_FORMAT_STRING const char *fmt, ... );
+
+	CUtlLinkedList< IUndoElement *, int >	m_UndoList;
+	CUtlStack< IUndoElement * >			m_RedoStack;
+	CUtlVector< IDmNotify* >			m_Notifiers;
+	int				m_nMaxUndoDepth;
+	int				m_nNesting;
+	int				m_nNotifyNesting;
+	CUtlSymbol		m_UndoDesc;
+	CUtlSymbol		m_RedoDesc;
+	int				m_nNotifySource;
+	int				m_nNotifyFlags;
+	const char*		m_pNotifyReason;
+	int				m_nItemsAddedSinceStartOfStream;
+	// Signals that next undo operation is the "Start" of a stream
+	bool			m_bStreamStart : 1;
+	bool			m_bTrace : 1;
+	bool			m_bDiscarded : 1;
+	bool			m_bEnabled : 1;
+	bool			m_bSuppressingNotify : 1;
+
+	static CUtlSymbolTableMT s_UndoSymbolTable;
+	int				m_nChainingID;
+	int				m_PreviousChainingID;
+};
+
+
+//-----------------------------------------------------------------------------
+// Is undo enabled
+//-----------------------------------------------------------------------------
+inline bool CUndoManager::IsEnabled() const
+{
+	return m_bEnabled;
+}
+
+inline void CUndoManager::NotifyState( int nNotifyFlags )
+{
+	// FIXME: Should suppress prevent notification being sent,
+	// or prevent notification flags from being set in the first place?
+	m_nNotifyFlags |= nNotifyFlags;
+}
+
+inline const char *CUndoManager::GetUndoString( CUtlSymbol sym )
+{
+	return s_UndoSymbolTable.String( sym );
+}
+
+
+#endif // UNDOMANAGER_H

dmserializers/dmebaseimporter.cpp

@@ -0,0 +1,42 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+
+CDmeBaseImporter::CDmeBaseImporter( char const *formatName, char const *nextFormatName ) :
+	m_pFormatName( formatName ),
+	m_pNextSerializer( nextFormatName )
+{
+}
+
+bool CDmeBaseImporter::IsLatestVersion() const
+{
+	return g_pDataModel->FindLegacyUpdater( m_pNextSerializer ) == NULL;
+}
+
+// Updates ppRoot to first non-legacy generic dmx format, returns false if the conversion fails
+bool CDmeBaseImporter::Update( CDmElement **ppRoot )
+{
+	if ( !DoFixup( *ppRoot ) )
+		return false;
+
+	if ( !m_pNextSerializer )
+		return true;
+
+	// Chain
+	IDmLegacyUpdater *pUpdater = g_pDataModel->FindLegacyUpdater( m_pNextSerializer );
+	if ( !pUpdater )
+		return true;
+
+	return pUpdater->Update( ppRoot );
+}
+
+
+
+CSFMBaseImporter::CSFMBaseImporter( char const *formatName, char const *nextFormatName ) :
+	BaseClass( formatName, nextFormatName )
+{
+}

dmserializers/dmebaseimporter.h

@@ -0,0 +1,43 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef DMEBASEIMPORTER_H
+#define DMEBASEIMPORTER_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "datamodel/idatamodel.h"
+
+class CDmeBaseImporter : public IDmLegacyUpdater
+{
+	typedef IDmLegacyUpdater BaseClass;
+
+public:
+	CDmeBaseImporter( char const *formatName, char const *nextFormatName );
+
+	virtual const char *GetName() const { return m_pFormatName; }
+	virtual bool IsLatestVersion() const;
+
+	virtual bool Update( CDmElement **ppRoot );
+
+private:
+	virtual bool DoFixup( CDmElement *pRoot ) = 0;
+
+protected:
+	char const *m_pFormatName;
+	char const *m_pNextSerializer;
+};
+
+class CSFMBaseImporter : public CDmeBaseImporter
+{
+	typedef CDmeBaseImporter BaseClass;
+
+public:
+	CSFMBaseImporter( char const *formatName, char const *nextFormatName );
+};
+
+#endif // DMEBASEIMPORTER_H

dmserializers/dmserializers.cpp

@@ -0,0 +1,163 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// The copyright to the contents herein is the property of Valve, L.L.C.
+// The contents may be used and/or copied only with the written permission of
+// Valve, L.L.C., or in accordance with the terms and conditions stipulated in
+// the agreement/contract under which the contents have been supplied.
+//
+// $Header: $
+// $NoKeywords: $
+//
+// Converts from any one DMX file format to another
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmserializers/idmserializers.h"
+#include "appframework/iappsystem.h"
+#include "filesystem.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelementfactoryhelper.h"
+#include "tier2/tier2.h"
+
+
+//-----------------------------------------------------------------------------
+// format updater macros
+//-----------------------------------------------------------------------------
+
+#define DECLARE_FORMAT_UPDATER( _name, _description, _extension, _version, _encoding ) \
+	class CDmFormatUpdater_ ## _name : public IDmFormatUpdater \
+	{ \
+	public: \
+		CDmFormatUpdater_ ## _name() {} \
+		virtual const char *GetName() const { return #_name; } \
+		virtual const char *GetDescription() const { return _description; } \
+		virtual const char *GetExtension() const { return _extension; } \
+		virtual const char *GetDefaultEncoding() const { return _encoding; } \
+		virtual int GetCurrentVersion() const { return _version; } \
+		virtual bool Update( CDmElement **pRoot, int nSourceVersion ) { return true; } \
+	}; \
+	static CDmFormatUpdater_ ## _name s_FormatUpdater ## _name; \
+	void InstallFormatUpdater_ ## _name( IDataModel *pFactory ) \
+	{ \
+		pFactory->AddFormatUpdater( &s_FormatUpdater ## _name ); \
+	}
+
+#define INSTALL_FORMAT_UPDATER( _name ) InstallFormatUpdater_ ## _name( g_pDataModel )
+
+
+//-----------------------------------------------------------------------------
+// format updaters
+//-----------------------------------------------------------------------------
+
+DECLARE_FORMAT_UPDATER( dmx,				"Generic DMX",					"dmx", 1, "binary" )
+DECLARE_FORMAT_UPDATER( movieobjects,		"Generic MovieObjects",			"dmx", 1, "binary" )
+DECLARE_FORMAT_UPDATER( sfm,				"Generic SFM",					"dmx", 1, "binary" )
+DECLARE_FORMAT_UPDATER( sfm_session,		"SFM Session",					"dmx", 1, "binary" )
+DECLARE_FORMAT_UPDATER( sfm_trackgroup,		"SFM TrackGroup",				"dmx", 1, "binary" )
+DECLARE_FORMAT_UPDATER( pcf,				"Particle Configuration File",	"dmx", 1, "binary" )
+DECLARE_FORMAT_UPDATER( preset,				"Preset File",					"dmx", 1, "keyvalues2" )
+DECLARE_FORMAT_UPDATER( facial_animation,	"Facial Animation File",		"dmx", 1, "binary" )
+DECLARE_FORMAT_UPDATER( model,				"DMX Model",					"dmx", 1, "binary" )
+//DECLARE_FORMAT_UPDATER( animation,			"DMX Animation",				"dmx", 1, "binary" )
+//DECLARE_FORMAT_UPDATER( dcc_makefile,			"DMX Makefile",					"dmx", 1, "keyvalues2" )
+
+
+//-----------------------------------------------------------------------------
+// The application object
+//-----------------------------------------------------------------------------
+class CDmSerializers : public CBaseAppSystem< IDmSerializers >
+{
+	typedef CBaseAppSystem< IDmSerializers > BaseClass;
+
+public:
+	// Inherited from IAppSystem 
+	virtual bool Connect( CreateInterfaceFn factory );
+	virtual void *QueryInterface( const char *pInterfaceName );
+	virtual InitReturnVal_t Init();
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton interface 
+//-----------------------------------------------------------------------------
+static CDmSerializers g_DmSerializers;
+IDmSerializers *g_pDmSerializers = &g_DmSerializers;
+
+
+//-----------------------------------------------------------------------------
+// Here's where the app systems get to learn about each other 
+//-----------------------------------------------------------------------------
+bool CDmSerializers::Connect( CreateInterfaceFn factory ) 
+{
+	if ( !BaseClass::Connect( factory ) )
+		return false;
+
+	if ( !factory( FILESYSTEM_INTERFACE_VERSION, NULL ) )
+	{
+		Warning( "DmSerializers needs the file system to function" );
+		return false;
+	}
+
+	// Here's the main point where all DM element classes get installed
+	// Necessary to do it here so all type symbols for all DME classes are set 
+	// up prior to install
+	InstallDmElementFactories( );
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Here's where systems can access other interfaces implemented by this object
+//-----------------------------------------------------------------------------
+void *CDmSerializers::QueryInterface( const char *pInterfaceName )
+{
+	if ( !V_strcmp( pInterfaceName, DMSERIALIZERS_INTERFACE_VERSION ) )
+		return (IDmSerializers*)this;
+
+	return NULL;
+}
+
+
+//-----------------------------------------------------------------------------
+// Init, shutdown
+//-----------------------------------------------------------------------------
+InitReturnVal_t CDmSerializers::Init() 
+{ 
+	InitReturnVal_t nRetVal = BaseClass::Init();
+	if ( nRetVal != INIT_OK )
+		return nRetVal;
+
+	// Install non-dmx importers
+	InstallActBusyImporter( g_pDataModel );
+	InstallVMTImporter( g_pDataModel );
+	InstallVMFImporter( g_pDataModel );
+
+	// Install legacy dmx importers
+	InstallSFMV1Importer( g_pDataModel );
+	InstallSFMV2Importer( g_pDataModel );
+	InstallSFMV3Importer( g_pDataModel );
+	InstallSFMV4Importer( g_pDataModel );
+	InstallSFMV5Importer( g_pDataModel );
+	InstallSFMV6Importer( g_pDataModel );
+	InstallSFMV7Importer( g_pDataModel );
+	InstallSFMV8Importer( g_pDataModel );
+	InstallSFMV9Importer( g_pDataModel );
+
+	// install dmx format updaters
+	INSTALL_FORMAT_UPDATER( dmx );
+	INSTALL_FORMAT_UPDATER( movieobjects );
+	INSTALL_FORMAT_UPDATER( sfm );
+	INSTALL_FORMAT_UPDATER( sfm_session );
+	INSTALL_FORMAT_UPDATER( sfm_trackgroup );
+	INSTALL_FORMAT_UPDATER( pcf );
+	INSTALL_FORMAT_UPDATER( preset );
+	INSTALL_FORMAT_UPDATER( facial_animation );
+	INSTALL_FORMAT_UPDATER( model );
+//	INSTALL_FORMAT_UPDATER( animation );
+//	INSTALL_FORMAT_UPDATER( dcc_makefile );
+
+	return INIT_OK; 
+}
+

dmserializers/dmserializers.h

@@ -0,0 +1,55 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// The copyright to the contents herein is the property of Valve, L.L.C.
+// The contents may be used and/or copied only with the written permission of
+// Valve, L.L.C., or in accordance with the terms and conditions stipulated in
+// the agreement/contract under which the contents have been supplied.
+//
+// $Header: $
+// $NoKeywords: $
+//
+// Main header file for the serializers DLL
+//
+//=============================================================================
+
+#ifndef DMSERIALIZERS_H
+#define DMSERIALIZERS_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "datamodel/dmelement.h"
+#include "datamodel/dmattribute.h"
+#include "datamodel/dmattributevar.h"
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class IDataModel;
+
+
+//-----------------------------------------------------------------------------
+// Externally defined importers
+//-----------------------------------------------------------------------------
+void InstallActBusyImporter( IDataModel *pFactory );
+void InstallVMTImporter( IDataModel *pFactory );
+void InstallSFMV1Importer( IDataModel *pFactory );
+void InstallSFMV2Importer( IDataModel *pFactory );
+void InstallSFMV3Importer( IDataModel *pFactory );
+void InstallSFMV4Importer( IDataModel *pFactory );
+void InstallSFMV5Importer( IDataModel *pFactory );
+void InstallSFMV6Importer( IDataModel *pFactory );
+void InstallSFMV7Importer( IDataModel *pFactory );
+void InstallSFMV8Importer( IDataModel *pFactory );
+void InstallSFMV9Importer( IDataModel *pFactory );
+void InstallVMFImporter( IDataModel *pFactory );
+
+void InstallDMXUpdater( IDataModel *pFactory );
+void InstallSFMSessionUpdater( IDataModel *pFactory );
+void InstallPCFUpdater( IDataModel *pFactory );
+
+
+#endif // DMSERIALIZERS_H
+
+

dmserializers/dmserializers.vpc

@@ -0,0 +1,46 @@
+//-----------------------------------------------------------------------------
+//	DMSERIALIZERS.VPC
+//
+//	Project Script
+//-----------------------------------------------------------------------------
+
+$Macro SRCDIR		".."
+$include "$SRCDIR\vpc_scripts\source_lib_base.vpc"
+
+$Configuration
+{
+	$Compiler
+	{
+		$PreprocessorDefinitions			"$BASE;DMSERIALIZERS_LIB"
+	}
+}
+
+$Project "Dmserializers"
+{
+	$Folder	"Source Files"
+	{
+		$File	"dmebaseimporter.cpp"
+		$File	"dmserializers.cpp"
+		$File	"importactbusy.cpp"
+		$File	"importkeyvaluebase.cpp"
+		$File	"importsfmv1.cpp"
+		$File	"importsfmv2.cpp"
+		$File	"importsfmv3.cpp"
+		$File	"importsfmv4.cpp"
+		$File	"importsfmv5.cpp"
+		$File	"importsfmv6.cpp"
+		$File	"importsfmv7.cpp"
+		$File	"importsfmv8.cpp"
+		$File	"importsfmv9.cpp"
+		$File	"importvmf.cpp"
+		$File	"importvmt.cpp"
+	}
+
+	$Folder	"Header Files"
+	{
+		$File	"dmebaseimporter.h"
+		$File	"dmserializers.h"
+		$File	"$SRCDIR\public\dmserializers\idmserializers.h"
+		$File	"importkeyvaluebase.h"
+	}
+}

dmserializers/importactbusy.cpp

@@ -0,0 +1,182 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "importkeyvaluebase.h"
+#include "dmserializers.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "datamodel/dmattribute.h"
+
+
+//-----------------------------------------------------------------------------
+// Serialization class for Key Values
+//-----------------------------------------------------------------------------
+class CImportActBusy : public CImportKeyValueBase
+{
+public:
+	virtual const char *GetName() const { return "actbusy"; }
+	virtual const char *GetDescription() const { return "ActBusy Script File"; }
+	virtual int GetCurrentVersion() const { return 0; } // doesn't store a version
+
+	bool Serialize( CUtlBuffer &outBuf, CDmElement *pRoot );
+	CDmElement* UnserializeFromKeyValues( KeyValues *pKeyValues );
+
+private:
+	// Reads a single element
+	bool UnserializeActBusyKey( CDmAttribute *pChildren, KeyValues *pKeyValues );
+
+	// Writes out the actbusy header
+	void SerializeHeader( CUtlBuffer &buf );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportActBusy s_ImportActBusy;
+
+void InstallActBusyImporter( IDataModel *pFactory )
+{
+	pFactory->AddSerializer( &s_ImportActBusy );
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out the actbusy header
+//-----------------------------------------------------------------------------
+void CImportActBusy::SerializeHeader( CUtlBuffer &buf )
+{
+	buf.Printf( "// \"act busy name\"\t\tThis is the name that a mapmaker must specify in the hint node.\n" );
+	buf.Printf( "// {\n" ); 
+	buf.Printf( "// \t\"busy_anim\"\t\t\t\"Activity Name\".\n" );
+	buf.Printf( "// \t\"entry_anim\"\t\t\"Activity Name\"\n" );
+	buf.Printf( "// \t\"exit_anim\"\t\t\t\"Activity Name\"\n" );
+	buf.Printf( "// \t\"busy_sequence\"\t\t\"Sequence Name\". If specified, this is used over the activity name. Specify it in the hint node.\n" );
+	buf.Printf( "// \t\"entry_sequence\"\t\"Sequence Name\". If specified, this is used over the entry anim.\n" );
+	buf.Printf( "// \t\"exit_sequence\"\t\t\"Sequence Name\". If specified, this is used over the exit anim.\n" );
+	buf.Printf( "// \t\"min_time\"\t\t\t\"Minimum time to spend in this busy anim\"\n" );
+	buf.Printf( "// \t\"max_time\"\t\t\t\"Maximum time to spend in this busy anim\"	0 = only stop when interrupted by external event\n" );
+	buf.Printf( "// \t\"interrupts\"\t\tOne of:\n" );
+	buf.Printf( "// \t\t\t\t\t\t\"BA_INT_NONE\"\t\tbreak out only when time runs out. No external influence will break me out.\n" );
+	buf.Printf( "// \t\t\t\t\t\t\"BA_INT_DANGER\"\t\tbreak out of this anim only if threatened\n" );
+	buf.Printf( "// \t\t\t\t\t\t\"BA_INT_PLAYER\"\t\tbreak out of this anim if I can see the player, or I'm threatened\n" );
+	buf.Printf( "// \t\t\t\t\t\t\"BA_INT_AMBUSH\"\t\tsomeone please define this - I have no idea what it does\n" );
+	buf.Printf( "// \t\t\t\t\t\t\"BA_INT_COMBAT\"\t\tbreak out of this anim if combat occurs in my line of sight (bullet hits, grenades, etc), -OR- the max time is reached\n" );
+	buf.Printf( "// }\n" );
+	buf.Printf( "//\n" );
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out a new actbusy file
+//-----------------------------------------------------------------------------
+bool CImportActBusy::Serialize( CUtlBuffer &buf, CDmElement *pRoot )
+{
+	SerializeHeader( buf );
+ 	buf.Printf( "\"ActBusy.txt\"\n" );
+	buf.Printf( "{\n" );
+
+	CDmAttribute *pChildren = pRoot->GetAttribute( "children" );
+	if ( !pChildren || pChildren->GetType() != AT_ELEMENT_ARRAY )
+		return NULL;
+
+	CDmrElementArray<> children( pChildren );
+	int nCount = children.Count();
+
+	buf.PushTab();
+	for ( int i = 0; i < nCount; ++i )
+	{
+		CDmElement *pChild = children[i];
+ 		buf.Printf( "\"%s\"\n", pChild->GetName() );
+		buf.Printf( "{\n" );
+
+		buf.PushTab();
+		PrintStringAttribute( pChild, buf, "busy_anim", true );
+		PrintStringAttribute( pChild, buf, "entry_anim", true );
+		PrintStringAttribute( pChild, buf, "exit_anim", true );
+		PrintStringAttribute( pChild, buf, "busy_sequence", true );
+		PrintStringAttribute( pChild, buf, "entry_sequence", true );
+		PrintStringAttribute( pChild, buf, "exit_sequence", true );
+		PrintFloatAttribute( pChild, buf, "min_time" );
+		PrintFloatAttribute( pChild, buf, "max_time" );
+		PrintStringAttribute( pChild, buf, "interrupts" );
+		buf.PopTab();
+
+		buf.Printf( "}\n" );
+	}
+	buf.PopTab();
+
+	buf.Printf( "}\n" );
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Reads a single element
+//-----------------------------------------------------------------------------
+bool CImportActBusy::UnserializeActBusyKey( CDmAttribute *pChildren, KeyValues *pKeyValues )
+{
+	CDmElement *pActBusy = CreateDmElement( "DmElement", pKeyValues->GetName(), NULL );
+	if ( !pActBusy )
+		return false;
+
+	// Each act busy needs to have an editortype associated with it so it displays nicely in editors
+	pActBusy->SetValue( "editorType", "actBusy" );
+
+	float flZero = 0.0f;
+	AddStringAttribute( pActBusy, pKeyValues, "busy_anim", "" );
+	AddStringAttribute( pActBusy, pKeyValues, "entry_anim", "" );
+	AddStringAttribute( pActBusy, pKeyValues, "exit_anim", "" );
+	AddStringAttribute( pActBusy, pKeyValues, "busy_sequence", "" );
+	AddStringAttribute( pActBusy, pKeyValues, "entry_sequence", "" );
+	AddStringAttribute( pActBusy, pKeyValues, "exit_sequence", "" );
+	AddFloatAttribute( pActBusy, pKeyValues, "min_time", &flZero );
+	AddFloatAttribute( pActBusy, pKeyValues, "max_time", &flZero );
+	AddStringAttribute( pActBusy, pKeyValues, "interrupts", "BA_INT_NONE" );
+
+	CDmrElementArray<> children( pChildren );
+	children.AddToTail( pActBusy );
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Main entry point for the unserialization
+//-----------------------------------------------------------------------------
+CDmElement* CImportActBusy::UnserializeFromKeyValues( KeyValues *pKeyValues )
+{
+	// Create the main element
+	CDmElement *pElement = CreateDmElement( "DmElement", "ActBusyList", NULL );
+	if ( !pElement )
+		return NULL;
+
+	// Each act busy list needs to have an editortype associated with it so it displays nicely in editors
+	pElement->SetValue( "editorType", "actBusyList" );
+
+	// All actbusy keys are elements of a single element array attribute 'children'
+	CDmAttribute *pChildren = pElement->AddAttribute( "children", AT_ELEMENT_ARRAY );
+	if ( !pChildren )
+		return NULL;
+
+	// Under the root are all the actbusy keys
+	for ( KeyValues *pActBusyKey = pKeyValues->GetFirstTrueSubKey(); pActBusyKey != NULL; pActBusyKey = pActBusyKey->GetNextTrueSubKey() )
+	{
+		if ( !UnserializeActBusyKey( pChildren, pActBusyKey ) )
+		{
+			Warning( "Error importing actbusy element %s\n", pActBusyKey->GetName() );
+			return NULL;
+		}
+	}
+
+	// Resolve all element references recursively
+	RecursivelyResolveElement( pElement );
+
+	return pElement;
+}

dmserializers/importkeyvaluebase.cpp

@@ -0,0 +1,292 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "importkeyvaluebase.h"
+#include "dmserializers.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include <limits.h>
+
+
+//-----------------------------------------------------------------------------
+// Default serialization method
+//-----------------------------------------------------------------------------
+bool CImportKeyValueBase::Serialize( CUtlBuffer &outBuf, CDmElement *pRoot )
+{
+	Warning( "Serialization not supported for importing from keyvalues files\n");
+	return false;
+}
+
+
+//-----------------------------------------------------------------------------
+// Creates a new element
+//-----------------------------------------------------------------------------
+CDmElement* CImportKeyValueBase::CreateDmElement( const char *pElementType, const char *pElementName, DmObjectId_t *pId )
+{
+	// See if we can create an element of that type
+	DmElementHandle_t hElement = g_pDataModel->CreateElement( pElementType, pElementName, DMFILEID_INVALID, pId );
+	if ( hElement == DMELEMENT_HANDLE_INVALID )
+	{
+		Warning("%s: Element uses unknown element type %s\n", m_pFileName, pElementType );
+		return NULL;
+	}
+
+	return g_pDataModel->GetElement( hElement );
+}
+
+
+//-----------------------------------------------------------------------------
+// Used to output typed attributes to keyvalues
+//-----------------------------------------------------------------------------
+void CImportKeyValueBase::PrintBoolAttribute( CDmElement* pElement, CUtlBuffer &outBuf, const char *pKeyName )
+{
+	if ( pElement->HasAttribute( pKeyName ) )
+	{
+		CDmAttribute *pAttribute = pElement->GetAttribute( pKeyName );
+		if ( pAttribute->GetType() == AT_BOOL )
+		{
+			outBuf.Printf("\"%s\" \"%d\"\n", pKeyName, pAttribute->GetValue<bool>( ) );
+		}
+	}
+}
+
+void CImportKeyValueBase::PrintIntAttribute( CDmElement* pElement, CUtlBuffer &outBuf, const char *pKeyName )
+{
+	if ( pElement->HasAttribute( pKeyName ) )
+	{
+		CDmAttribute *pAttribute = pElement->GetAttribute( pKeyName );
+		if ( pAttribute->GetType() == AT_INT )
+		{
+			outBuf.Printf("\"%s\" \"%d\"\n", pKeyName, pAttribute->GetValue<int>( ) );
+		}
+	}
+}
+
+void CImportKeyValueBase::PrintFloatAttribute( CDmElement* pElement, CUtlBuffer &outBuf, const char *pKeyName )
+{
+	if ( pElement->HasAttribute( pKeyName ) )
+	{
+		CDmAttribute *pAttribute = pElement->GetAttribute( pKeyName );
+		if ( pAttribute->GetType() == AT_FLOAT )
+		{
+			outBuf.Printf("\"%s\" \"%.10f\"\n", pKeyName, pAttribute->GetValue<float>( ) );
+		}
+	}
+}
+
+void CImportKeyValueBase::PrintStringAttribute( CDmElement* pElement, CUtlBuffer &outBuf, const char *pKeyName, bool bSkipEmptryStrings, bool bPrintValueOnly )
+{
+	if ( pElement->HasAttribute( pKeyName ) )
+	{
+		CDmAttribute *pAttribute = pElement->GetAttribute( pKeyName );
+		if ( pAttribute->GetType() == AT_STRING )
+		{
+			const char *pValue = pAttribute->GetValueString();
+			if ( !bSkipEmptryStrings || pValue[0] )
+			{
+				if ( !bPrintValueOnly )
+				{
+					outBuf.Printf("\"%s\" \"%s\"\n", pKeyName, pValue );
+				}
+				else
+				{
+					outBuf.Printf("\"%s\"\n", pValue );
+				}
+			}
+		}
+	}
+}
+
+	
+//-----------------------------------------------------------------------------
+// Used to add typed attributes from keyvalues
+//-----------------------------------------------------------------------------
+bool CImportKeyValueBase::AddBoolAttribute( CDmElement* pElement, KeyValues *pKeyValues, const char *pKeyName, bool *pDefault )
+{
+	KeyValues *pKey = pKeyValues->FindKey( pKeyName );
+	bool bValue;
+	if ( pKey )
+	{
+		bValue = pKey->GetInt() != 0;
+	}
+	else
+	{
+		if ( !pDefault )
+			return true;
+		bValue = *pDefault;
+	}
+
+	return pElement->SetValue( pKeyName, bValue ) != NULL;
+}
+
+	
+//-----------------------------------------------------------------------------
+// Used to add typed attributes from keyvalues
+//-----------------------------------------------------------------------------
+bool CImportKeyValueBase::AddIntAttribute( CDmElement* pElement, KeyValues *pKeyValues, const char *pKeyName, int *pDefault )
+{
+	KeyValues *pKey = pKeyValues->FindKey( pKeyName );
+	int nValue;
+	if ( pKey )
+	{
+		nValue = pKey->GetInt();
+	}
+	else
+	{
+		if ( !pDefault )
+			return true;
+		nValue = *pDefault;
+	}
+
+	return pElement->SetValue( pKeyName, nValue ) != NULL;
+}
+
+bool CImportKeyValueBase::AddFloatAttribute( CDmElement* pElement, KeyValues *pKeyValues, const char *pKeyName, float *pDefault )
+{
+	KeyValues *pKey = pKeyValues->FindKey( pKeyName );
+	float flValue;
+	if ( pKey )
+	{
+		flValue = pKey->GetFloat();
+	}
+	else
+	{
+		if ( !pDefault )
+			return true;
+		flValue = *pDefault;
+	}
+
+	return pElement->SetValue( pKeyName, flValue ) != NULL;
+}
+
+bool CImportKeyValueBase::AddStringAttribute( CDmElement* pElement, KeyValues *pKeyValues, const char *pKeyName, const char *pDefault )
+{
+	KeyValues *pKey = pKeyValues->FindKey( pKeyName );
+	const char *pValue = "";
+	if ( pKey )
+	{
+		pValue = pKey->GetString();
+	}
+	else
+	{
+		if ( !pDefault )
+			return true;
+		pValue = pDefault;
+	}
+
+	return pElement->SetValue( pKeyName, pValue ) != NULL;
+}
+
+
+//-----------------------------------------------------------------------------
+// Used to add typed attributes from keyvalues
+//-----------------------------------------------------------------------------
+bool CImportKeyValueBase::AddBoolAttributeFlags( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, int nFlags, bool *pDefault )
+{
+	if ( !AddBoolAttribute( pElement, pKeyValue, pKeyName, pDefault )  )
+		return false;
+
+	CDmAttribute *pAttribute = pElement->GetAttribute( pKeyName );
+	pAttribute->AddFlag( nFlags );
+	return true;
+}
+
+bool CImportKeyValueBase::AddIntAttributeFlags( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, int nFlags, int *pDefault )
+{
+	if ( !AddIntAttribute( pElement, pKeyValue, pKeyName, pDefault ) )
+		return false;
+
+	CDmAttribute *pAttribute = pElement->GetAttribute( pKeyName );
+	pAttribute->AddFlag( nFlags );
+	return true;
+}
+
+bool CImportKeyValueBase::AddFloatAttributeFlags( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, int nFlags, float *pDefault )
+{
+	if ( !AddFloatAttribute( pElement, pKeyValue, pKeyName, pDefault ) )
+		return false;
+
+	CDmAttribute *pAttribute = pElement->GetAttribute( pKeyName );
+	pAttribute->AddFlag( nFlags );
+	return true;
+}
+
+bool CImportKeyValueBase::AddStringAttributeFlags( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, int nFlags, const char *pDefault )
+{
+	if ( !AddStringAttribute( pElement, pKeyValue, pKeyName, pDefault ) )
+		return false;
+
+	CDmAttribute *pAttribute = pElement->GetAttribute( pKeyName );
+	pAttribute->AddFlag( nFlags );
+	return true;
+}
+
+	
+//-----------------------------------------------------------------------------
+// Recursively resolves all attributes pointing to elements
+//-----------------------------------------------------------------------------
+void CImportKeyValueBase::RecursivelyResolveElement( CDmElement* pElement )
+{
+	if ( !pElement )
+		return;
+
+	pElement->Resolve();
+
+	CDmAttribute *pAttribute = pElement->FirstAttribute();
+	while ( pAttribute )
+	{
+		switch ( pAttribute->GetType() )
+		{
+		case AT_ELEMENT:
+			{
+				CDmElement *pElementAt = pAttribute->GetValueElement<CDmElement>();
+				RecursivelyResolveElement( pElementAt );
+			}
+			break;
+
+		case AT_ELEMENT_ARRAY:
+			{
+				CDmrElementArray<> array( pAttribute );
+				int nCount = array.Count();
+				for ( int i = 0; i < nCount; ++i )
+				{
+					CDmElement *pElementAt = array[ i ];
+					RecursivelyResolveElement( pElementAt );
+				}
+			}
+			break;
+		}
+
+		pAttribute = pAttribute->NextAttribute( );
+	}
+}
+
+	
+//-----------------------------------------------------------------------------
+// Main entry point for the unserialization
+//-----------------------------------------------------------------------------
+bool CImportKeyValueBase::Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion,
+									   const char *pSourceFormatName, int nSourceFormatVersion,
+									   DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot )
+{
+	*ppRoot = NULL;
+	m_pFileName = g_pDataModel->GetFileName( fileid );
+
+	KeyValues *kv = new KeyValues( "dmx file" );
+	if ( !kv )
+		return false;
+
+	bool bOk = kv->LoadFromBuffer( "dmx file", buf );
+	if ( bOk )
+	{
+		*ppRoot = UnserializeFromKeyValues( kv );
+	}
+
+	kv->deleteThis();
+	return bOk;
+}

dmserializers/importkeyvaluebase.h

@@ -0,0 +1,84 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef IMPORTKEYVALUEBASE_H
+#define IMPORTKEYVALUEBASE_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "datamodel/idatamodel.h"
+
+
+//-----------------------------------------------------------------------------
+// Forward declarations
+//-----------------------------------------------------------------------------
+class CUtlBuffer;
+class KeyValues;
+class CDmElement;
+
+
+//-----------------------------------------------------------------------------
+// Serialization class for Key Values
+//-----------------------------------------------------------------------------
+abstract_class CImportKeyValueBase : public IDmSerializer
+{
+public:
+	// Inherited from IDMSerializer
+	virtual bool StoresVersionInFile() const { return false; }
+	virtual bool IsBinaryFormat() const { return false; }
+	virtual bool Serialize( CUtlBuffer &buf, CDmElement *pRoot );
+	virtual bool Unserialize( CUtlBuffer &buf, const char *pEncodingName, int nEncodingVersion,
+							  const char *pSourceFormatName, int nSourceFormatVersion,
+							  DmFileId_t fileid, DmConflictResolution_t idConflictResolution, CDmElement **ppRoot );
+
+protected:
+	// Main entry point for derived classes to implement unserialization
+	virtual CDmElement* UnserializeFromKeyValues( KeyValues *pKeyValues ) = 0;
+
+	// Returns the file name associated with the unserialization
+	const char *FileName() const;
+
+	// Creates new elements
+	CDmElement* CreateDmElement( const char *pElementType, const char *pElementName, DmObjectId_t *pId );
+
+	// Recursively resolves all attributes pointing to elements
+	void RecursivelyResolveElement( CDmElement* pElement );
+
+	// Used to add typed attributes from keyvalues
+	bool AddBoolAttribute( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, bool *pDefault = NULL );
+	bool AddIntAttribute( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, int *pDefault = NULL );
+	bool AddFloatAttribute( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, float *pDefault = NULL );
+	bool AddStringAttribute( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, const char *pDefault = NULL );
+
+	// Used to add typed attributes from keyvalues
+	bool AddBoolAttributeFlags( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, int nFlags, bool *pDefault = NULL );
+	bool AddIntAttributeFlags( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, int nFlags, int *pDefault = NULL );
+	bool AddFloatAttributeFlags( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, int nFlags, float *pDefault = NULL );
+	bool AddStringAttributeFlags( CDmElement* pElement, KeyValues *pKeyValue, const char *pKeyName, int nFlags, const char *pDefault = NULL );
+
+	// Used to output typed attributes to keyvalues
+	void PrintBoolAttribute( CDmElement* pElement, CUtlBuffer &outBuf, const char *pKeyName );
+	void PrintIntAttribute( CDmElement* pElement, CUtlBuffer &outBuf, const char *pKeyName );
+	void PrintFloatAttribute( CDmElement* pElement, CUtlBuffer &outBuf, const char *pKeyName );
+	void PrintStringAttribute( CDmElement* pElement, CUtlBuffer &outBuf, const char *pKeyName, bool bSkipEmptryStrings = false, bool bPrintValueOnly = false );
+
+private:
+	const char *m_pFileName;
+};
+
+
+//-----------------------------------------------------------------------------
+// Returns the file name associated with the unserialization
+//-----------------------------------------------------------------------------
+inline const char *CImportKeyValueBase::FileName() const
+{
+	return m_pFileName;
+}
+
+
+#endif // IMPORTKEYVALUEBASE_H

dmserializers/importsfmv1.cpp

@@ -0,0 +1,219 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmattribute.h"
+#include "datamodel/dmelement.h"
+
+#include <math.h>
+
+//-----------------------------------------------------------------------------
+// Format converter
+//-----------------------------------------------------------------------------
+class CImportSFMV1 : public CSFMBaseImporter
+{
+	typedef CSFMBaseImporter BaseClass;
+public:
+	CImportSFMV1( char const *formatName, char const *nextFormatName );
+
+private:
+	virtual bool DoFixup( CDmElement *pSourceRoot );
+
+	// Fixes up a single time attribute - converting from float seconds to int tenths-of-a-millisecond
+	void ConvertTimeAttribute( CDmElement *pElementInternal, const char *pOldName, const char *pNewName );
+
+	// Fixes up a single timeframe
+	void FixupTimeframe( CDmElement *pElementInternal );
+
+	// Fixes up a single log - converting from int milliseconds to int tenths-of-a-millisecond
+	void FixupLog( CDmElement *pElementInternal );
+
+	CUtlRBTree< CDmElement*, int > m_fixedElements;
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportSFMV1 s_ImportDmxV1( "sfm_v1", "sfm_v2" );
+
+void InstallSFMV1Importer( IDataModel *pFactory )
+{
+	pFactory->AddLegacyUpdater( &s_ImportDmxV1 );
+}
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CImportSFMV1::CImportSFMV1( char const *formatName, char const *nextFormatName ) : 
+	BaseClass( formatName, nextFormatName )	
+{
+	m_fixedElements.SetLessFunc( DefLessFunc( CDmElement * ) );
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+bool CImportSFMV1::DoFixup( CDmElement *pElementInternal )
+{
+	if ( !pElementInternal )
+		return true;
+
+	if ( m_fixedElements.Find( pElementInternal ) != m_fixedElements.InvalidIndex() )
+		return true;
+
+	m_fixedElements.Insert( pElementInternal );
+
+	const char *pType = pElementInternal->GetTypeString();
+	if ( !Q_strcmp( pType, "DmeTimeFrame" ) )
+	{
+		FixupTimeframe( pElementInternal );
+	}
+	else if ( !Q_strcmp( pType, "DmeLog" ) ||
+			!Q_strcmp( pType, "DmeIntLog" ) ||
+			!Q_strcmp( pType, "DmeFloatLog" ) ||
+			!Q_strcmp( pType, "DmeBoolLog" ) ||
+			!Q_strcmp( pType, "DmeColorLog" ) ||
+			!Q_strcmp( pType, "DmeVector2Log" ) ||
+			!Q_strcmp( pType, "DmeVector3Log" ) ||
+			!Q_strcmp( pType, "DmeVector4Log" ) ||
+			!Q_strcmp( pType, "DmeQAngleLog" ) ||
+			!Q_strcmp( pType, "DmeQuaternionLog" ) ||
+			!Q_strcmp( pType, "DmeVMatrixLog" ) )
+	{
+		FixupLog( pElementInternal );
+	}
+
+
+	for ( CDmAttribute *pAttribute = pElementInternal->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->GetType() == AT_ELEMENT )
+		{
+			CDmElement *pElement = pAttribute->GetValueElement<CDmElement>( );
+			DoFixup( pElement );
+			continue;
+		}
+
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+		{
+			CDmrElementArray<> array( pAttribute );
+			int nCount = array.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pChild = array[ i ];
+				DoFixup( pChild );
+			}
+			continue;
+		}
+	}
+
+	return true;
+}
+
+//-----------------------------------------------------------------------------
+// Fixes up a single time attribute - converting from float seconds to int tenths-of-a-millisecond
+//-----------------------------------------------------------------------------
+void CImportSFMV1::ConvertTimeAttribute( CDmElement *pElementInternal, const char *pOldName, const char *pNewName )
+{
+	float time = 0.0f;
+	CDmAttribute *pOldAttr = pElementInternal->GetAttribute( pOldName );
+	if ( !pOldAttr )
+	{
+		Warning( "*** Problem in file encountered!\n" );
+		Warning( "*** TimeFrame \"%s\" is missing attribute \"%s\"!\n", pElementInternal->GetName(), pOldName );
+		Warning( "*** Setting new attribute \"%s\" to 0\n", pNewName );
+	}
+	else if ( pOldAttr->GetType() != AT_FLOAT )
+	{
+		Warning( "*** Problem in file encountered!\n" );
+		Warning( "*** TimeFrame \"%s\" has attribute \"%s\" with an unexpected type (expected float)!\n", pElementInternal->GetName(), pOldName );
+	}
+	else
+	{
+		time = pOldAttr->GetValue< float >();
+		pElementInternal->RemoveAttribute( pOldName );
+	}
+
+	CDmAttribute *pNewAttr = NULL;
+
+	// this is disabled because even dmxconvert installs *some* movieobjects factories, when it probably shouldn't
+	// the method of installing movieobjects factories will change at some point in the future, and we can turn on this safety check then
+#if 0
+	int i = g_pDataModel->GetFirstFactory();
+	if ( g_pDataModel->IsValidFactory( i ) )
+	{
+		// factories installed - most likely from within movieobjects.lib
+		// ie there may be different ways of allocating attributes, so it's not safe to add them here
+		pNewAttr = pElementInternal->GetAttribute( pNewName );
+		if ( !pNewAttr || pNewAttr->GetType() != AT_INT )
+		{
+			Assert( 0 );
+			Warning( "*** Converter error - expected element \"%s\" to contain int attribute \"%s\"!\n", pElementInternal->GetName(), pNewName );
+			Warning( "***                 - if you get this error, the converter is out of sync with the element library!\n" );
+			return;
+		}
+	}
+	else
+	{
+#endif
+		// no factories installed - most likely from within dmxconvert.exe
+		// ie we're just working with CDmElement subclasses, so it's safe to add attributes
+		pNewAttr = pElementInternal->AddAttribute( pNewName, AT_INT );
+		if ( !pNewAttr )
+		{
+			Assert( 0 );
+			Warning( "*** Converter error - element \"%s\" already has a non-int attribute \"%s\"!\n", pElementInternal->GetName(), pNewName );
+			return;
+		}
+#if 0
+	}
+#endif
+
+	pNewAttr->SetValue< int >( floor( time * 10000 + 0.5f ) );
+}
+
+//-----------------------------------------------------------------------------
+// Fixes up a single timeframe
+//-----------------------------------------------------------------------------
+void CImportSFMV1::FixupTimeframe( CDmElement *pElementInternal )
+{
+	ConvertTimeAttribute( pElementInternal, "start", "startTime" );
+	ConvertTimeAttribute( pElementInternal, "duration", "durationTime" );
+	ConvertTimeAttribute( pElementInternal, "offset", "offsetTime" );
+}
+
+//-----------------------------------------------------------------------------
+// Fixes up a single log - converting from int milliseconds to int tenths-of-a-millisecond
+//-----------------------------------------------------------------------------
+void CImportSFMV1::FixupLog( CDmElement *pElementInternal )
+{
+	CDmAttribute *pAttr = pElementInternal->GetAttribute( "times" );
+	if ( !pAttr )
+	{
+		Warning( "*** Problem in file encountered!\n" );
+		Warning( "*** Log \"%s\" is missing attribute \"%s\"!\n", pElementInternal->GetName(), "times" );
+		return;
+	}
+
+	if ( pAttr->GetType() != AT_INT_ARRAY )
+	{
+		Warning( "*** Problem in file encountered!\n" );
+		Warning( "*** Log \"%s\" has attribute \"%s\" with an unexpected type (expected int array)!\n", pElementInternal->GetName(), "times" );
+		return;
+	}
+
+	CDmrArray<int> array( pAttr );
+	int c = array.Count();
+	for ( int i = 0; i < c; ++i )
+	{
+		// convert all log times from int milliseconds to int tenths-of-a-millisecond
+		array.Set( i, 10 * array[i] );
+	}
+}

dmserializers/importsfmv2.cpp

@@ -0,0 +1,294 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/utlmap.h"
+#include <limits.h>
+
+
+//-----------------------------------------------------------------------------
+// Format converter
+//-----------------------------------------------------------------------------
+class CImportSFMV2 : public CSFMBaseImporter
+{
+	typedef CSFMBaseImporter BaseClass;
+public:
+	CImportSFMV2( char const *formatName, char const *nextFormatName );
+
+private:
+	virtual bool DoFixup( CDmElement *pSourceRoot );
+
+
+	void FixupElement( CDmElement *pElement );
+	// Fixes up all elements
+	void BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportSFMV2 s_ImportSFMV2( "sfm_v2", "sfm_v3" );
+
+void InstallSFMV2Importer( IDataModel *pFactory )
+{
+	pFactory->AddLegacyUpdater( &s_ImportSFMV2 );
+}
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CImportSFMV2::CImportSFMV2( char const *formatName, char const *nextFormatName ) : 
+	BaseClass( formatName, nextFormatName )	
+{
+}
+
+
+struct LayerType_t
+{
+	char const *loglayertype;
+	int datatype;
+	char const *logtype;
+};
+
+static LayerType_t g_LayerTypes[] = 
+{
+	{ "DmeIntLogLayer",	AT_INT_ARRAY, "DmeIntLog" },
+	{ "DmeFloatLogLayer", AT_FLOAT_ARRAY, "DmeFloatLog" },
+	{ "DmeBoolLogLayer", AT_BOOL_ARRAY, "DmeBoolLog" },
+	// AT_STRING_ARRAY,
+	// AT_VOID_ARRAY,
+	// AT_OBJECTID_ARRAY,
+	{ "DmeColorLogLayer", AT_COLOR_ARRAY, "DmeColorLog" },
+	{ "DmeVector2LogLayer", AT_VECTOR2_ARRAY, "DmeVector2Log" },
+	{ "DmeVector3LogLayer", AT_VECTOR3_ARRAY, "DmeVector3Log" },
+	{ "DmeVector4LogLayer", AT_VECTOR4_ARRAY, "DmeVector4Log" },
+	{ "DmeQAngleLogLayer", AT_QANGLE_ARRAY, "DmeQAngleLog" },
+	{ "DmeQuaternionLogLayer", AT_QUATERNION_ARRAY, "DmeQuaternionLog" },
+	{ "DmeVMatrixLogLayer", AT_VMATRIX_ARRAY, "DmeVMatrixLog" },
+	// AT_ELEMENT_ARRAY
+	// NO ARRAY TYPES EITHER!!!
+};
+
+int GetLogType( char const *type )
+{
+	int c = ARRAYSIZE( g_LayerTypes );
+	for ( int i = 0; i < c; ++i )
+	{
+		if ( !Q_stricmp( type, g_LayerTypes[ i ].logtype ) )
+			return g_LayerTypes[ i ].datatype;
+	}
+	return AT_UNKNOWN;
+}
+
+char const *GetLogLayerType( int nDataType )
+{
+	int c = ARRAYSIZE( g_LayerTypes );
+	for ( int i = 0; i < c; ++i )
+	{
+		if ( nDataType == g_LayerTypes[ i ].datatype )
+			return g_LayerTypes[ i ].loglayertype;
+	}
+	return NULL;
+}
+
+char const *GetLogLayerType( char const *logType )
+{
+	int c = ARRAYSIZE( g_LayerTypes );
+	for ( int i = 0; i < c; ++i )
+	{
+		if ( !Q_stricmp( logType, g_LayerTypes[ i ].logtype ) )
+			return g_LayerTypes[ i ].loglayertype;
+	}
+	return NULL;
+}
+
+template< class T >
+void CopyValues( int layerType, CDmElement *pElement, CDmElement *pLayer, CDmAttribute *pInTimeAttribute, CDmAttribute *pInCurveTypeAttribute )
+{
+	CDmAttribute *pInValueAttribute = pElement->GetAttribute( "values" );
+	if ( !pInValueAttribute )
+	{
+		Assert( 0 );
+		return;
+	}
+
+	CDmrArray<T> outValues( pLayer->AddAttribute( "values", (DmAttributeType_t)layerType ) );
+	CDmrArray<int> outTimes( pLayer->AddAttribute( "times", AT_INT_ARRAY ) );
+	CDmrArray<int> outCurveTypes;
+	if ( pInCurveTypeAttribute )
+	{
+		outCurveTypes.Init( pLayer->AddAttribute( "curvetypes", AT_INT_ARRAY ) );
+	}
+
+	CDmrArray<T> inValues( pInValueAttribute );
+	CDmrArray<int> inTimes( pInTimeAttribute );
+	CDmrArray<int> inCurveTypes( pInCurveTypeAttribute );
+
+	Assert( inValues.Count() == inTimes.Count() );
+	int c = inValues.Count();
+	for ( int i = 0; i < c; ++i )
+	{
+		outTimes.AddToTail( inTimes[ i ] );
+		outValues.AddToTail( inValues[ i ] );
+		if ( outCurveTypes.IsValid() )
+		{
+			outCurveTypes.AddToTail( inCurveTypes[ i ] );
+		}
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV2::FixupElement( CDmElement *pElement )
+{
+	if ( !pElement )
+		return;
+
+	// Perform the fixup
+	const char *pType = pElement->GetTypeString();
+	int layerType = GetLogType( pType );
+	if ( layerType != AT_UNKNOWN )
+	{
+		/*
+		char buf[ 128 ];
+		g_pDataModel->ToString( pElement->GetId(), buf, sizeof( buf ) );
+
+		Msg( "Processing %s %s id %s\n",
+			pElement->GetTypeString(), pElement->GetName(), buf );
+		*/
+
+		// Find attribute arrays for times, values and curvetypes
+		CDmAttribute *pTimes = pElement->GetAttribute( "times" );
+		CDmAttribute *pCurveTypes = NULL;
+
+		// FIX
+		CDmAttribute *pAttr = pElement->AddAttribute( "usecurvetypes", AT_BOOL );
+		if ( pAttr->GetValue<bool>() )
+		{
+			pCurveTypes = pElement->GetAttribute( "curvetypes" );
+		}
+
+		// Get the default layer (added when the new style log is created)
+		CDmrElementArray<> layers( pElement->AddAttribute( "layers", AT_ELEMENT_ARRAY ) );
+		CDmElement *layer = NULL;
+		if ( layers.Count() == 0 )
+		{
+			DmElementHandle_t hElement = g_pDataModel->CreateElement( GetLogLayerType( layerType ), GetLogLayerType( layerType ), pElement->GetFileId() );
+			layer = g_pDataModel->GetElement( hElement );
+			layers.AddToTail( layer );
+		}
+		else
+		{
+			Assert( layers.Count() == 1 );
+			layer = layers[ 0 ];
+		}
+
+		// Copy data
+		switch ( layerType )
+		{
+		default:
+		case AT_UNKNOWN:
+			break;
+		case AT_FLOAT_ARRAY:
+			CopyValues< float >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		case AT_INT_ARRAY:
+			CopyValues< int >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		case AT_BOOL_ARRAY:
+			CopyValues< bool >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		case AT_COLOR_ARRAY:
+			CopyValues< Color >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		case AT_VECTOR2_ARRAY:
+			CopyValues< Vector2D >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		case AT_VECTOR3_ARRAY:
+			CopyValues< Vector >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		case AT_VECTOR4_ARRAY:
+			CopyValues< Vector4D >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		case AT_QANGLE_ARRAY:
+			CopyValues< QAngle >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		case AT_QUATERNION_ARRAY:
+			CopyValues< Quaternion >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		case AT_VMATRIX_ARRAY:
+			CopyValues< VMatrix >( layerType, pElement, layer, pTimes, pCurveTypes );
+			break;
+		}
+
+		// Set the back pointer
+		CDmAttribute *ownerLog = layer->AddAttribute( "ownerlog", AT_ELEMENT );
+		Assert( ownerLog );
+		ownerLog->SetValue( pElement->GetHandle() );
+
+		// Delete the base attributes
+		pElement->RemoveAttribute( "times" );
+		pElement->RemoveAttribute( "values" );
+		pElement->RemoveAttribute( "curvetypes" );
+	}
+}
+
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV2::BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list )
+{
+	if ( !pElement )
+		return;
+
+	if ( list.Find( pElement ) != list.InvalidIndex() )
+		return;
+
+	list.Insert( pElement );
+
+	// Descene to bottom of tree, then do fixup coming back up the tree
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->GetType() == AT_ELEMENT )
+		{
+			CDmElement *pElementAt = pAttribute->GetValueElement<CDmElement>( );
+			BuildList( pElementAt, list );
+			continue;
+		}
+
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+		{
+			CDmrElementArray<> array( pAttribute );
+			int nCount = array.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pChild = array[ i ];
+				BuildList( pChild, list );
+			}
+			continue;
+		}
+	}
+}
+
+bool CImportSFMV2::DoFixup( CDmElement *pSourceRoot )
+{
+	CUtlRBTree< CDmElement *, int >	fixlist( 0, 0, DefLessFunc( CDmElement * ) );
+	BuildList( pSourceRoot, fixlist );
+	for ( int i = fixlist.FirstInorder(); i != fixlist.InvalidIndex() ; i = fixlist.NextInorder( i ) )
+	{
+		// Search and replace in the entire tree!
+		FixupElement( fixlist[ i ] );
+	}
+	return true;
+}

dmserializers/importsfmv3.cpp

@@ -0,0 +1,227 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/utlmap.h"
+#include <limits.h>
+
+
+//-----------------------------------------------------------------------------
+// Format converter
+//-----------------------------------------------------------------------------
+class CImportSFMV3 : public CSFMBaseImporter
+{
+	typedef CSFMBaseImporter BaseClass;
+public:
+	CImportSFMV3( char const *formatName, char const *nextFormatName );
+
+private:
+	virtual bool DoFixup( CDmElement *pSourceRoot );
+
+
+	void FixupElement( CDmElement *pElement );
+	// Fixes up all elements
+	void BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportSFMV3 s_ImportSFMV3( "sfm_v3", "sfm_v4" );
+
+void InstallSFMV3Importer( IDataModel *pFactory )
+{
+	pFactory->AddLegacyUpdater( &s_ImportSFMV3 );
+}
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CImportSFMV3::CImportSFMV3( char const *formatName, char const *nextFormatName ) : 
+	BaseClass( formatName, nextFormatName )	
+{
+}
+
+
+struct LogToCurveInfoTypeMap_t
+{
+	const char *pLogType;
+	const char *pLogLayerType;
+	const char *pCurveInfoType;
+};
+
+LogToCurveInfoTypeMap_t g_typeMap[] =
+{
+	{ "DmeIntLog",			"DmeIntLogLayer",		"DmeIntCurveInfo" },
+	{ "DmeFloatLog",		"DmeFloatLogLayer",		"DmeFloatCurveInfo" },
+	{ "DmeBoolLog",			"DmeBoolLogLayer",		"DmeBoolCurveInfo" },
+	// string,
+	// void,
+	// objectid,
+	{ "DmeColorLog",		"DmeColorLogLayer",		"DmeColorCurveInfo" },
+	{ "DmeVector2Log",		"DmeVector2LogLayer",	"DmeVector2CurveInfo" },
+	{ "DmeVector3Log",		"DmeVector3LogLayer",	"DmeVector3CurveInfo" },
+	{ "DmeVector4Log",		"DmeVector4LogLayer",	"DmeVector4CurveInfo" },
+	{ "DmeQAngleLog",		"DmeQAngleLogLayer",	"DmeQAngleCurveInfo" },
+	{ "DmeQuaternionLog",	"DmeQuaternionLogLayer","DmeQuaternionCurveInfo" },
+	{ "DmeVMatrixLog",		"DmeVMatrixLogLayer",	"DmeVMatrixCurveInfo" },
+};
+
+const char *GetCurveInfoTypeFromLogType( const char *pLogType )
+{
+	int c = ARRAYSIZE( g_typeMap );
+	for ( int i = 0; i < c; ++i )
+	{
+		if ( !Q_stricmp( pLogType, g_typeMap[ i ].pLogType ) )
+			return g_typeMap[ i ].pCurveInfoType;
+	}
+	return NULL;
+}
+
+bool IsLogLayerType( const char *pLogLayerType )
+{
+	int c = ARRAYSIZE( g_typeMap );
+	for ( int i = 0; i < c; ++i )
+	{
+		if ( !Q_stricmp( pLogLayerType, g_typeMap[ i ].pLogLayerType ) )
+			return true;
+	}
+	return false;
+}
+
+void MoveAttribute( CDmElement *pFromElement, const char *pFromAttrName, CDmElement *pToElement = NULL, const char *pToAttrName = NULL, DmAttributeType_t toType = AT_UNKNOWN )
+{
+	if ( !pToAttrName )
+	{
+		pToAttrName = pFromAttrName;
+	}
+
+	if ( pToElement )
+	{
+		CDmAttribute *pFromAttr = pFromElement->GetAttribute( pFromAttrName );
+		const void *pValue = pFromAttr->GetValueUntyped();
+		DmAttributeType_t fromType = pFromAttr->GetType();
+		if ( toType == AT_UNKNOWN )
+		{
+			toType = fromType;
+		}
+
+		CDmAttribute *pToAttr = pToElement->AddAttribute( pToAttrName, toType );
+		if ( !pToAttr )
+		{
+			Warning( "*** Problem in converter encountered!\n" );
+			Warning( "*** Unable to find or add attribute \"%s\" to element \"%s\"!\n", pToAttrName, pToElement->GetName() );
+		}
+		else if ( fromType != toType )
+		{
+			Warning( "*** Problem in file encountered!\n" );
+			Warning( "*** Element \"%s\" has attribute \"%s\" with an unexpected type!\n", pFromElement->GetName(), pFromAttrName );
+		}
+		else
+		{
+			pToAttr->SetValue( toType, pValue );
+		}
+	}
+
+	pFromElement->RemoveAttribute( pFromAttrName );
+}
+
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV3::FixupElement( CDmElement *pElement )
+{
+	if ( !pElement )
+		return;
+
+	const char *pType = pElement->GetTypeString();
+
+	// log layer
+	if ( IsLogLayerType( pType ) )
+	{
+		pElement->RemoveAttribute( "ownerlog" );
+		return;
+	}
+
+	// log
+	const char *pCurveInfoType = GetCurveInfoTypeFromLogType( pType );
+	if ( !pCurveInfoType )
+		return;
+
+	CDmElement *pCurveInfo = NULL;
+	CDmAttribute *pUseCurveTypeAttr = pElement->GetAttribute( "usecurvetypes" );
+	if ( pUseCurveTypeAttr && pUseCurveTypeAttr->GetValue<bool>() )
+	{
+		DmElementHandle_t hElement = g_pDataModel->CreateElement( "curve info", pCurveInfoType, pElement->GetFileId() );
+		pCurveInfo = g_pDataModel->GetElement( hElement );
+	}
+	pElement->RemoveAttribute( "usecurvetypes" );
+
+	MoveAttribute( pElement, "defaultcurvetype",	pCurveInfo, "defaultCurveType",		AT_INT );
+	MoveAttribute( pElement, "defaultedgezerovalue",pCurveInfo, "defaultEdgeZeroValue" );
+	MoveAttribute( pElement, "useedgeinfo",			pCurveInfo, "useEdgeInfo",			AT_BOOL );
+	MoveAttribute( pElement, "rightedgetime",		pCurveInfo, "rightEdgeTime",		AT_INT );
+	MoveAttribute( pElement, "left_edge_active",	pCurveInfo, "leftEdgeActive",		AT_BOOL );
+	MoveAttribute( pElement, "right_edge_active",	pCurveInfo, "rightEdgeActive",		AT_BOOL );
+	MoveAttribute( pElement, "left_edge_curvetype", pCurveInfo, "leftEdgeCurveType",	AT_INT );
+	MoveAttribute( pElement, "right_edge_curvetype",pCurveInfo, "rightEdgeCurveType",	AT_INT );
+	MoveAttribute( pElement, "left_edge_value",		pCurveInfo, "leftEdgeValue" );
+	MoveAttribute( pElement, "right_edge_value",	pCurveInfo, "rightEdgeValue" );
+}
+
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV3::BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list )
+{
+	if ( !pElement )
+		return;
+
+	if ( list.Find( pElement ) != list.InvalidIndex() )
+		return;
+
+	list.Insert( pElement );
+
+	// Descend to bottom of tree, then do fixup coming back up the tree
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->GetType() == AT_ELEMENT )
+		{
+			CDmElement *pElementAt = pAttribute->GetValueElement<CDmElement>( );
+			BuildList( pElementAt, list );
+			continue;
+		}
+
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+		{
+			CDmrElementArray<> array( pAttribute );
+			int nCount = array.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pChild = array[ i ];
+				BuildList( pChild, list );
+			}
+			continue;
+		}
+	}
+}
+
+bool CImportSFMV3::DoFixup( CDmElement *pSourceRoot )
+{
+	CUtlRBTree< CDmElement *, int >	fixlist( 0, 0, DefLessFunc( CDmElement * ) );
+	BuildList( pSourceRoot, fixlist );
+	for ( int i = fixlist.FirstInorder(); i != fixlist.InvalidIndex() ; i = fixlist.NextInorder( i ) )
+	{
+		// Search and replace in the entire tree!
+		FixupElement( fixlist[ i ] );
+	}
+	return true;
+}

dmserializers/importsfmv4.cpp

@@ -0,0 +1,123 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/utlmap.h"
+#include <limits.h>
+
+
+//-----------------------------------------------------------------------------
+// Format converter
+//-----------------------------------------------------------------------------
+class CImportSFMV4 : public CSFMBaseImporter
+{
+	typedef CSFMBaseImporter BaseClass;
+public:
+	CImportSFMV4( char const *formatName, char const *nextFormatName );
+
+private:
+	virtual bool DoFixup( CDmElement *pSourceRoot );
+
+
+	void FixupElement( CDmElement *pElement );
+	// Fixes up all elements
+	void BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportSFMV4 s_ImportSFMV4( "sfm_v4", "sfm_v5" );
+
+void InstallSFMV4Importer( IDataModel *pFactory )
+{
+	pFactory->AddLegacyUpdater( &s_ImportSFMV4 );
+}
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CImportSFMV4::CImportSFMV4( char const *formatName, char const *nextFormatName ) : 
+	BaseClass( formatName, nextFormatName )	
+{
+}
+
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV4::FixupElement( CDmElement *pElement )
+{
+	if ( !pElement )
+		return;
+
+	const char *pType = pElement->GetTypeString();
+
+	if ( !V_stricmp( pType, "DmeCamera" ) )
+	{
+		CDmAttribute *pOldToneMapScaleAttr = pElement->GetAttribute( "toneMapScale" );
+		float fNewBloomScale = pOldToneMapScaleAttr->GetValue<float>( );
+
+		Assert( !pElement->HasAttribute("bloomScale") );
+
+		CDmAttribute *pNewBloomScaleAttr = pElement->AddAttribute( "bloomScale", AT_FLOAT );
+		pNewBloomScaleAttr->SetValue( fNewBloomScale );
+		pOldToneMapScaleAttr->SetValue( 1.0f );
+	}
+}
+
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV4::BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list )
+{
+	if ( !pElement )
+		return;
+
+	if ( list.Find( pElement ) != list.InvalidIndex() )
+		return;
+
+	list.Insert( pElement );
+
+	// Descend to bottom of tree, then do fixup coming back up the tree
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->GetType() == AT_ELEMENT )
+		{
+			CDmElement *pElementAt = pAttribute->GetValueElement<CDmElement>( );
+			BuildList( pElementAt, list );
+			continue;
+		}
+
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+		{
+			CDmrElementArray<> array( pAttribute );
+			int nCount = array.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pChild = array[ i ];
+				BuildList( pChild, list );
+			}
+			continue;
+		}
+	}
+}
+
+bool CImportSFMV4::DoFixup( CDmElement *pSourceRoot )
+{
+	CUtlRBTree< CDmElement *, int >	fixlist( 0, 0, DefLessFunc( CDmElement * ) );
+	BuildList( pSourceRoot, fixlist );
+	for ( int i = fixlist.FirstInorder(); i != fixlist.InvalidIndex() ; i = fixlist.NextInorder( i ) )
+	{
+		// Search and replace in the entire tree!
+		FixupElement( fixlist[ i ] );
+	}
+	return true;
+}

dmserializers/importsfmv5.cpp

@@ -0,0 +1,120 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/utlmap.h"
+#include <limits.h>
+
+
+//-----------------------------------------------------------------------------
+// Format converter
+//-----------------------------------------------------------------------------
+class CImportSFMV5 : public CSFMBaseImporter
+{
+	typedef CSFMBaseImporter BaseClass;
+public:
+	CImportSFMV5( char const *formatName, char const *nextFormatName );
+
+private:
+	virtual bool DoFixup( CDmElement *pSourceRoot );
+
+
+	void FixupElement( CDmElement *pElement );
+	// Fixes up all elements
+	void BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportSFMV5 s_ImportSFMV5( "sfm_v5", "sfm_v6" );
+
+void InstallSFMV5Importer( IDataModel *pFactory )
+{
+	pFactory->AddLegacyUpdater( &s_ImportSFMV5 );
+}
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CImportSFMV5::CImportSFMV5( char const *formatName, char const *nextFormatName ) : 
+	BaseClass( formatName, nextFormatName )	
+{
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV5::FixupElement( CDmElement *pElement )
+{
+	if ( !pElement )
+		return;
+
+	const char *pType = pElement->GetTypeString();
+
+	if ( !V_stricmp( pType, "DmeSpotLight" ) )
+	{
+		pElement->SetType( "DmeProjectedLight" );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV5::BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list )
+{
+	if ( !pElement )
+		return;
+
+	if ( list.Find( pElement ) != list.InvalidIndex() )
+		return;
+
+	list.Insert( pElement );
+
+	// Descend to bottom of tree, then do fixup coming back up the tree
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->GetType() == AT_ELEMENT )
+		{
+			CDmElement *pElementAt = pAttribute->GetValueElement<CDmElement>( );
+			BuildList( pElementAt, list );
+			continue;
+		}
+
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+		{
+			CDmrElementArray<> array( pAttribute );
+			int nCount = array.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pChild = array[ i ];
+				BuildList( pChild, list );
+			}
+			continue;
+		}
+	}
+}
+
+bool CImportSFMV5::DoFixup( CDmElement *pSourceRoot )
+{
+	CUtlRBTree< CDmElement *, int >	fixlist( 0, 0, DefLessFunc( CDmElement * ) );
+	BuildList( pSourceRoot, fixlist );
+	for ( int i = fixlist.FirstInorder(); i != fixlist.InvalidIndex() ; i = fixlist.NextInorder( i ) )
+	{
+		// Search and replace in the entire tree!
+		FixupElement( fixlist[ i ] );
+	}
+	return true;
+}

dmserializers/importsfmv6.cpp

@@ -0,0 +1,140 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/utlmap.h"
+#include <limits.h>
+
+
+//-----------------------------------------------------------------------------
+// Format converter
+//-----------------------------------------------------------------------------
+class CImportSFMV6 : public CSFMBaseImporter
+{
+	typedef CSFMBaseImporter BaseClass;
+public:
+	CImportSFMV6( char const *formatName, char const *nextFormatName );
+
+private:
+	virtual bool DoFixup( CDmElement *pSourceRoot );
+
+	Quaternion DirectionToOrientation( const Vector &dir );
+
+	void FixupElement( CDmElement *pElement );
+	// Fixes up all elements
+	void BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportSFMV6 s_ImportSFMV6( "sfm_v6", "sfm_v7" );
+
+void InstallSFMV6Importer( IDataModel *pFactory )
+{
+	pFactory->AddLegacyUpdater( &s_ImportSFMV6 );
+}
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CImportSFMV6::CImportSFMV6( char const *formatName, char const *nextFormatName ) : 
+	BaseClass( formatName, nextFormatName )	
+{
+}
+
+Quaternion CImportSFMV6::DirectionToOrientation( const Vector &dir )
+{
+	Vector up( 0, 0, 1 );
+	Vector right = CrossProduct( dir, up );
+	if ( right.IsLengthLessThan( 0.001f ) )
+	{
+		up.Init( 1, 0, 0 );
+		right = CrossProduct( dir, up );
+	}
+	right.NormalizeInPlace();
+	up = CrossProduct( right, dir );
+
+	Quaternion q;
+	BasisToQuaternion( dir, right, up, q );
+	return q;
+}
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV6::FixupElement( CDmElement *pElement )
+{
+	if ( !pElement )
+		return;
+
+	const char *pType = pElement->GetTypeString();
+
+	if ( !V_stricmp( pType, "DmeProjectedLight" ) )
+	{
+		Vector vDir = pElement->GetValue<Vector>( "direction" );
+		pElement->RemoveAttribute( "direction" );
+		Quaternion q = DirectionToOrientation( vDir );
+		pElement->SetValue<Quaternion>( "orientation", q );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV6::BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list )
+{
+	if ( !pElement )
+		return;
+
+	if ( list.Find( pElement ) != list.InvalidIndex() )
+		return;
+
+	list.Insert( pElement );
+
+	// Descend to bottom of tree, then do fixup coming back up the tree
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->GetType() == AT_ELEMENT )
+		{
+			CDmElement *pElementAt = pAttribute->GetValueElement<CDmElement>( );
+			BuildList( pElementAt, list );
+			continue;
+		}
+
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+		{
+			CDmrElementArray<> array( pAttribute );
+			int nCount = array.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pChild = array[ i ];
+				BuildList( pChild, list );
+			}
+			continue;
+		}
+	}
+}
+
+bool CImportSFMV6::DoFixup( CDmElement *pSourceRoot )
+{
+	CUtlRBTree< CDmElement *, int >	fixlist( 0, 0, DefLessFunc( CDmElement * ) );
+	BuildList( pSourceRoot, fixlist );
+	for ( int i = fixlist.FirstInorder(); i != fixlist.InvalidIndex() ; i = fixlist.NextInorder( i ) )
+	{
+		// Search and replace in the entire tree!
+		FixupElement( fixlist[ i ] );
+	}
+	return true;
+}

dmserializers/importsfmv7.cpp

@@ -0,0 +1,144 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/utlmap.h"
+#include <limits.h>
+
+
+//-----------------------------------------------------------------------------
+// Format converter
+//-----------------------------------------------------------------------------
+class CImportSFMV7 : public CSFMBaseImporter
+{
+	typedef CSFMBaseImporter BaseClass;
+public:
+	CImportSFMV7( char const *formatName, char const *nextFormatName );
+
+private:
+	virtual bool DoFixup( CDmElement *pSourceRoot );
+
+
+	void FixupElement( CDmElement *pElement );
+	// Fixes up all elements
+	void BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportSFMV7 s_ImportSFMV7( "sfm_v7", "sfm_v8" );
+
+void InstallSFMV7Importer( IDataModel *pFactory )
+{
+	pFactory->AddLegacyUpdater( &s_ImportSFMV7 );
+}
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CImportSFMV7::CImportSFMV7( char const *formatName, char const *nextFormatName ) : 
+	BaseClass( formatName, nextFormatName )	
+{
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV7::FixupElement( CDmElement *pElement )
+{
+	if ( !pElement )
+		return;
+
+	const char *pType = pElement->GetTypeString();
+
+	if ( !V_stricmp( pType, "DmeAnimationSet" ) )
+	{
+		// Add a level of indirection in animation sets
+		// Modify the type of all controls from DmElement to DmeAnimationSetControl
+		CDmrElementArray<> srcPresets( pElement, "presets" );
+		if ( srcPresets.IsValid() )
+		{
+			CDmrElementArray<> presetGroupArray( pElement, "presetGroups", true );
+			CDmElement *pPresetGroup = CreateElement< CDmElement >( "custom", pElement->GetFileId() );
+			pPresetGroup->SetType( "DmePresetGroup" );
+			CDmrElementArray<> presets( pPresetGroup, "presets", true );
+			presetGroupArray.AddToTail( pPresetGroup );
+
+			int nCount = srcPresets.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pPreset = srcPresets[i];
+				if ( pPreset )
+				{
+					pPreset->SetType( "DmePreset" );
+					presets.AddToTail( pPreset );
+				}
+			}
+
+			srcPresets.RemoveAll();
+		}
+		pElement->RemoveAttribute( "presets" );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV7::BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list )
+{
+	if ( !pElement )
+		return;
+
+	if ( list.Find( pElement ) != list.InvalidIndex() )
+		return;
+
+	list.Insert( pElement );
+
+	// Descend to bottom of tree, then do fixup coming back up the tree
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->GetType() == AT_ELEMENT )
+		{
+			CDmElement *pElementAt = pAttribute->GetValueElement<CDmElement>( );
+			BuildList( pElementAt, list );
+			continue;
+		}
+
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+		{
+			CDmrElementArray<> array( pAttribute );
+			int nCount = array.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pChild = array[ i ];
+				BuildList( pChild, list );
+			}
+			continue;
+		}
+	}
+}
+
+bool CImportSFMV7::DoFixup( CDmElement *pSourceRoot )
+{
+	CUtlRBTree< CDmElement *, int >	fixlist( 0, 0, DefLessFunc( CDmElement * ) );
+	BuildList( pSourceRoot, fixlist );
+	for ( int i = fixlist.FirstInorder(); i != fixlist.InvalidIndex() ; i = fixlist.NextInorder( i ) )
+	{
+		// Search and replace in the entire tree!
+		FixupElement( fixlist[ i ] );
+	}
+	return true;
+}

dmserializers/importsfmv8.cpp

@@ -0,0 +1,139 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/utlmap.h"
+#include <limits.h>
+
+
+//-----------------------------------------------------------------------------
+// Format converter
+//-----------------------------------------------------------------------------
+class CImportSFMV8 : public CSFMBaseImporter
+{
+	typedef CSFMBaseImporter BaseClass;
+public:
+	CImportSFMV8( const char *formatName, const char *nextFormatName );
+
+private:
+	virtual bool DoFixup( CDmElement *pSourceRoot );
+
+
+	void FixupElement( CDmElement *pElement );
+	// Fixes up all elements
+	void BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportSFMV8 s_ImportSFMV8( "sfm_v8", "sfm_v9" );
+
+void InstallSFMV8Importer( IDataModel *pFactory )
+{
+	pFactory->AddLegacyUpdater( &s_ImportSFMV8 );
+}
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CImportSFMV8::CImportSFMV8( const char *formatName, const char *nextFormatName ) : 
+	BaseClass( formatName, nextFormatName )	
+{
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV8::FixupElement( CDmElement *pElement )
+{
+	if ( !pElement )
+		return;
+
+	const char *pType = pElement->GetTypeString();
+
+	if ( !V_stricmp( pType, "DmeAnimationSet" ) )
+	{
+		// Remove 'midpoint' from all controls, and 
+		// Add 'defaultBalance' and 'defaultMultilevel' to all non-transform controls
+		CDmrElementArray<> srcControls( pElement, "controls" );
+		if ( srcControls.IsValid() )
+		{
+			int nCount = srcControls.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pControl = srcControls[i];
+				if ( pControl )
+				{
+					if ( !pControl->GetValue<bool>( "transform" ) )
+					{
+						pControl->InitValue( "defaultBalance", 0.5f );
+						pControl->InitValue( "defaultMultilevel", 0.5f );
+						pControl->RemoveAttribute( "midpoint" );
+					}
+				}
+			}
+		}
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV8::BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list )
+{
+	if ( !pElement )
+		return;
+
+	if ( list.Find( pElement ) != list.InvalidIndex() )
+		return;
+
+	list.Insert( pElement );
+
+	// Descend to bottom of tree, then do fixup coming back up the tree
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->GetType() == AT_ELEMENT )
+		{
+			CDmElement *pElementAt = pAttribute->GetValueElement<CDmElement>( );
+			BuildList( pElementAt, list );
+			continue;
+		}
+
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+		{
+			CDmrElementArray<> array( pAttribute );
+			int nCount = array.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pChild = array[ i ];
+				BuildList( pChild, list );
+			}
+			continue;
+		}
+	}
+}
+
+bool CImportSFMV8::DoFixup( CDmElement *pSourceRoot )
+{
+	CUtlRBTree< CDmElement *, int >	fixlist( 0, 0, DefLessFunc( CDmElement * ) );
+	BuildList( pSourceRoot, fixlist );
+	for ( int i = fixlist.FirstInorder(); i != fixlist.InvalidIndex() ; i = fixlist.NextInorder( i ) )
+	{
+		// Search and replace in the entire tree!
+		FixupElement( fixlist[ i ] );
+	}
+	return true;
+}

dmserializers/importsfmv9.cpp

@@ -0,0 +1,142 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: fixed "color" attribute of lights to be of type Color, rather than Vector4
+// this should have been put in a *long* time ago, but I somehow missed creating the updater between 3 and 4
+// fortunately, since all updates happen on untyped elements, it's reasonably safe to do this out of order
+//
+//=============================================================================
+
+#include "dmserializers.h"
+#include "dmebaseimporter.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/utlmap.h"
+#include <limits.h>
+
+
+//-----------------------------------------------------------------------------
+// Format converter
+//-----------------------------------------------------------------------------
+class CImportSFMV9 : public CSFMBaseImporter
+{
+	typedef CSFMBaseImporter BaseClass;
+public:
+	CImportSFMV9( const char *formatName, const char *nextFormatName );
+
+private:
+	virtual bool DoFixup( CDmElement *pSourceRoot );
+
+
+	void FixupElement( CDmElement *pElement );
+	// Fixes up all elements
+	void BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportSFMV9 s_ImportSFMV9( "sfm_v9", "sfm_v10" );
+
+void InstallSFMV9Importer( IDataModel *pFactory )
+{
+	pFactory->AddLegacyUpdater( &s_ImportSFMV9 );
+}
+
+
+//-----------------------------------------------------------------------------
+// Constructor
+//-----------------------------------------------------------------------------
+CImportSFMV9::CImportSFMV9( const char *formatName, const char *nextFormatName ) : 
+BaseClass( formatName, nextFormatName )	
+{
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV9::FixupElement( CDmElement *pElement )
+{
+	if ( !pElement )
+		return;
+
+	const char *pType = pElement->GetTypeString();
+	if ( !V_stricmp( pType, "DmeLight" ) ||
+		!V_stricmp( pType, "DmeDirectionalLight" ) ||
+		!V_stricmp( pType, "DmeProjectedLight" ) ||
+		!V_stricmp( pType, "DmePointLight" ) ||
+		!V_stricmp( pType, "DmeSpotLight" ) ||
+		!V_stricmp( pType, "DmeAmbientLight" ) )
+	{
+		const CDmAttribute *pOldAttr = pElement->GetAttribute( "color", AT_VECTOR4 );
+		if ( !pOldAttr )
+			return;
+
+		Color color;
+
+		{ // scoping this section of code since vecColor is invalid after RemoveAttribute
+			const Vector4D &vecColor = pOldAttr->GetValue< Vector4D >();
+			for ( int i = 0; i < 4; ++i )
+			{
+				color[ i ] = ( int )clamp( vecColor[ i ], 0.0f, 255.0f );
+			}
+
+			pElement->RemoveAttribute( "color" );
+		}
+
+		CDmAttribute *pNewAttr = pElement->AddAttribute( "color", AT_COLOR );
+		pNewAttr->SetValue( color );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Fixes up all elements
+//-----------------------------------------------------------------------------
+void CImportSFMV9::BuildList( CDmElement *pElement, CUtlRBTree< CDmElement *, int >& list )
+{
+	if ( !pElement )
+		return;
+
+	if ( list.Find( pElement ) != list.InvalidIndex() )
+		return;
+
+	list.Insert( pElement );
+
+	// Descend to bottom of tree, then do fixup coming back up the tree
+	for ( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->GetType() == AT_ELEMENT )
+		{
+			CDmElement *pElementAt = pAttribute->GetValueElement<CDmElement>( );
+			BuildList( pElementAt, list );
+			continue;
+		}
+
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+		{
+			CDmrElementArray<> array( pAttribute );
+			int nCount = array.Count();
+			for ( int i = 0; i < nCount; ++i )
+			{
+				CDmElement *pChild = array[ i ];
+				BuildList( pChild, list );
+			}
+			continue;
+		}
+	}
+}
+
+bool CImportSFMV9::DoFixup( CDmElement *pSourceRoot )
+{
+	CUtlRBTree< CDmElement *, int >	fixlist( 0, 0, DefLessFunc( CDmElement * ) );
+	BuildList( pSourceRoot, fixlist );
+	for ( int i = fixlist.FirstInorder(); i != fixlist.InvalidIndex() ; i = fixlist.NextInorder( i ) )
+	{
+		// Search and replace in the entire tree!
+		FixupElement( fixlist[ i ] );
+	}
+	return true;
+}

dmserializers/importvmf.cpp

@@ -0,0 +1,629 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "importkeyvaluebase.h"
+#include "dmserializers.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "datamodel/dmattribute.h"
+
+
+//-----------------------------------------------------------------------------
+// Serialization class for VMF files (map files)
+//-----------------------------------------------------------------------------
+class CImportVMF : public CImportKeyValueBase
+{
+public:
+	virtual const char *GetName() const { return "vmf"; }
+	virtual const char *GetDescription() const { return "Valve Map File"; }
+	virtual int GetCurrentVersion() const { return 0; } // doesn't store a version
+
+	bool Serialize( CUtlBuffer &outBuf, CDmElement *pRoot );
+	CDmElement* UnserializeFromKeyValues( KeyValues *pKeyValues );
+
+private:
+	// Reads a single entity
+	bool UnserializeEntityKey( CDmAttribute *pEntities, KeyValues *pKeyValues );
+
+	// Reads entity editor keys
+	bool UnserializeEntityEditorKey( CDmAttribute *pEditor, KeyValues *pKeyValues );
+
+	// Reads keys that we currently do nothing with
+	bool UnserializeUnusedKeys( DmElementHandle_t hOther, KeyValues *pKeyValues );
+
+	// Writes out all everything other than entities
+	bool SerializeOther( CUtlBuffer &buf, CDmAttribute *pOther, const char **ppFilter = 0 );
+
+	// Writes out all entities
+	bool SerializeEntities( CUtlBuffer &buf, CDmAttribute *pEntities );
+
+	// Writes out a single attribute recursively
+	bool SerializeAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, bool bElementArrays );
+
+	// Writes entity editor keys
+	bool SerializeEntityEditorKey( CUtlBuffer &buf, DmElementHandle_t hEditor );
+
+	// Updates the max hammer id
+	void UpdateMaxHammerId( KeyValues *pKeyValue );
+
+	// Max id read from the file
+	int m_nMaxHammerId;
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportVMF s_ImportVMF;
+
+void InstallVMFImporter( IDataModel *pFactory )
+{
+	pFactory->AddSerializer( &s_ImportVMF );
+}
+
+
+//-----------------------------------------------------------------------------
+// Deals with poorly-named key values for the DME system
+//-----------------------------------------------------------------------------
+static const char *s_pKeyRemapNames[][2] = 
+{
+	{ "id", "__id" },
+	{ "name", "__name" },
+	{ "type", "__type" },
+	{ NULL, NULL },
+};
+
+
+//-----------------------------------------------------------------------------
+// Gets remap name for unserialization/serailzation
+//-----------------------------------------------------------------------------
+static const char *GetRemapName( const char *pName, bool bSerialization )
+{
+	for ( int i = 0; s_pKeyRemapNames[i][0]; ++i )
+	{
+		if ( !Q_stricmp( pName, s_pKeyRemapNames[i][bSerialization] ) )
+			return s_pKeyRemapNames[i][1 - bSerialization];
+	}
+	return pName;
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out a single attribute recursively
+//-----------------------------------------------------------------------------
+bool CImportVMF::SerializeAttribute( CUtlBuffer &buf, CDmAttribute *pAttribute, bool bElementArrays )
+{
+	if ( pAttribute->IsFlagSet( FATTRIB_STANDARD | FATTRIB_DONTSAVE ) )
+		return true;
+
+	const char *pFieldName = GetRemapName( pAttribute->GetName(), true );
+	if ( !Q_stricmp( pFieldName, "editorType" ) )
+		return true;
+
+	if ( !IsArrayType( pAttribute->GetType() ) )
+	{
+		if ( !bElementArrays )
+		{
+			buf.Printf( "\"%s\" ", pFieldName );
+			if ( pAttribute->GetType() != AT_STRING )
+			{
+				buf.Printf( "\"" );
+			}
+			g_pDataModel->SetSerializationDelimiter( GetCStringCharConversion() );
+			pAttribute->Serialize( buf );
+			g_pDataModel->SetSerializationDelimiter( NULL );
+			if ( pAttribute->GetType() != AT_STRING )
+			{
+				buf.Printf( "\"" );
+			}
+			buf.Printf( "\n" );
+		}
+	}
+	else
+	{
+		if ( bElementArrays )
+		{
+			Assert( pAttribute->GetType() == AT_ELEMENT_ARRAY );
+			if ( !SerializeOther( buf, pAttribute ) )
+				return false;
+		}
+	}
+
+	return true;
+}
+
+	
+//-----------------------------------------------------------------------------
+// Writes out all everything other than entities
+//-----------------------------------------------------------------------------
+bool CImportVMF::SerializeOther( CUtlBuffer &buf, CDmAttribute *pOther, const char **ppFilter )
+{
+	CDmrElementArray<> array( pOther );
+	int nCount = array.Count();
+	for ( int i = 0; i < nCount; ++i )
+	{
+		CDmElement *pElement = array[i];
+		const char *pElementName = pElement->GetName();
+		if ( ppFilter )
+		{
+			int j;
+			for ( j = 0; ppFilter[j]; ++j )
+			{
+				if ( !Q_stricmp( pElementName, ppFilter[j] ) )
+					break;
+			}
+
+			if ( !ppFilter[j] )
+				continue;
+		}
+
+		int nLen = Q_strlen( pElementName ) + 1;
+		char *pTemp = (char*)_alloca( nLen );
+		Q_strncpy( pTemp, pElementName, nLen );
+		Q_strlower( pTemp );
+		buf.Printf( "%s\n", pTemp );
+		buf.Printf( "{\n" );
+		buf.PushTab();
+
+		// Normal attributes first
+	    for( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+		{
+			if ( !SerializeAttribute( buf, pAttribute, false ) )
+				return false;
+		}
+
+		// Subkeys later
+	    for( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+		{
+			if ( !SerializeAttribute( buf, pAttribute, true ) )
+				return false;
+		}
+
+		buf.PopTab();
+		buf.Printf( "}\n" );
+	}
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes entity editor keys
+//-----------------------------------------------------------------------------
+bool CImportVMF::SerializeEntityEditorKey( CUtlBuffer &buf, DmElementHandle_t hEditor )
+{
+	CDmElement *pEditorElement = g_pDataModel->GetElement( hEditor );
+	if ( !pEditorElement )
+		return true;
+
+	buf.Printf( "editor\n" );
+	buf.Printf( "{\n" );
+	buf.PushTab();
+
+	{
+		CDmAttribute *pAttribute = pEditorElement->GetAttribute( "color" );
+		if ( pAttribute )
+		{
+			Color c = pAttribute->GetValue<Color>();
+			buf.Printf( "\"color\" \"%d %d %d\"\n", c.r(), c.g(), c.b() );
+		}
+	}
+	PrintIntAttribute( pEditorElement, buf, "id" ); // FIXME - id is a DmObjectId_t!!! This should never print anything!
+	PrintStringAttribute( pEditorElement, buf, "comments" );
+	PrintBoolAttribute( pEditorElement, buf, "visgroupshown" );
+	PrintBoolAttribute( pEditorElement, buf, "visgroupautoshown" );
+
+	for ( CDmAttribute *pAttribute = pEditorElement->FirstAttribute(); pAttribute != NULL; pAttribute = pAttribute->NextAttribute() )
+	{
+		if ( pAttribute->IsFlagSet( FATTRIB_STANDARD | FATTRIB_DONTSAVE ) )
+			continue;
+
+		const char *pKeyName = pAttribute->GetName();
+		if ( Q_stricmp( pKeyName, "color" ) && Q_stricmp( pKeyName, "id" ) && 
+			Q_stricmp( pKeyName, "comments" ) && Q_stricmp( pKeyName, "visgroupshown" ) &&
+			Q_stricmp( pKeyName, "visgroupautoshown" ) )
+		{
+			PrintStringAttribute( pEditorElement, buf, pKeyName );
+		}
+	}
+
+	buf.PopTab();
+	buf.Printf( "}\n" );
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out all entities
+//-----------------------------------------------------------------------------
+bool CImportVMF::SerializeEntities( CUtlBuffer &buf, CDmAttribute *pEntities )
+{
+	// FIXME: Make this serialize in the order in which it appears in the FGD
+	// to minimize diffs
+	CDmrElementArray<> array( pEntities );
+
+	int nCount = array.Count();
+	for ( int i = 0; i < nCount; ++i )
+	{
+		CDmElement *pElement = array[i];
+		buf.Printf( "entity\n" );
+		buf.Printf( "{\n" );
+		buf.PushTab();
+		buf.Printf( "\"id\" \"%s\"\n", pElement->GetName() );
+
+	    for( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+		{
+			// Do 'editor' at the end to preserve ordering and not make terrible diffs
+			if ( !Q_stricmp( pAttribute->GetName(), "editor" ) )
+				continue;
+
+			if ( !SerializeAttribute( buf, pAttribute, false ) )
+				return false;
+		}
+
+	    for( CDmAttribute *pAttribute = pElement->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+		{
+			// Do 'editor' at the end to preserve ordering and not make terrible diffs
+			if ( !Q_stricmp( pAttribute->GetName(), "editor" ) )
+				continue;
+
+			if ( !SerializeAttribute( buf, pAttribute, true ) )
+				return false;
+		}
+
+		// Do the 'editor'
+		CDmAttribute *pEditor = pElement->GetAttribute( "editor" );
+		if ( pEditor )
+		{
+			SerializeEntityEditorKey( buf, pEditor->GetValue<DmElementHandle_t>() );
+		}
+
+		buf.PopTab();
+		buf.Printf( "}\n" );
+	}
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out a new VMF file
+//-----------------------------------------------------------------------------
+bool CImportVMF::Serialize( CUtlBuffer &buf, CDmElement *pRoot )
+{
+	// This is done in this strange way (namely, serializing other twice) to minimize diffs
+	const char *pOtherFilter1[] = 
+	{
+		"versioninfo", "visgroups", "viewsettings", "world", NULL
+	};
+
+	const char *pOtherFilter2[] = 
+	{
+		"cameras", "cordon", "hidden", NULL
+	};
+
+	CDmAttribute *pOther = pRoot->GetAttribute( "other" );
+	if ( pOther && pOther->GetType() == AT_ELEMENT_ARRAY )
+	{
+		if ( !SerializeOther( buf, pOther, pOtherFilter1 ) )
+			return false;
+	}
+
+	// Serialize entities
+	CDmAttribute *pEntities = pRoot->GetAttribute( "entities" );
+	if ( pEntities && pEntities->GetType() == AT_ELEMENT_ARRAY )
+	{
+		if ( !SerializeEntities( buf, pEntities ) )
+			return false;
+	}
+
+	if ( pOther && pOther->GetType() == AT_ELEMENT_ARRAY )
+	{
+		if ( !SerializeOther( buf, pOther, pOtherFilter2 ) )
+			return false;
+	}
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Updates the max hammer id
+//-----------------------------------------------------------------------------
+void CImportVMF::UpdateMaxHammerId( KeyValues *pField )
+{
+	if ( !Q_stricmp( pField->GetName(), "id" ) )
+	{
+		int nId = atoi( pField->GetString() );
+		if ( nId > m_nMaxHammerId )
+		{
+			m_nMaxHammerId = nId;
+		}
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Reads entity editor keys
+//-----------------------------------------------------------------------------
+bool CImportVMF::UnserializeEntityEditorKey( CDmAttribute *pEditorAttribute, KeyValues *pKeyValues )
+{
+	CDmElement *pEditor;
+	DmElementHandle_t hEditor = pEditorAttribute->GetValue<DmElementHandle_t>();
+	if ( hEditor == DMELEMENT_HANDLE_INVALID )
+	{
+		pEditor = CreateDmElement( "DmElement", "editor", NULL );;
+		if ( !pEditor )
+			return false;
+		hEditor = pEditor->GetHandle();
+		pEditorAttribute->SetValue( hEditor );
+	}
+	else
+	{
+		pEditor = g_pDataModel->GetElement( hEditor );
+	}
+
+	int r, g, b;
+	if ( sscanf( pKeyValues->GetString( "color", "" ), "%d %d %d", &r, &g, &b ) == 3 )
+	{
+		Color c( r, g, b, 255 );
+		if ( !pEditor->SetValue( "color", c ) )
+			return false;
+	}
+	KeyValues *pIdKey = pKeyValues->FindKey( "id" );
+	if ( pIdKey )
+	{
+		UpdateMaxHammerId( pIdKey );
+	}
+	AddIntAttribute( pEditor, pKeyValues, "id" );
+	AddStringAttribute( pEditor, pKeyValues, "comments" );
+	AddBoolAttribute( pEditor, pKeyValues, "visgroupshown" );
+	AddBoolAttribute( pEditor, pKeyValues, "visgroupautoshown" );
+
+	for ( KeyValues *pUserKey = pKeyValues->GetFirstValue(); pUserKey != NULL; pUserKey = pUserKey->GetNextValue() )
+	{
+		const char *pKeyName = pUserKey->GetName();
+		if ( Q_stricmp( pKeyName, "color" ) && Q_stricmp( pKeyName, "id" ) && 
+			Q_stricmp( pKeyName, "comments" ) && Q_stricmp( pKeyName, "visgroupshown" ) &&
+			Q_stricmp( pKeyName, "visgroupautoshown" ) )
+		{
+			AddStringAttribute( pEditor, pKeyValues, pKeyName );
+		}
+	}
+
+	return true;
+}
+
+	
+//-----------------------------------------------------------------------------
+// Reads a single entity
+//-----------------------------------------------------------------------------
+bool CImportVMF::UnserializeEntityKey( CDmAttribute *pEntities, KeyValues *pKeyValues )
+{
+	CDmElement *pEntity = CreateDmElement( "DmeVMFEntity", pKeyValues->GetString( "id", "-1" ), NULL );
+	if ( !pEntity )
+		return false;
+
+	CDmrElementArray<> array( pEntities );
+	array.AddToTail( pEntity );
+
+	// Each act busy needs to have an editortype associated with it so it displays nicely in editors
+	pEntity->SetValue( "editorType", "vmfEntity" );
+
+	const char *pClassName = pKeyValues->GetString( "classname", NULL );
+	if ( !pClassName )
+		return false;
+
+	// Read the actual fields
+	for ( KeyValues *pField = pKeyValues->GetFirstValue(); pField != NULL; pField = pField->GetNextValue() )
+	{
+		// FIXME: Knowing the FGD here would be useful for type determination.
+		// Look up the field by name based on class name
+		// In the meantime, just use the keyvalues type?
+		char pFieldName[512];
+		Q_strncpy( pFieldName, pField->GetName(), sizeof(pFieldName) );
+		Q_strlower( pFieldName );
+
+		// Don't do id: it's used as the name
+		// Not to mention it's a protected name
+		if ( !Q_stricmp( pFieldName, "id" ) )
+		{
+			UpdateMaxHammerId( pField );
+			continue;
+		}
+
+		// Type, name, and editortype are protected names
+		Assert( Q_stricmp( pFieldName, "type" ) && Q_stricmp( pFieldName, "name" ) && Q_stricmp( pFieldName, "editortype" ) );
+
+		switch( pField->GetDataType() )
+		{
+		case KeyValues::TYPE_INT:
+			if ( !AddIntAttributeFlags( pEntity, pKeyValues, pFieldName, FATTRIB_USERDEFINED ) )
+				return false;
+			break;
+
+		case KeyValues::TYPE_FLOAT:
+			if ( !AddFloatAttributeFlags( pEntity, pKeyValues, pFieldName, FATTRIB_USERDEFINED ) )
+				return false;
+			break;
+
+		case KeyValues::TYPE_STRING:
+			{
+				const char* pString = pField->GetString();
+				if (!pString || !pString[0])
+					return false;
+
+				// Look for vectors
+				Vector4D v;
+				if ( sscanf( pString, "%f %f %f %f", &v.x, &v.y, &v.z, &v.w ) == 4 )
+				{
+					if ( !pEntity->SetValue( pFieldName, v ) )
+						return false;
+					CDmAttribute *pAttribute = pEntity->GetAttribute( pFieldName );
+					pAttribute->AddFlag( FATTRIB_USERDEFINED );
+				}
+				else if ( sscanf( pString, "%f %f %f", &v.x, &v.y, &v.z ) == 3 )
+				{
+					if ( !pEntity->SetValue( pFieldName, v.AsVector3D() ) )
+					{
+						QAngle ang( v.x, v.y, v.z );
+						if ( !pEntity->SetValue( pFieldName, ang ) )
+							return false;
+					}
+					CDmAttribute *pAttribute = pEntity->GetAttribute( pFieldName );
+					pAttribute->AddFlag( FATTRIB_USERDEFINED );
+				}
+				else
+				{
+					if ( !AddStringAttributeFlags( pEntity, pKeyValues, pFieldName, FATTRIB_USERDEFINED ) )
+						return false;
+				}
+			}
+			break;
+		}
+	}
+
+	// Read the subkeys
+	CDmAttribute *pEditor = pEntity->AddAttribute( "editor", AT_ELEMENT );
+	CDmrElementArray<> otherKeys( pEntity->AddAttribute( "other", AT_ELEMENT_ARRAY ) );
+	for ( KeyValues *pSubKey = pKeyValues->GetFirstTrueSubKey(); pSubKey != NULL; pSubKey = pSubKey->GetNextTrueSubKey() )
+	{
+		bool bOk = false;
+		if ( !Q_stricmp( pSubKey->GetName(), "editor" ) )
+		{
+			bOk = UnserializeEntityEditorKey( pEditor, pSubKey );
+		}
+		else
+		{
+			// We don't currently do anything with the other keys
+			CDmElement *pOther = CreateDmElement( "DmElement", pSubKey->GetName(), NULL );
+			otherKeys.AddToTail( pOther );
+			bOk = UnserializeUnusedKeys( pOther->GetHandle(), pSubKey );
+		}
+
+		if ( !bOk )
+			return false;
+	}
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Reads keys that we currently do nothing with
+//-----------------------------------------------------------------------------
+bool CImportVMF::UnserializeUnusedKeys( DmElementHandle_t hOther, KeyValues *pKeyValues )
+{
+	CDmElement *pOther = g_pDataModel->GetElement( hOther );
+
+	// Read the actual fields
+	for ( KeyValues *pField = pKeyValues->GetFirstValue(); pField != NULL; pField = pField->GetNextValue() )
+	{
+		UpdateMaxHammerId( pField );
+		const char *pFieldName = GetRemapName( pField->GetName(), false );
+		pOther->SetValue( pFieldName, pField->GetString() );
+	}
+
+	// Read the subkeys
+	CDmrElementArray<> subKeys( pOther->AddAttribute( "subkeys", AT_ELEMENT_ARRAY ) );
+	for ( KeyValues *pSubKey = pKeyValues->GetFirstTrueSubKey(); pSubKey != NULL; pSubKey = pSubKey->GetNextTrueSubKey() )
+	{
+		CDmElement *pSubElement = CreateDmElement( "DmElement", pSubKey->GetName(), NULL );
+		subKeys.AddToTail( pSubElement );
+		if ( !UnserializeUnusedKeys( pSubElement->GetHandle(), pSubKey ) )
+			return false;
+	}
+	return true;
+}
+
+	
+/*
+//-----------------------------------------------------------------------------
+// Reads the cordon data
+//-----------------------------------------------------------------------------
+bool CImportVMF::UnserializeCordonKey( IDmAttributeElement *pCordon, KeyValues *pKeyValues )
+{
+	DmElementHandle_t hCordon = pCordon->GetValue().Get();
+	if ( hCordon == DMELEMENT_HANDLE_INVALID )
+	{
+		hCordon = CreateDmElement( "DmElement", "cordon", NULL );
+		if ( hCordon == DMELEMENT_HANDLE_INVALID )
+			return false;
+		pCordon->SetValue( hCordon );
+	}
+
+	AddBoolAttribute( hCordon, pKeyValues, "active" );
+
+	Vector v;
+	if ( sscanf( pKeyValues->GetString( "mins", "" ), "(%f %f %f)", &v.x, &v.y, &v.z ) == 3 )
+	{
+		if ( !DmElementAddAttribute( hCordon, "mins", v ) )
+			return false;
+	}
+	if ( sscanf( pKeyValues->GetString( "maxs", "" ), "(%f %f %f)", &v.x, &v.y, &v.z ) == 3 )
+	{
+		if ( !DmElementAddAttribute( hCordon, "maxs", v ) )
+			return false;
+	}
+	return true;
+}
+*/
+
+
+//-----------------------------------------------------------------------------
+// Main entry point for the unserialization
+//-----------------------------------------------------------------------------
+CDmElement* CImportVMF::UnserializeFromKeyValues( KeyValues *pKeyValues )
+{
+	m_nMaxHammerId = 0;
+
+	// Create the main element
+	CDmElement *pElement = CreateDmElement( "DmElement", "VMF", NULL );
+	if ( !pElement )
+		return NULL;
+
+	// Each vmf needs to have an editortype associated with it so it displays nicely in editors
+	pElement->SetValue( "editorType", "VMF" );
+
+	// The VMF is a series of keyvalue blocks; either 
+	// 'entity', 'cameras', 'cordon', 'world', 'versioninfo', or 'viewsettings' 
+	CDmAttribute *pEntityArray = pElement->AddAttribute( "entities", AT_ELEMENT_ARRAY );
+
+	// All main keys are root keys
+	CDmrElementArray<> otherKeys( pElement->AddAttribute( "other", AT_ELEMENT_ARRAY ) );
+	for ( ; pKeyValues != NULL; pKeyValues = pKeyValues->GetNextKey() )
+	{
+		bool bOk = false;
+		if ( !Q_stricmp( pKeyValues->GetName(), "entity" ) )
+		{
+			bOk = UnserializeEntityKey( pEntityArray, pKeyValues );
+		}
+		else
+		{
+			// We don't currently do anything with 
+			CDmElement *pOther = CreateDmElement( "DmElement", pKeyValues->GetName(), NULL );
+			otherKeys.AddToTail( pOther );
+			bOk = UnserializeUnusedKeys( pOther->GetHandle(), pKeyValues );
+		}
+
+		if ( !bOk )
+		{
+			Warning( "Error importing VMF element %s\n", pKeyValues->GetName() );
+			return NULL;
+		}
+	}
+
+	// Resolve all element references recursively
+	RecursivelyResolveElement( pElement );
+
+	// Add the max id read in from the file to the root entity
+	pElement->SetValue( "maxHammerId", m_nMaxHammerId );
+
+	return pElement;
+}

dmserializers/importvmt.cpp

@@ -0,0 +1,738 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "importkeyvaluebase.h"
+#include "dmserializers.h"
+#include "datamodel/idatamodel.h"
+#include "datamodel/dmelement.h"
+#include "tier1/KeyValues.h"
+#include "tier1/utlbuffer.h"
+#include "datamodel/dmattribute.h"
+#include "filesystem.h"
+#include "tier2/tier2.h"
+
+
+//-----------------------------------------------------------------------------
+// Serialization class for Key Values
+//-----------------------------------------------------------------------------
+class CImportVMT : public CImportKeyValueBase
+{
+public:
+	virtual const char *GetName() const { return "vmt"; }
+	virtual const char *GetDescription() const { return "Valve Material File"; }
+	virtual int GetCurrentVersion() const { return 0; } // doesn't store a version
+
+	bool Serialize( CUtlBuffer &outBuf, CDmElement *pRoot );
+	CDmElement* UnserializeFromKeyValues( KeyValues *pKeyValues );
+
+private:
+	// Unserialize fallbacks
+	bool UnserializeFallbacks( CDmElement *pRoot, KeyValues *pFallbackKeyValues );
+
+	// Unserialize proxies
+	bool UnserializeProxies( CDmElement *pRoot, KeyValues *pKeyValues );
+
+	// Creates a shader parameter from a key value
+	bool UnserializeShaderParam( CDmElement *pRoot, KeyValues* pKeyValue );
+
+	// Creates a matrix material var
+	bool CreateMatrixMaterialVarFromKeyValue( CDmElement *pRoot, const char *pParamName, const char *pString );
+
+	// Creates a vector shader parameter
+	bool CreateVectorMaterialVarFromKeyValue( CDmElement *pRoot, const char *pParamName, const char *pString );
+
+	// Writes out a single shader parameter
+	bool SerializeShaderParameter( CUtlBuffer &buf, CDmAttribute *pAttribute );
+
+	// Writes out all shader parameters
+	bool SerializeShaderParameters( CUtlBuffer &buf, CDmElement *pRoot );
+
+	// Writes out all shader fallbacks
+	bool SerializeFallbacks( CUtlBuffer &buf, CDmElement *pRoot );
+
+	// Writes out all material proxies
+	bool SerializeProxies( CUtlBuffer &buf, CDmElement *pRoot );
+
+	// Handle patch files
+	void ExpandPatchFile( KeyValues *pKeyValues );
+};
+
+
+//-----------------------------------------------------------------------------
+// Singleton instance
+//-----------------------------------------------------------------------------
+static CImportVMT s_ImportVMT;
+
+void InstallVMTImporter( IDataModel *pFactory )
+{
+	pFactory->AddSerializer( &s_ImportVMT );
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out a single shader parameter
+//-----------------------------------------------------------------------------
+bool CImportVMT::SerializeShaderParameter( CUtlBuffer &buf, CDmAttribute *pAttribute )
+{
+	// We have a shader parameter at this point.
+	switch ( pAttribute->GetType() )
+	{
+	case AT_INT:
+		buf.Printf( "\"%s\" \"%d\"\n", pAttribute->GetName(), pAttribute->GetValue<int>( ) );
+		break;
+
+	case AT_BOOL:
+		buf.Printf( "\"%s\" \"%d\"\n", pAttribute->GetName(), pAttribute->GetValue<bool>( ) );
+		break;
+
+	case AT_FLOAT:
+		buf.Printf( "\"%s\" \"%f\"\n", pAttribute->GetName(), pAttribute->GetValue<float>( ) );
+		break;
+
+	case AT_STRING:
+		buf.Printf( "\"%s\" \"%s\"\n", pAttribute->GetName(), pAttribute->GetValue<CUtlString>( ).Get() );
+		break;
+
+	case AT_VECTOR2:
+		{
+			const Vector2D &vec = pAttribute->GetValue<Vector2D>( );
+			buf.Printf( "\"%s\" \"[ %f %f ]\"\n", pAttribute->GetName(), vec.x, vec.y );
+		}
+		break;
+
+	case AT_VECTOR3:
+		{
+			const Vector &vec = pAttribute->GetValue<Vector>( );
+			buf.Printf( "\"%s\" \"[ %f %f %f ]\"\n", pAttribute->GetName(), vec.x, vec.y, vec.z );
+		}
+		break;
+
+	case AT_VECTOR4:
+		{
+			const Vector4D &vec = pAttribute->GetValue<Vector4D>( );
+			buf.Printf( "\"%s\" \"[ %f %f %f %f ]\"\n", pAttribute->GetName(), vec.x, vec.y, vec.z, vec.w );
+		}
+		break;
+
+	case AT_COLOR:
+		{
+			// NOTE: VMTs only support 3 component color (no alpha)
+			const Color &color = pAttribute->GetValue<Color>( );
+			buf.Printf( "\"%s\" \"{ %d %d %d }\"\n", pAttribute->GetName(), color.r(), color.g(), color.b() );
+		}
+		break;
+
+	case AT_VMATRIX:
+		{
+			const VMatrix &mat = pAttribute->GetValue<VMatrix>( );
+			buf.Printf( "\"%s\" \"[ %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f ]\"\n", pAttribute->GetName(), 
+				mat[0][0], mat[0][1], mat[0][2], mat[0][3],
+				mat[1][0], mat[1][1], mat[1][2], mat[1][3],
+				mat[2][0], mat[2][1], mat[2][2], mat[2][3],
+				mat[3][0], mat[3][1], mat[3][2], mat[3][3] );
+		}
+		break;
+
+	default:
+		Warning( "Attempted to serialize an unsupported shader parameter type %s (%s)\n", 
+			pAttribute->GetName(), g_pDataModel->GetAttributeNameForType( pAttribute->GetType() ) );
+		return false;
+	}
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out all shader parameters
+//-----------------------------------------------------------------------------
+bool CImportVMT::SerializeShaderParameters( CUtlBuffer &buf, CDmElement *pRoot )
+{
+	for ( CDmAttribute *pAttribute = pRoot->FirstAttribute(); pAttribute; pAttribute = pAttribute->NextAttribute() )
+	{
+		// Skip the standard attributes
+		if ( pAttribute->IsFlagSet( FATTRIB_STANDARD ) )
+			continue;
+
+		// Skip the shader name
+		const char *pName = pAttribute->GetName();
+		if ( !Q_stricmp( pAttribute->GetName(), "shader" ) )
+			continue;
+
+		// Names that don't start with a $ or a % are not shader parameters
+		if ( pName[0] != '$' && pName[0] != '%' )
+			continue;
+
+		// Skip element array children; we'll handle them separately.
+		if ( pAttribute->GetType() == AT_ELEMENT_ARRAY )
+			continue;
+
+		// Write out the shader parameter
+		if ( !SerializeShaderParameter( buf, pAttribute ) )
+			return false;
+	}
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out all shader fallbacks
+//-----------------------------------------------------------------------------
+bool CImportVMT::SerializeFallbacks( CUtlBuffer &buf, CDmElement *pRoot )
+{
+	if ( !pRoot->HasAttribute( "fallbacks" ) )
+		return true;
+
+	CDmAttribute *pFallbacks = pRoot->GetAttribute( "fallbacks" );
+	if ( pFallbacks->GetType() != AT_ELEMENT_ARRAY )
+		return false;
+
+	CDmrElementArray<> array( pFallbacks );
+	int nCount = array.Count();
+	for ( int i = 0; i < nCount; ++i )
+	{
+		CDmElement *pFallback = array[i];
+		Assert( pFallback );
+
+		PrintStringAttribute( pFallback, buf, "shader", false, true );
+		buf.Printf( "{\n" );
+		buf.PushTab();
+		if ( !SerializeShaderParameters( buf, pFallback ) )
+			return false;
+		buf.PopTab();
+		buf.Printf( "}\n" );
+	}
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out all material proxies
+//-----------------------------------------------------------------------------
+bool CImportVMT::SerializeProxies( CUtlBuffer &buf, CDmElement *pRoot )
+{
+	if ( !pRoot->HasAttribute( "proxies" ) )
+		return true;
+
+	CDmAttribute *pProxies = pRoot->GetAttribute( "proxies" );
+	if ( pProxies->GetType() != AT_ELEMENT_ARRAY )
+		return false;
+
+	CDmrElementArray<> array( pProxies );
+	int nCount = array.Count();
+	if ( nCount == 0 )
+		return true;
+
+	buf.Printf( "\"Proxies\"\n" );
+	buf.Printf( "{\n" );
+	buf.PushTab();
+	for ( int i = 0; i < nCount; ++i )
+	{
+		CDmElement *pProxy = array[i];
+		Assert( pProxy );
+
+		PrintStringAttribute( pProxy, buf, "proxyType", false, true );
+		buf.Printf( "{\n" );
+		buf.PushTab();
+		if ( !SerializeShaderParameters( buf, pProxy ) )
+			return false;
+		buf.PopTab();
+		buf.Printf( "}\n" );
+	}
+	buf.PopTab();
+	buf.Printf( "}\n" );
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Writes out a new vmt file
+//-----------------------------------------------------------------------------
+bool CImportVMT::Serialize( CUtlBuffer &buf, CDmElement *pRoot )
+{
+	PrintStringAttribute( pRoot, buf, "shader", false, true );
+	buf.Printf( "{\n" );
+	buf.PushTab();
+
+	if ( !SerializeShaderParameters( buf, pRoot ) )
+		return false;
+
+	if ( !SerializeFallbacks( buf, pRoot ) )
+		return false;
+
+	if ( !SerializeProxies( buf, pRoot ) )
+		return false;
+
+	buf.PopTab();
+	buf.Printf( "}\n" );
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Parser utilities
+//-----------------------------------------------------------------------------
+static inline bool IsWhitespace( char c )
+{
+	return c == ' ' || c == '\t';
+}
+
+static inline bool IsEndline( char c )
+{
+	return c == '\n' || c == '\0';
+}
+
+static inline bool IsVector( char const* v )
+{
+	while (IsWhitespace(*v))
+	{
+		++v;
+		if (IsEndline(*v))
+			return false;
+	}
+	return *v == '[' || *v == '{';
+}
+
+
+//-----------------------------------------------------------------------------
+// Creates a vector material var
+//-----------------------------------------------------------------------------
+int ParseVectorFromKeyValueString( const char *pParamName, const char* pScan, const char *pMaterialName, float vecVal[4] )
+{
+	bool divideBy255 = false;
+
+	// skip whitespace
+	while( IsWhitespace(*pScan) )
+	{
+		++pScan;
+	}
+
+	if( *pScan == '{' )
+	{
+		divideBy255 = true;
+	}
+	else
+	{
+		Assert( *pScan == '[' );
+	}
+	
+	// skip the '['
+	++pScan;
+	int i;
+	for( i = 0; i < 4; i++ )
+	{
+		// skip whitespace
+		while( IsWhitespace(*pScan) )
+		{
+			++pScan;
+		}
+
+		if( IsEndline(*pScan) || *pScan == ']' || *pScan == '}' )
+		{
+			if (*pScan != ']' && *pScan != '}')
+			{
+				Warning( "Warning in .VMT file (%s): no ']' or '}' found in vector key \"%s\".\n"
+					"Did you forget to surround the vector with \"s?\n", pMaterialName, pParamName );
+			}
+
+			// allow for vec2's, etc.
+			vecVal[i] = 0.0f;
+			break;
+		}
+
+		char* pEnd;
+
+		vecVal[i] = strtod( pScan, &pEnd );
+		if (pScan == pEnd)
+		{
+			Warning( "Error in .VMT file: error parsing vector element \"%s\" in \"%s\"\n", pParamName, pMaterialName );
+			return 0;
+		}
+
+		pScan = pEnd;
+	}
+
+	if( divideBy255 )
+	{
+		vecVal[0] *= ( 1.0f / 255.0f );
+		vecVal[1] *= ( 1.0f / 255.0f );
+		vecVal[2] *= ( 1.0f / 255.0f );
+		vecVal[3] *= ( 1.0f / 255.0f );
+	}
+
+	return i;
+}
+
+
+//-----------------------------------------------------------------------------
+// Sets shader parameter attributes
+//-----------------------------------------------------------------------------
+template< class T >
+inline bool SetShaderParamAttribute( CDmElement *pElement, const char *pAttributeName, const T &value )
+{
+	if ( !pElement )
+		return false;
+
+	if ( !pElement->SetValue( pAttributeName, value ) )
+		return false;
+
+	CDmAttribute *pAttribute = pElement->GetAttribute( pAttributeName );
+	pAttribute->AddFlag( FATTRIB_USERDEFINED );
+	return true;
+}
+
+inline bool SetShaderParamAttribute( CDmElement *pElement, const char *pAttributeName, const char *value )
+{
+	if ( !pElement )
+		return false;
+
+	if ( !pElement->SetValue( pAttributeName, value ) )
+		return false;
+
+	CDmAttribute *pAttribute = pElement->GetAttribute( pAttributeName );
+	pAttribute->AddFlag( FATTRIB_USERDEFINED );
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Creates a vector shader parameter
+//-----------------------------------------------------------------------------
+bool CImportVMT::CreateVectorMaterialVarFromKeyValue( CDmElement *pElement, const char *pParamName, const char *pString )
+{
+	Vector4D vecVal;
+	int nDim = ParseVectorFromKeyValueString( pParamName, pString, FileName(), vecVal.Base() );
+	if ( nDim == 0 )
+		return false;
+
+	// Create the variable!
+	switch ( nDim )
+	{
+	case 1:
+		return SetShaderParamAttribute( pElement, pParamName, vecVal[0] );
+	case 2:
+		return SetShaderParamAttribute( pElement, pParamName, vecVal.AsVector2D() );
+	case 3:
+		return SetShaderParamAttribute( pElement, pParamName, vecVal.AsVector3D() );
+	case 4:
+		return SetShaderParamAttribute( pElement, pParamName, vecVal );
+	}
+
+	return false;
+}
+
+
+//-----------------------------------------------------------------------------
+// Creates a matrix shader parameter
+//-----------------------------------------------------------------------------
+bool CImportVMT::CreateMatrixMaterialVarFromKeyValue( CDmElement *pElement, const char *pParamName, const char *pScan )
+{
+	// Matrices can be specified one of two ways:
+	// [ # # # #  # # # #  # # # #  # # # # ]
+	// or
+	// center # # scale # # rotate # translate # #
+
+	VMatrix mat;
+	int count = sscanf( pScan, " [ %f %f %f %f  %f %f %f %f  %f %f %f %f  %f %f %f %f ]",
+		&mat.m[0][0], &mat.m[0][1], &mat.m[0][2], &mat.m[0][3],
+		&mat.m[1][0], &mat.m[1][1], &mat.m[1][2], &mat.m[1][3],
+		&mat.m[2][0], &mat.m[2][1], &mat.m[2][2], &mat.m[2][3],
+		&mat.m[3][0], &mat.m[3][1], &mat.m[3][2], &mat.m[3][3] );
+	if (count == 16)
+	{
+		return SetShaderParamAttribute( pElement, pParamName, mat );
+	}
+
+	Vector2D scale, center;
+	float angle;
+	Vector2D translation;
+	count = sscanf( pScan, " center %f %f scale %f %f rotate %f translate %f %f",
+		&center.x, &center.y, &scale.x, &scale.y, &angle, &translation.x, &translation.y );
+	if (count != 7)
+		return false;
+
+	VMatrix temp;
+	MatrixBuildTranslation( mat, -center.x, -center.y, 0.0f );
+	MatrixBuildScale( temp, scale.x, scale.y, 1.0f );
+	MatrixMultiply( temp, mat, mat );
+	MatrixBuildRotateZ( temp, angle );
+	MatrixMultiply( temp, mat, mat );
+	MatrixBuildTranslation( temp, center.x + translation.x, center.y + translation.y, 0.0f );
+	MatrixMultiply( temp, mat, mat );
+
+	// Create the variable!
+	return SetShaderParamAttribute( pElement, pParamName, mat );
+}
+
+
+//-----------------------------------------------------------------------------
+// Creates a shader parameter from a key value
+//-----------------------------------------------------------------------------
+bool CImportVMT::UnserializeShaderParam( CDmElement *pRoot, KeyValues* pKeyValues )
+{
+	char pParamName[512];
+	Q_strncpy( pParamName, pKeyValues->GetName(), sizeof(pParamName) );
+	Q_strlower( pParamName );
+
+	switch( pKeyValues->GetDataType() )
+	{
+	case KeyValues::TYPE_INT:
+		return SetShaderParamAttribute( pRoot, pParamName, pKeyValues->GetInt() );
+
+	case KeyValues::TYPE_FLOAT:
+		return SetShaderParamAttribute( pRoot, pParamName, pKeyValues->GetFloat() );
+
+	case KeyValues::TYPE_STRING:
+		{
+			char const* pString = pKeyValues->GetString();
+
+			// Only valid if it's a texture attribute
+			if ( !pString || !pString[0] )
+				return SetShaderParamAttribute( pRoot, pParamName, pString );
+
+			// Look for matrices
+			if ( CreateMatrixMaterialVarFromKeyValue( pRoot, pParamName, pString ) )
+				return true;
+
+			// Look for vectors
+			if ( !IsVector( pString ) )
+				return SetShaderParamAttribute( pRoot, pParamName, pString );
+
+			// Parse the string as a vector...
+			return CreateVectorMaterialVarFromKeyValue( pRoot, pParamName, pString );
+		}
+	}
+
+	return false;
+}
+
+
+//-----------------------------------------------------------------------------
+// Unserialize proxies
+//-----------------------------------------------------------------------------
+bool CImportVMT::UnserializeProxies( CDmElement *pElement, KeyValues *pKeyValues )
+{
+	// Create a child element array to contain all material proxies
+	CDmAttribute *pProxies = pElement->AddAttribute( "proxies", AT_ELEMENT_ARRAY );
+	if ( !pProxies )
+		return false;
+
+	CDmrElementArray<> array( pProxies );
+
+	// Proxies are a list of sub-keys, the name is the proxy name, subkeys are values
+	for ( KeyValues *pProxy = pKeyValues->GetFirstTrueSubKey(); pProxy != NULL; pProxy = pProxy->GetNextTrueSubKey() )
+	{
+		CDmElement *pProxyElement = CreateDmElement( "DmElement", pProxy->GetName(), NULL ); 
+		array.AddToTail( pProxyElement );
+		pProxyElement->SetValue( "proxyType", pKeyValues->GetName() );
+		pProxyElement->SetValue( "editorType", "vmtProxy" );
+
+		// Normal keys are proxy parameters
+		for ( KeyValues *pProxyParam = pProxy->GetFirstValue(); pProxyParam != NULL; pProxyParam = pProxyParam->GetNextValue() )
+		{
+			switch( pProxyParam->GetDataType() )
+			{
+			case KeyValues::TYPE_INT:
+				pProxyElement->SetValue( pProxyParam->GetName(), pProxyParam->GetInt() );
+				return true;
+
+			case KeyValues::TYPE_FLOAT:
+				pProxyElement->SetValue( pProxyParam->GetName(), pProxyParam->GetFloat() );
+				return true;
+
+			case KeyValues::TYPE_STRING:
+				pProxyElement->SetValue( pProxyParam->GetName(), pProxyParam->GetString() );
+				return true;
+
+			default:
+				Warning( "Unhandled proxy keyvalues type (proxy %s var %s)\n", pProxy->GetName(), pProxyParam->GetName() );
+				return false;
+			}
+		}
+	}
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// Unserialize fallbacks
+//-----------------------------------------------------------------------------
+bool CImportVMT::UnserializeFallbacks( CDmElement *pElement, KeyValues *pFallbackKeyValues )
+{
+	// Create a child element array to contain all material proxies
+	CDmAttribute *pFallbacks = pElement->AddAttribute( "fallbacks", AT_ELEMENT_ARRAY );
+	if ( !pFallbacks )
+		return false;
+
+	CDmrElementArray<> array( pFallbacks );
+
+	CDmElement *pFallback = CreateDmElement( "DmElement", pFallbackKeyValues->GetName(), NULL ); 
+	array.AddToTail( pFallback );
+	pFallback->SetValue( "editorType", "vmtFallback" );
+
+	// Normal keys are shader parameters
+	for ( KeyValues *pShaderParam = pFallbackKeyValues->GetFirstValue(); pShaderParam != NULL; pShaderParam = pShaderParam->GetNextValue() )
+	{
+		if ( !UnserializeShaderParam( pFallback, pShaderParam ) )
+		{
+			Warning( "Error importing vmt shader parameter %s\n", pShaderParam->GetName() );
+			return NULL;
+		}
+	}
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// VMT parser
+//-----------------------------------------------------------------------------
+void InsertKeyValues( KeyValues& dst, KeyValues& src, bool bCheckForExistence )
+{
+	KeyValues *pSrcVar = src.GetFirstSubKey();
+	while( pSrcVar )
+	{
+		if ( !bCheckForExistence || dst.FindKey( pSrcVar->GetName() ) )
+		{
+			switch( pSrcVar->GetDataType() )
+			{
+			case KeyValues::TYPE_STRING:
+				dst.SetString( pSrcVar->GetName(), pSrcVar->GetString() );
+				break;
+			case KeyValues::TYPE_INT:
+				dst.SetInt( pSrcVar->GetName(), pSrcVar->GetInt() );
+				break;
+			case KeyValues::TYPE_FLOAT:
+				dst.SetFloat( pSrcVar->GetName(), pSrcVar->GetFloat() );
+				break;
+			case KeyValues::TYPE_PTR:
+				dst.SetPtr( pSrcVar->GetName(), pSrcVar->GetPtr() );
+				break;
+			}
+		}
+		pSrcVar = pSrcVar->GetNextKey();
+	}
+
+	if( bCheckForExistence )
+	{
+		for( KeyValues *pScan = dst.GetFirstTrueSubKey(); pScan; pScan = pScan->GetNextTrueSubKey() )
+		{
+			KeyValues *pTmp = src.FindKey( pScan->GetName() );
+			if( !pTmp )
+				continue;
+			// make sure that this is a subkey.
+			if( pTmp->GetDataType() != KeyValues::TYPE_NONE )
+				continue;
+			InsertKeyValues( *pScan, *pTmp, bCheckForExistence );
+		}
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Handle patch files
+//-----------------------------------------------------------------------------
+void CImportVMT::ExpandPatchFile( KeyValues *pKeyValues )
+{
+	int count = 0;
+	while( count < 10 && stricmp( pKeyValues->GetName(), "patch" ) == 0 )
+	{
+//		WriteKeyValuesToFile( "patch.txt", keyValues );
+		const char *pIncludeFileName = pKeyValues->GetString( "include" );
+		if( pIncludeFileName )
+		{
+			KeyValues * includeKeyValues = new KeyValues( "vmt" );
+			bool success = includeKeyValues->LoadFromFile( g_pFullFileSystem, pIncludeFileName, IsX360() ? "GAME" : NULL );
+			if( success )
+			{
+				KeyValues *pInsertSection = pKeyValues->FindKey( "insert" );
+				if( pInsertSection )
+				{
+					InsertKeyValues( *includeKeyValues, *pInsertSection, false );
+				}
+				
+				KeyValues *pReplaceSection = pKeyValues->FindKey( "replace" );
+				if( pReplaceSection )
+				{
+					InsertKeyValues( *includeKeyValues, *pReplaceSection, true );
+				}
+
+				*pKeyValues = *includeKeyValues;
+				includeKeyValues->deleteThis();
+				// Could add other commands here, like "delete", "rename", etc.
+			}
+			else
+			{
+				includeKeyValues->deleteThis();
+				return;
+			}
+		}
+		else
+		{
+			return;
+		}
+		count++;
+	}
+	if( count >= 10 )
+	{
+		Warning( "Infinite recursion in patch file?\n" );
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// Main entry point for the unserialization
+//-----------------------------------------------------------------------------
+CDmElement* CImportVMT::UnserializeFromKeyValues( KeyValues *pKeyValues )
+{
+	ExpandPatchFile( pKeyValues );
+
+	// Create the main element
+	CDmElement *pRoot = CreateDmElement( "DmElement", "VMT", NULL );
+	if ( !pRoot )
+		return NULL;
+
+	// Each material needs to have an editortype associated with it so it displays nicely in editors
+	pRoot->SetValue( "editorType", "vmt" );
+
+	// Each material needs a proxy list and a fallback list
+	if ( !pRoot->AddAttribute( "proxies", AT_ELEMENT_ARRAY ) )
+		return NULL;
+	if ( !pRoot->AddAttribute( "fallbacks", AT_ELEMENT_ARRAY ) )
+		return NULL;
+
+	// The keyvalues name is the shader name
+	pRoot->SetValue( "shader", pKeyValues->GetName() );
+
+	// Normal keys are shader parameters
+	for ( KeyValues *pShaderParam = pKeyValues->GetFirstValue(); pShaderParam != NULL; pShaderParam = pShaderParam->GetNextValue() )
+	{
+		if ( !UnserializeShaderParam( pRoot, pShaderParam ) )
+		{
+			Warning( "Error importing vmt shader parameter %s\n", pShaderParam->GetName() );
+			return NULL;
+		}
+	}
+
+	// Subkeys are either proxies or fallbacks
+	for ( KeyValues *pSubKey = pKeyValues->GetFirstTrueSubKey(); pSubKey != NULL; pSubKey = pSubKey->GetNextTrueSubKey() )
+	{
+		if ( !Q_stricmp( pSubKey->GetName(), "Proxies" ) )
+		{
+			UnserializeProxies( pRoot, pSubKey );
+		}
+		else
+		{
+			UnserializeFallbacks( pRoot, pSubKey );
+		}
+	}
+
+	// Resolve all element references recursively
+	RecursivelyResolveElement( pRoot );
+
+	return pRoot;
+}

vpc_scripts/projects.vgc

@@ -12,23 +12,39 @@ $Project "appframework"
 {
 	"appframework\appframework.vpc" 
 }
+
 $Project "captioncompiler"
 {
-	"utils\captioncompiler\captioncompiler.vpc" [$WIN32]
+	"utils\captioncompiler\captioncompiler.vpc" 
 }
 
 
 $Project "client"
 {
-	"game\client\client_hl2mp.vpc"		[($WIN32||$POSIX) && $HL2MP]
-	"game\client\client_episodic.vpc"	[($WIN32||$POSIX) && $EPISODIC]
-	"game\client\client_hl2.vpc"		[($WIN32||$POSIX) && $HL2]
-	"game\client\client_sdk.vpc"		[($WIN32||$POSIX) && $SDK]
+	"game\client\client_hl2mp.vpc"		[($WIN32||$WIN64||$POSIX) && $HL2MP]
+	"game\client\client_episodic.vpc"	[($WIN32||$WIN64||$POSIX) && $EPISODIC]
+	"game\client\client_hl2.vpc"		[($WIN32||$WIN64||$POSIX) && $HL2]
+	"game\client\client_sdk.vpc"		[($WIN32||$WIN64||$POSIX) && $SDK]
+}
+
+$Project "datamodel"
+{
+	"datamodel\datamodel.vpc"	
+}
+
+$Project "datacache"
+{
+	"datacache\datacache.vpc"	
+}
+
+$Project "dmserializers"
+{
+	"dmserializers\dmserializers.vpc"		
 }
 
 $Project "fgdlib"
 {
-	"fgdlib\fgdlib.vpc" [$WIN32]
+	"fgdlib\fgdlib.vpc" 
 }
 
 $Project "game_shader_dx9"