//=====================================================================
//
// FastMemcpy.c - skywind3000@163.com, 2015
//
// feature:
// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1)
//
//=====================================================================
#ifndef __FAST_MEMCPY_H__
#define __FAST_MEMCPY_H__

#include <stddef.h>
#include <stdint.h>
#include <emmintrin.h>


//---------------------------------------------------------------------
// force inline for compilers
//---------------------------------------------------------------------
#ifndef INLINE
#ifdef __GNUC__
#if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))
#define INLINE         __inline__ __attribute__((always_inline))
#else
#define INLINE         __inline__
#endif
#elif defined(_MSC_VER)
#define INLINE __forceinline
#elif (defined(__BORLANDC__) || defined(__WATCOMC__))
#define INLINE __inline
#else
#define INLINE 
#endif
#endif



//---------------------------------------------------------------------
// fast copy for different sizes
//---------------------------------------------------------------------
static INLINE void memcpy_sse2_16( void *dst, const void *src )
{
	__m128i m0 = _mm_loadu_si128( ( ( const __m128i * )src ) + 0 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 0, m0 );
}

static INLINE void memcpy_sse2_32( void *dst, const void *src )
{
	__m128i m0 = _mm_loadu_si128( ( ( const __m128i * )src ) + 0 );
	__m128i m1 = _mm_loadu_si128( ( ( const __m128i * )src ) + 1 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 0, m0 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 1, m1 );
}

static INLINE void memcpy_sse2_64( void *dst, const void *src )
{
	__m128i m0 = _mm_loadu_si128( ( ( const __m128i * )src ) + 0 );
	__m128i m1 = _mm_loadu_si128( ( ( const __m128i * )src ) + 1 );
	__m128i m2 = _mm_loadu_si128( ( ( const __m128i * )src ) + 2 );
	__m128i m3 = _mm_loadu_si128( ( ( const __m128i * )src ) + 3 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 0, m0 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 1, m1 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 2, m2 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 3, m3 );
}

static INLINE void memcpy_sse2_128( void *dst, const void *src )
{
	__m128i m0 = _mm_loadu_si128( ( ( const __m128i * )src ) + 0 );
	__m128i m1 = _mm_loadu_si128( ( ( const __m128i * )src ) + 1 );
	__m128i m2 = _mm_loadu_si128( ( ( const __m128i * )src ) + 2 );
	__m128i m3 = _mm_loadu_si128( ( ( const __m128i * )src ) + 3 );
	__m128i m4 = _mm_loadu_si128( ( ( const __m128i * )src ) + 4 );
	__m128i m5 = _mm_loadu_si128( ( ( const __m128i * )src ) + 5 );
	__m128i m6 = _mm_loadu_si128( ( ( const __m128i * )src ) + 6 );
	__m128i m7 = _mm_loadu_si128( ( ( const __m128i * )src ) + 7 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 0, m0 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 1, m1 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 2, m2 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 3, m3 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 4, m4 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 5, m5 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 6, m6 );
	_mm_storeu_si128( ( ( __m128i * )dst ) + 7, m7 );
}


//---------------------------------------------------------------------
// tiny memory copy with jump table optimized
//---------------------------------------------------------------------
static INLINE void *memcpy_tiny( void *dst, const void *src, size_t size )
{
	unsigned char *dd = ( (unsigned char *)dst ) + size;
	const unsigned char *ss = ( (const unsigned char *)src ) + size;

	switch ( size )
	{
	case 64:
		memcpy_sse2_64( dd - 64, ss - 64 );
	case 0:
		break;

	case 65:
		memcpy_sse2_64( dd - 65, ss - 65 );
	case 1:
		dd[-1] = ss[-1];
		break;

	case 66:
		memcpy_sse2_64( dd - 66, ss - 66 );
	case 2:
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 67:
		memcpy_sse2_64( dd - 67, ss - 67 );
	case 3:
		*( (uint16_t *)( dd - 3 ) ) = *( (uint16_t *)( ss - 3 ) );
		dd[-1] = ss[-1];
		break;

	case 68:
		memcpy_sse2_64( dd - 68, ss - 68 );
	case 4:
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 69:
		memcpy_sse2_64( dd - 69, ss - 69 );
	case 5:
		*( (uint32_t *)( dd - 5 ) ) = *( (uint32_t *)( ss - 5 ) );
		dd[-1] = ss[-1];
		break;

	case 70:
		memcpy_sse2_64( dd - 70, ss - 70 );
	case 6:
		*( (uint32_t *)( dd - 6 ) ) = *( (uint32_t *)( ss - 6 ) );
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 71:
		memcpy_sse2_64( dd - 71, ss - 71 );
	case 7:
		*( (uint32_t *)( dd - 7 ) ) = *( (uint32_t *)( ss - 7 ) );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 72:
		memcpy_sse2_64( dd - 72, ss - 72 );
	case 8:
		*( (uint64_t *)( dd - 8 ) ) = *( (uint64_t *)( ss - 8 ) );
		break;

	case 73:
		memcpy_sse2_64( dd - 73, ss - 73 );
	case 9:
		*( (uint64_t *)( dd - 9 ) ) = *( (uint64_t *)( ss - 9 ) );
		dd[-1] = ss[-1];
		break;

	case 74:
		memcpy_sse2_64( dd - 74, ss - 74 );
	case 10:
		*( (uint64_t *)( dd - 10 ) ) = *( (uint64_t *)( ss - 10 ) );
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 75:
		memcpy_sse2_64( dd - 75, ss - 75 );
	case 11:
		*( (uint64_t *)( dd - 11 ) ) = *( (uint64_t *)( ss - 11 ) );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 76:
		memcpy_sse2_64( dd - 76, ss - 76 );
	case 12:
		*( (uint64_t *)( dd - 12 ) ) = *( (uint64_t *)( ss - 12 ) );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 77:
		memcpy_sse2_64( dd - 77, ss - 77 );
	case 13:
		*( (uint64_t *)( dd - 13 ) ) = *( (uint64_t *)( ss - 13 ) );
		*( (uint32_t *)( dd - 5 ) ) = *( (uint32_t *)( ss - 5 ) );
		dd[-1] = ss[-1];
		break;

	case 78:
		memcpy_sse2_64( dd - 78, ss - 78 );
	case 14:
		*( (uint64_t *)( dd - 14 ) ) = *( (uint64_t *)( ss - 14 ) );
		*( (uint64_t *)( dd - 8 ) ) = *( (uint64_t *)( ss - 8 ) );
		break;

	case 79:
		memcpy_sse2_64( dd - 79, ss - 79 );
	case 15:
		*( (uint64_t *)( dd - 15 ) ) = *( (uint64_t *)( ss - 15 ) );
		*( (uint64_t *)( dd - 8 ) ) = *( (uint64_t *)( ss - 8 ) );
		break;

	case 80:
		memcpy_sse2_64( dd - 80, ss - 80 );
	case 16:
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 81:
		memcpy_sse2_64( dd - 81, ss - 81 );
	case 17:
		memcpy_sse2_16( dd - 17, ss - 17 );
		dd[-1] = ss[-1];
		break;

	case 82:
		memcpy_sse2_64( dd - 82, ss - 82 );
	case 18:
		memcpy_sse2_16( dd - 18, ss - 18 );
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 83:
		memcpy_sse2_64( dd - 83, ss - 83 );
	case 19:
		memcpy_sse2_16( dd - 19, ss - 19 );
		*( (uint16_t *)( dd - 3 ) ) = *( (uint16_t *)( ss - 3 ) );
		dd[-1] = ss[-1];
		break;

	case 84:
		memcpy_sse2_64( dd - 84, ss - 84 );
	case 20:
		memcpy_sse2_16( dd - 20, ss - 20 );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 85:
		memcpy_sse2_64( dd - 85, ss - 85 );
	case 21:
		memcpy_sse2_16( dd - 21, ss - 21 );
		*( (uint32_t *)( dd - 5 ) ) = *( (uint32_t *)( ss - 5 ) );
		dd[-1] = ss[-1];
		break;

	case 86:
		memcpy_sse2_64( dd - 86, ss - 86 );
	case 22:
		memcpy_sse2_16( dd - 22, ss - 22 );
		*( (uint32_t *)( dd - 6 ) ) = *( (uint32_t *)( ss - 6 ) );
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 87:
		memcpy_sse2_64( dd - 87, ss - 87 );
	case 23:
		memcpy_sse2_16( dd - 23, ss - 23 );
		*( (uint32_t *)( dd - 7 ) ) = *( (uint32_t *)( ss - 7 ) );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 88:
		memcpy_sse2_64( dd - 88, ss - 88 );
	case 24:
		memcpy_sse2_16( dd - 24, ss - 24 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 89:
		memcpy_sse2_64( dd - 89, ss - 89 );
	case 25:
		memcpy_sse2_16( dd - 25, ss - 25 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 90:
		memcpy_sse2_64( dd - 90, ss - 90 );
	case 26:
		memcpy_sse2_16( dd - 26, ss - 26 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 91:
		memcpy_sse2_64( dd - 91, ss - 91 );
	case 27:
		memcpy_sse2_16( dd - 27, ss - 27 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 92:
		memcpy_sse2_64( dd - 92, ss - 92 );
	case 28:
		memcpy_sse2_16( dd - 28, ss - 28 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 93:
		memcpy_sse2_64( dd - 93, ss - 93 );
	case 29:
		memcpy_sse2_16( dd - 29, ss - 29 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 94:
		memcpy_sse2_64( dd - 94, ss - 94 );
	case 30:
		memcpy_sse2_16( dd - 30, ss - 30 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 95:
		memcpy_sse2_64( dd - 95, ss - 95 );
	case 31:
		memcpy_sse2_16( dd - 31, ss - 31 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 96:
		memcpy_sse2_64( dd - 96, ss - 96 );
	case 32:
		memcpy_sse2_32( dd - 32, ss - 32 );
		break;

	case 97:
		memcpy_sse2_64( dd - 97, ss - 97 );
	case 33:
		memcpy_sse2_32( dd - 33, ss - 33 );
		dd[-1] = ss[-1];
		break;

	case 98:
		memcpy_sse2_64( dd - 98, ss - 98 );
	case 34:
		memcpy_sse2_32( dd - 34, ss - 34 );
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 99:
		memcpy_sse2_64( dd - 99, ss - 99 );
	case 35:
		memcpy_sse2_32( dd - 35, ss - 35 );
		*( (uint16_t *)( dd - 3 ) ) = *( (uint16_t *)( ss - 3 ) );
		dd[-1] = ss[-1];
		break;

	case 100:
		memcpy_sse2_64( dd - 100, ss - 100 );
	case 36:
		memcpy_sse2_32( dd - 36, ss - 36 );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 101:
		memcpy_sse2_64( dd - 101, ss - 101 );
	case 37:
		memcpy_sse2_32( dd - 37, ss - 37 );
		*( (uint32_t *)( dd - 5 ) ) = *( (uint32_t *)( ss - 5 ) );
		dd[-1] = ss[-1];
		break;

	case 102:
		memcpy_sse2_64( dd - 102, ss - 102 );
	case 38:
		memcpy_sse2_32( dd - 38, ss - 38 );
		*( (uint32_t *)( dd - 6 ) ) = *( (uint32_t *)( ss - 6 ) );
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 103:
		memcpy_sse2_64( dd - 103, ss - 103 );
	case 39:
		memcpy_sse2_32( dd - 39, ss - 39 );
		*( (uint32_t *)( dd - 7 ) ) = *( (uint32_t *)( ss - 7 ) );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 104:
		memcpy_sse2_64( dd - 104, ss - 104 );
	case 40:
		memcpy_sse2_32( dd - 40, ss - 40 );
		*( (uint64_t *)( dd - 8 ) ) = *( (uint64_t *)( ss - 8 ) );
		break;

	case 105:
		memcpy_sse2_64( dd - 105, ss - 105 );
	case 41:
		memcpy_sse2_32( dd - 41, ss - 41 );
		*( (uint64_t *)( dd - 9 ) ) = *( (uint64_t *)( ss - 9 ) );
		dd[-1] = ss[-1];
		break;

	case 106:
		memcpy_sse2_64( dd - 106, ss - 106 );
	case 42:
		memcpy_sse2_32( dd - 42, ss - 42 );
		*( (uint64_t *)( dd - 10 ) ) = *( (uint64_t *)( ss - 10 ) );
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 107:
		memcpy_sse2_64( dd - 107, ss - 107 );
	case 43:
		memcpy_sse2_32( dd - 43, ss - 43 );
		*( (uint64_t *)( dd - 11 ) ) = *( (uint64_t *)( ss - 11 ) );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 108:
		memcpy_sse2_64( dd - 108, ss - 108 );
	case 44:
		memcpy_sse2_32( dd - 44, ss - 44 );
		*( (uint64_t *)( dd - 12 ) ) = *( (uint64_t *)( ss - 12 ) );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 109:
		memcpy_sse2_64( dd - 109, ss - 109 );
	case 45:
		memcpy_sse2_32( dd - 45, ss - 45 );
		*( (uint64_t *)( dd - 13 ) ) = *( (uint64_t *)( ss - 13 ) );
		*( (uint32_t *)( dd - 5 ) ) = *( (uint32_t *)( ss - 5 ) );
		dd[-1] = ss[-1];
		break;

	case 110:
		memcpy_sse2_64( dd - 110, ss - 110 );
	case 46:
		memcpy_sse2_32( dd - 46, ss - 46 );
		*( (uint64_t *)( dd - 14 ) ) = *( (uint64_t *)( ss - 14 ) );
		*( (uint64_t *)( dd - 8 ) ) = *( (uint64_t *)( ss - 8 ) );
		break;

	case 111:
		memcpy_sse2_64( dd - 111, ss - 111 );
	case 47:
		memcpy_sse2_32( dd - 47, ss - 47 );
		*( (uint64_t *)( dd - 15 ) ) = *( (uint64_t *)( ss - 15 ) );
		*( (uint64_t *)( dd - 8 ) ) = *( (uint64_t *)( ss - 8 ) );
		break;

	case 112:
		memcpy_sse2_64( dd - 112, ss - 112 );
	case 48:
		memcpy_sse2_32( dd - 48, ss - 48 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 113:
		memcpy_sse2_64( dd - 113, ss - 113 );
	case 49:
		memcpy_sse2_32( dd - 49, ss - 49 );
		memcpy_sse2_16( dd - 17, ss - 17 );
		dd[-1] = ss[-1];
		break;

	case 114:
		memcpy_sse2_64( dd - 114, ss - 114 );
	case 50:
		memcpy_sse2_32( dd - 50, ss - 50 );
		memcpy_sse2_16( dd - 18, ss - 18 );
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 115:
		memcpy_sse2_64( dd - 115, ss - 115 );
	case 51:
		memcpy_sse2_32( dd - 51, ss - 51 );
		memcpy_sse2_16( dd - 19, ss - 19 );
		*( (uint16_t *)( dd - 3 ) ) = *( (uint16_t *)( ss - 3 ) );
		dd[-1] = ss[-1];
		break;

	case 116:
		memcpy_sse2_64( dd - 116, ss - 116 );
	case 52:
		memcpy_sse2_32( dd - 52, ss - 52 );
		memcpy_sse2_16( dd - 20, ss - 20 );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 117:
		memcpy_sse2_64( dd - 117, ss - 117 );
	case 53:
		memcpy_sse2_32( dd - 53, ss - 53 );
		memcpy_sse2_16( dd - 21, ss - 21 );
		*( (uint32_t *)( dd - 5 ) ) = *( (uint32_t *)( ss - 5 ) );
		dd[-1] = ss[-1];
		break;

	case 118:
		memcpy_sse2_64( dd - 118, ss - 118 );
	case 54:
		memcpy_sse2_32( dd - 54, ss - 54 );
		memcpy_sse2_16( dd - 22, ss - 22 );
		*( (uint32_t *)( dd - 6 ) ) = *( (uint32_t *)( ss - 6 ) );
		*( (uint16_t *)( dd - 2 ) ) = *( (uint16_t *)( ss - 2 ) );
		break;

	case 119:
		memcpy_sse2_64( dd - 119, ss - 119 );
	case 55:
		memcpy_sse2_32( dd - 55, ss - 55 );
		memcpy_sse2_16( dd - 23, ss - 23 );
		*( (uint32_t *)( dd - 7 ) ) = *( (uint32_t *)( ss - 7 ) );
		*( (uint32_t *)( dd - 4 ) ) = *( (uint32_t *)( ss - 4 ) );
		break;

	case 120:
		memcpy_sse2_64( dd - 120, ss - 120 );
	case 56:
		memcpy_sse2_32( dd - 56, ss - 56 );
		memcpy_sse2_16( dd - 24, ss - 24 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 121:
		memcpy_sse2_64( dd - 121, ss - 121 );
	case 57:
		memcpy_sse2_32( dd - 57, ss - 57 );
		memcpy_sse2_16( dd - 25, ss - 25 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 122:
		memcpy_sse2_64( dd - 122, ss - 122 );
	case 58:
		memcpy_sse2_32( dd - 58, ss - 58 );
		memcpy_sse2_16( dd - 26, ss - 26 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 123:
		memcpy_sse2_64( dd - 123, ss - 123 );
	case 59:
		memcpy_sse2_32( dd - 59, ss - 59 );
		memcpy_sse2_16( dd - 27, ss - 27 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 124:
		memcpy_sse2_64( dd - 124, ss - 124 );
	case 60:
		memcpy_sse2_32( dd - 60, ss - 60 );
		memcpy_sse2_16( dd - 28, ss - 28 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 125:
		memcpy_sse2_64( dd - 125, ss - 125 );
	case 61:
		memcpy_sse2_32( dd - 61, ss - 61 );
		memcpy_sse2_16( dd - 29, ss - 29 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 126:
		memcpy_sse2_64( dd - 126, ss - 126 );
	case 62:
		memcpy_sse2_32( dd - 62, ss - 62 );
		memcpy_sse2_16( dd - 30, ss - 30 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 127:
		memcpy_sse2_64( dd - 127, ss - 127 );
	case 63:
		memcpy_sse2_32( dd - 63, ss - 63 );
		memcpy_sse2_16( dd - 31, ss - 31 );
		memcpy_sse2_16( dd - 16, ss - 16 );
		break;

	case 128:
		memcpy_sse2_128( dd - 128, ss - 128 );
		break;
	}

	return dst;
}


//---------------------------------------------------------------------
// main routine
//---------------------------------------------------------------------
static void *memcpy_fast( void *destination, const void *source, size_t size )
{
	unsigned char *dst = (unsigned char *)destination;
	const unsigned char *src = (const unsigned char *)source;
	static size_t cachesize = 0x200000; // L2-cache size
	size_t padding;

	// small memory copy
	if ( size <= 128 )
	{
		return memcpy_tiny( dst, src, size );
	}

	// align destination to 16 bytes boundary
	padding = ( 16 - ( ( (size_t)dst ) & 15 ) ) & 15;

	if ( padding > 0 )
	{
		__m128i head = _mm_loadu_si128( ( const __m128i * )src );
		_mm_storeu_si128( ( __m128i * )dst, head );
		dst += padding;
		src += padding;
		size -= padding;
	}

	// medium size copy
	if ( size <= cachesize )
	{
		__m128i c0, c1, c2, c3, c4, c5, c6, c7;

		for ( ; size >= 128; size -= 128 )
		{
			c0 = _mm_loadu_si128( ( ( const __m128i * )src ) + 0 );
			c1 = _mm_loadu_si128( ( ( const __m128i * )src ) + 1 );
			c2 = _mm_loadu_si128( ( ( const __m128i * )src ) + 2 );
			c3 = _mm_loadu_si128( ( ( const __m128i * )src ) + 3 );
			c4 = _mm_loadu_si128( ( ( const __m128i * )src ) + 4 );
			c5 = _mm_loadu_si128( ( ( const __m128i * )src ) + 5 );
			c6 = _mm_loadu_si128( ( ( const __m128i * )src ) + 6 );
			c7 = _mm_loadu_si128( ( ( const __m128i * )src ) + 7 );
			_mm_prefetch( (const char *)( src + 256 ), _MM_HINT_NTA );
			src += 128;
			_mm_store_si128( ( ( ( __m128i * )dst ) + 0 ), c0 );
			_mm_store_si128( ( ( ( __m128i * )dst ) + 1 ), c1 );
			_mm_store_si128( ( ( ( __m128i * )dst ) + 2 ), c2 );
			_mm_store_si128( ( ( ( __m128i * )dst ) + 3 ), c3 );
			_mm_store_si128( ( ( ( __m128i * )dst ) + 4 ), c4 );
			_mm_store_si128( ( ( ( __m128i * )dst ) + 5 ), c5 );
			_mm_store_si128( ( ( ( __m128i * )dst ) + 6 ), c6 );
			_mm_store_si128( ( ( ( __m128i * )dst ) + 7 ), c7 );
			dst += 128;
		}
	}
	else
	{		// big memory copy
		__m128i c0, c1, c2, c3, c4, c5, c6, c7;

		_mm_prefetch( (const char *)( src ), _MM_HINT_NTA );

		if ( ( ( (size_t)src ) & 15 ) == 0 )
		{	// source aligned
			for ( ; size >= 128; size -= 128 )
			{
				c0 = _mm_load_si128( ( ( const __m128i * )src ) + 0 );
				c1 = _mm_load_si128( ( ( const __m128i * )src ) + 1 );
				c2 = _mm_load_si128( ( ( const __m128i * )src ) + 2 );
				c3 = _mm_load_si128( ( ( const __m128i * )src ) + 3 );
				c4 = _mm_load_si128( ( ( const __m128i * )src ) + 4 );
				c5 = _mm_load_si128( ( ( const __m128i * )src ) + 5 );
				c6 = _mm_load_si128( ( ( const __m128i * )src ) + 6 );
				c7 = _mm_load_si128( ( ( const __m128i * )src ) + 7 );
				_mm_prefetch( (const char *)( src + 256 ), _MM_HINT_NTA );
				src += 128;
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 0 ), c0 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 1 ), c1 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 2 ), c2 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 3 ), c3 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 4 ), c4 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 5 ), c5 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 6 ), c6 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 7 ), c7 );
				dst += 128;
			}
		}
		else
		{							// source unaligned
			for ( ; size >= 128; size -= 128 )
			{
				c0 = _mm_loadu_si128( ( ( const __m128i * )src ) + 0 );
				c1 = _mm_loadu_si128( ( ( const __m128i * )src ) + 1 );
				c2 = _mm_loadu_si128( ( ( const __m128i * )src ) + 2 );
				c3 = _mm_loadu_si128( ( ( const __m128i * )src ) + 3 );
				c4 = _mm_loadu_si128( ( ( const __m128i * )src ) + 4 );
				c5 = _mm_loadu_si128( ( ( const __m128i * )src ) + 5 );
				c6 = _mm_loadu_si128( ( ( const __m128i * )src ) + 6 );
				c7 = _mm_loadu_si128( ( ( const __m128i * )src ) + 7 );
				_mm_prefetch( (const char *)( src + 256 ), _MM_HINT_NTA );
				src += 128;
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 0 ), c0 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 1 ), c1 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 2 ), c2 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 3 ), c3 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 4 ), c4 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 5 ), c5 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 6 ), c6 );
				_mm_stream_si128( ( ( ( __m128i * )dst ) + 7 ), c7 );
				dst += 128;
			}
		}
		_mm_sfence();
	}

	memcpy_tiny( dst, src, size );

	return destination;
}


#endif