/*---------------------------------------------------------------------------* Project: Compress/uncompress library File: CXUncompression.h Programmer: Makoto Takano Copyright 2005 Nintendo. All rights reserved. These coded instructions, statements, and computer programs contain proprietary information of Nintendo of America Inc. and/or Nintendo Company Ltd., and are protected by Federal copyright law. They may not be disclosed to third parties or copied or duplicated in any form, in whole or in part, without the prior written consent of Nintendo. *---------------------------------------------------------------------------*/ #include #include #include #include "CXUtil.h" //====================================================================== // Expanding compressed data //====================================================================== /*---------------------------------------------------------------------------* Name: CXGetUncompressedSize Description: Gets the data size after decompression. This function can be used for data in all compression formats handled by CX. Arguments: srcp : Starting address of the compressed data Returns: Data size after decompression *---------------------------------------------------------------------------*/ u32 CXGetUncompressedSize( const void *srcp ) { u32 size = CXiConvertEndian_( *(u32*)srcp ) >> 8; if ( size == 0 ) { size = CXiConvertEndian_( *((u32*)srcp + 1) ); } return size; } /*---------------------------------------------------------------------------* Name: CXUncompressAny Description: Determines the compression type from the data header and executes the proper decompression process. Decompression processes for all compression types are linked so it might be better to execute functions for each compression type if not using compression formats other than specified format. Arguments: srcp Source address destp Destination address Returns: None. *---------------------------------------------------------------------------*/ void CXUncompressAny( const void* srcp, void* destp ) { switch ( CXGetCompressionType( srcp ) ) { // Run-length compressed data case CX_COMPRESSION_RL: CXUncompressRL( srcp, destp ); break; // LZ77-compressed data case CX_COMPRESSION_LZ: CXUncompressLZ( srcp, destp ); break; // Huffman-compressed data case CX_COMPRESSION_HUFFMAN: CXUncompressHuffman( srcp, destp ); break; // Difference filter case CX_COMPRESSION_DIFF: CXUnfilterDiff( srcp, destp ); break; default: ASSERTMSG( 0, "Unknown compressed format" ); } } /*---------------------------------------------------------------------------* Name: CXUncompressRL Description: 8-bit decompression of run-length compressed data - Decompresses run-length compressed data, writing in 8 bit units. - Use 4 byte alignment for the source address. - Data header u32 :4 Reserved compType:4 Compression type( = 3) destSize:24 Data size after decompression - Flag data format u8 length:7 Decompressed data length - 1 (When not compressed) Decompressed data length - 3 (only compress when the contiguous length is 3 bytes or greater) flag:1 (0, 1) = (not compressed, compressed) Arguments: *srcp Source address *destp Destination address Returns: None. *---------------------------------------------------------------------------*/ void CXUncompressRL( const void *srcp, void *destp ) { const u8 *pSrc = srcp; u8 *pDst = destp; u32 destCount = CXiConvertEndian_( *(u32*)pSrc ) >> 8; pSrc += 4; if ( destCount == 0 ) { destCount = CXiConvertEndian_( *(u32*)pSrc ); pSrc += 4; } while ( destCount > 0 ) { u8 flags = *pSrc++; u32 length = flags & 0x7fU; if ( !(flags & 0x80) ) { length++; if ( length > destCount ) // Measures for buffer overrun when invalid data is decompressed. { length = destCount; } destCount -= length; do { *pDst++ = *pSrc++; } while ( --length > 0 ); } else { u8 srcTmp; length += 3; if ( length > destCount ) // Measures for buffer overrun when invalid data is decompressed. { length = destCount; } destCount -= length; srcTmp = *pSrc++; do { *pDst++ = srcTmp; } while ( --length > 0 ); } } } /*---------------------------------------------------------------------------* Name: CXUncompressLZ Description: 8-bit decompression of LZ77 compressed data * Expands LZ77-compressed data and writes it in 8-bit units. - Use 4 byte alignment for the source address. - Data header u32 :4 Reserved compType:4 Compression type( = 1) destSize:24 Data size after decompression - Flag data format u8 flags Compression/no compression flag (0, 1) = (not compressed, compressed) - Code data format (Big Endian) u16 length:4 Decompressed data length - 3 (only compress when the match length is 3 bytes or greater) offset:12 Match data offset - 1 Arguments: *srcp Source address *destp Destination address Returns: None. *---------------------------------------------------------------------------*/ void CXUncompressLZ( const void *srcp, void *destp ) { const u8* pSrc = srcp; u8* pDst = destp; u32 destCount = CXiConvertEndian_( *(u32 *)pSrc ) >> 8; BOOL exFormat = (*pSrc & 0x0F)? TRUE : FALSE; pSrc += 4; if ( destCount == 0 ) { destCount = CXiConvertEndian_( *(u32*)pSrc ); pSrc += 4; } while ( destCount > 0 ) { u32 i; u32 flags = *pSrc++; for ( i = 0; i < 8; ++i ) { if ( !(flags & 0x80) ) { *pDst++ = *pSrc++; destCount--; } else { s32 length = (*pSrc >> 4); s32 offset; if ( ! exFormat ) { length += 3; } else { // LZ77 extended format if ( length == 1 ) { length = (*pSrc++ & 0x0F) << 12; length |= (*pSrc++) << 4; length |= (*pSrc >> 4); length += 0xFF + 0xF + 3; } else if ( length == 0 ) { length = (*pSrc++ & 0x0F) << 4; length |= (*pSrc >> 4); length += 0xF + 2; } else { length += 1; } } offset = (*pSrc++ & 0x0f) << 8; offset = (offset | *pSrc++) + 1; if ( length > destCount ) // Measures for buffer overrun when invalid data is decompressed. { length = (s32)destCount; } destCount -= length; do { *pDst++ = pDst[ -offset ]; } while ( --length > 0 ); } if ( destCount <= 0 ) { break; } flags <<= 1; } } } /*---------------------------------------------------------------------------* Name: CXUncompressHuffman Description: Decompression of Huffman compressed data - Decompresses Huffman compressed data, writing in 32 bit units. - Use 4 byte alignment for the source address. - Use 4-byte alignment for the destination address. - The target decompression buffer size must be prepared in 4-byte multiples. - Data header u32 bitSize:4 1 data bit size (Normally 4|8) compType:4 Compression type( = 2) destSize:24 Data size after decompression - Tree table u8 treeSize Tree table size/2 - 1 TreeNodeData nodeRoot Root node TreeNodeData nodeLeft Root left node TreeNodeData nodeRight Root right node TreeNodeData nodeLeftLeft Left left node TreeNodeData nodeLeftRight Left right node TreeNodeData nodeRightLeft Right left node TreeNodeData nodeRightRight Right right node E E The compressed data itself follows - TreeNodeData structure u8 nodeNextOffset:6 : Offset to the next node data - 1 (2 byte units) rightEndFlag:1 Right node end flag leftEndzflag:1 Left node end flag When the end flag is set There is data in next node Arguments: *srcp Source address *destp Destination address Returns: None. *---------------------------------------------------------------------------*/ void CXUncompressHuffman( const void *srcp, void *destp ) { #define TREE_END 0x80 const u32 *pSrc = srcp; u32 *pDst = destp; s32 destCount = (s32)( CXiConvertEndian_( *pSrc ) >> 8 ); u8 *treep = (destCount != 0)? ((u8*)pSrc + 4) : ((u8*)pSrc + 8); u8 *treeStartp = treep + 1; u32 dataBit = *(u8*)pSrc & 0x0FU; u32 destTmp = 0; u32 destTmpCount = 0; u32 destTmpDataNum = 4 + ( dataBit & 0x7 ); if ( destCount == 0 ) { destCount = (s32)( CXiConvertEndian_( *(pSrc + 1) ) ); } pSrc = (u32*)( treep + ((*treep + 1) << 1) ); treep = treeStartp; while ( destCount > 0 ) { s32 srcCount = 32; u32 srcTmp = CXiConvertEndian_( *pSrc++ ); // Endian strategy while ( --srcCount >= 0 ) { u32 treeShift = (srcTmp >> 31) & 0x1; u32 treeCheck = *treep; treeCheck <<= treeShift; treep = (u8*)( (((u32)treep >> 1) << 1) + (((*treep & 0x3f) + 1) << 1) + treeShift ); if ( treeCheck & TREE_END ) { destTmp >>= dataBit; destTmp |= *treep << (32 - dataBit); treep = treeStartp; if ( ++destTmpCount == destTmpDataNum ) { // Over-access until the last 4-byte alignment of the decompression buffer *pDst++ = CXiConvertEndian_(destTmp); // Endian strategy destCount -= 4; destTmpCount = 0; } } if ( destCount <= 0 ) { break; } srcTmp <<= 1; } } } /*---------------------------------------------------------------------------* Name: CXiHuffImportTree Description: Expands the Huffman table. Arguments: pTable srcp bitSize Returns: *---------------------------------------------------------------------------*/ static u32 CXiHuffImportTree( u16* pTable, const u8* srcp, u8 bitSize ) { u32 tableSize; u32 idx = 1; u32 data = 0; u32 bitNum = 0; u32 bitMask = (1 << bitSize) - 1U; u32 srcCnt = 0; const u32 MAX_IDX = (u32)( (1 << bitSize) * 2 ); if ( bitSize > 8 ) { tableSize = CXiConvertEndian16_( *(u16*)srcp ); srcp += 2; srcCnt += 2; } else { tableSize = *srcp; srcp += 1; srcCnt += 1; } tableSize = (tableSize + 1) * 4; while ( srcCnt < tableSize ) { while ( bitNum < bitSize ) { data <<= 8; data |= *srcp++; ++srcCnt; bitNum += 8; } if ( idx < MAX_IDX ) { pTable[ idx++ ] = (u16)( ( data >> (bitNum - bitSize) ) & bitMask ); } bitNum -= bitSize; } return tableSize; } typedef struct { const u8* srcp; u32 cnt; u32 stream; u32 stream_len; } BitReader; static inline void BitReader_Init( BitReader* context, const u8* srcp ) { context->srcp = srcp; context->cnt = 0; context->stream = 0; context->stream_len = 0; } static u8 BitReader_Read( BitReader* context ) { u8 bit; if ( context->stream_len == 0 ) { context->stream = context->srcp[context->cnt++]; context->stream_len = 8; } bit = (u8)( (context->stream >> (context->stream_len - 1)) & 0x1 ); context->stream_len--; return bit; } #define ENC_OFFSET_WIDTH /*---------------------------------------------------------------------------* Name: CXUncompressLH Description: Decompress LZ-Huffman (combined) compression. Arguments: *srcp Source address *destp Destination address *work Work buffer The required size is CX_UNCOMPRESS_LH_WORK_SIZE (2176B) Returns: None. *---------------------------------------------------------------------------*/ void CXUncompressLH( const u8* srcp, u8* dstp, void* work ) { #define LENGTH_BITS 9 #if defined(ENC_OFFSET_WIDTH) #define OFFSET_BITS 5 #define OFFSET_MASK 0x07 #define LEAF_FLAG 0x10 u16 offset_bit; #else #define OFFSET_BITS 12 #define OFFSET_MASK 0x3FF #define LEAF_FLAG 0x800 #endif u32 dstSize; u32 dstCnt = 0; const u8 *pSrc = srcp; BitReader stream; u16* huffTable9; u16* huffTable12; huffTable9 = work; huffTable12 = (u16*)work + (1 << LENGTH_BITS) * 2; // load the header dstSize = CXiConvertEndian_( *(u32*)pSrc ) >> 8; pSrc += 4; if ( dstSize == 0 ) { dstSize = CXiConvertEndian_( *(u32*)pSrc ); pSrc += 4; } // read the Huffman table pSrc += CXiHuffImportTree( huffTable9, pSrc, LENGTH_BITS ); pSrc += CXiHuffImportTree( huffTable12, pSrc, OFFSET_BITS ); BitReader_Init( &stream, pSrc ); while ( dstCnt < dstSize ) { u16* nodep = huffTable9 + 1; u16 val; do { u8 bit = BitReader_Read( &stream ); u32 offset = (((*nodep & 0x7F) + 1U) << 1) + bit; if ( *nodep & (0x100 >> bit) ) { nodep = (u16*)((u32)nodep & ~0x3); val = *(nodep + offset); break; } else { nodep = (u16*)((u32)nodep & ~0x3); nodep += offset; } } while ( 1 ); if ( val < 0x100 ) // uncompressed data { dstp[dstCnt++] = (u8)val; } else // compressed data { u16 length = (u16)( (val & 0xFF) + 3 ); u16* nodep = huffTable12 + 1; do { u8 bit = BitReader_Read( &stream ); u32 offset = (((*nodep & OFFSET_MASK) + 1U) << 1) + bit; if ( *nodep & (LEAF_FLAG >> bit) ) { nodep = (u16*)((u32)nodep & ~0x3); val = *(nodep + offset); break; } else { nodep = (u16*)((u32)nodep & ~0x3); nodep += offset; } } while ( 1 ); #if defined(ENC_OFFSET_WIDTH) offset_bit = val; val = 0; if ( offset_bit > 0 ) { val = 1; while ( --offset_bit > 0 ) { val <<= 1; val |= BitReader_Read( &stream ); } } #endif val += 1; // Measures for buffer overrun when invalid data is decompressed. if ( dstCnt + length > dstSize ) { length = (u16)( dstSize - dstCnt ); } while ( length-- > 0 ) { dstp[dstCnt] = dstp[dstCnt - val]; ++dstCnt; } } } #undef LENGTH_BITS #undef OFFSET_BITS #undef OFFSET_MASK #undef LEAF_FLAG } // Structure for the range coder state typedef struct { u32 low; u32 range; u32 code; // only used during decompression u8 carry; // only used during compression u32 carry_cnt; // only used during compression } RCState; // Range coder structure typedef struct { u32 *freq; // Table for occurrence frequency: (1 << bitSize) * sizeof(u32) bytes u32 *low_cnt; // Table for the LOW border value: (1 << bitSize) * sizeof(u32) bytes u32 total; // Total: 4 bytes u8 bitSize; // Bit size: 1 byte u8 padding_[1]; // } RCCompressionInfo; #define RC_MAX_RANGE 0x80000000 /*---------------------------------------------------------------------------* Name: RCInitState_ Description: Initializes the RC state. Arguments: state: Pointer to an RC state structure Returns: None. *---------------------------------------------------------------------------*/ static inline void RCInitState_( RCState* state ) { state->low = 0; state->range = RC_MAX_RANGE; state->code = 0; state->carry = 0; state->carry_cnt = 0; } /*---------------------------------------------------------------------------* Name: RCInitInfo_ Description: Initialize the table for the adaptive range coder. All occurrence frequencies are initialized to 1. Arguments: info: Pointer to an RC table information structure Returns: None. *---------------------------------------------------------------------------*/ static inline void RCInitInfo_( RCCompressionInfo* info, u8 bitSize, void* work ) { u32 tableSize = (u32)(1 << bitSize); u32 i; info->bitSize = bitSize; info->freq = (u32*)work; info->low_cnt = (u32*)( (u32)work + tableSize * sizeof(u32) ); for ( i = 0; i < tableSize; i++ ) { info->freq[ i ] = 1; info->low_cnt[ i ] = i; } info->total = tableSize; } /*---------------------------------------------------------------------------* Name: RCAAddCount_ Description: Updates the frequency table for an adaptive range coder. Arguments: info: Table information for a range coder val: Value to add Returns: None. *---------------------------------------------------------------------------*/ static void RCAddCount_( RCCompressionInfo* info, u16 val ) { u32 i; u32 tableSize = (u32)(1 << info->bitSize); info->freq[ val ]++; info->total++; for ( i = (u32)(val + 1); i < tableSize; i++ ) { info->low_cnt[ i ]++; } // Reconstruct if the total exceeds the maximum value. if ( info->total >= 0x00010000 ) { if ( info->freq[ 0 ] > 1 ) { info->freq[ 0 ] = info->freq[ 0 ] / 2; } info->low_cnt[ 0 ] = 0; info->total = info->freq[ 0 ]; for ( i = 1; i < tableSize; i++ ) { if ( info->freq[ i ] > 1 ) { info->freq[ i ] >>= 1; } info->low_cnt[ i ] = info->low_cnt[ i - 1 ] + info->freq[ i - 1 ]; info->total += info->freq[ i ]; } } } /*---------------------------------------------------------------------------* Name: RCSearch_ Description: Searches for the value that must be obtained next from the code, range, and low values. Arguments: info: Table information for a range coder code: Current code value range: Current Range value low: Current Low value Returns: *---------------------------------------------------------------------------*/ static u16 RCSearch_( RCCompressionInfo* info, u32 code, u32 range, u32 low ) { u32 tableSize = (u32)(1 << info->bitSize); u32 codeVal = code - low; u32 i; u32 temp = range / info->total; u32 tempVal = codeVal / temp; // Binary search u32 left = 0; u32 right = tableSize - 1; while ( left < right ) { i = (left + right) / 2; if ( info->low_cnt[ i ] > tempVal ) { right = i; } else { left = i + 1; } } i = left; while ( info->low_cnt[ i ] > tempVal ) { --i; } return (u16)i; } /*---------------------------------------------------------------------------* Name: RCGetData_ Description: Gets the next value from the state in RCState, then updates the state. Arguments: stream info state adaptive Returns: *---------------------------------------------------------------------------*/ static u16 RCGetData_( const u8* srcp, RCCompressionInfo* info, RCState* state, u32* pSrcCnt ) { #define MIN_RANGE 0x01000000 u16 val = RCSearch_( info, state->code, state->range, state->low ); u32 cnt = 0; { u32 tmp; tmp = state->range / info->total; state->low += info->low_cnt[ val ] * tmp; state->range = info->freq[ val ] * tmp; } // Update the table for occurrence frequency RCAddCount_( info, val ); while ( state->range < MIN_RANGE ) { state->code <<= 8; state->code += srcp[ cnt++ ]; state->range <<= 8; state->low <<= 8; } *pSrcCnt = cnt; return val; #undef MIN_RANGE } /*---------------------------------------------------------------------------* Name: CXUncompressLRC Description: LZ/Range Coder combined compression Arguments: *srcp Source address *destp Destination address *work Work buffer The required size is CX_UNCOMPRESS_LRC_WORK_SIZE (36864B) Returns: None. *---------------------------------------------------------------------------*/ void CXUncompressLRC( const u8* srcp, u8* dstp, void* work ) { #define LENGTH_BITS 9 #define OFFSET_BITS 12 RCCompressionInfo infoVal; RCCompressionInfo infoOfs; RCState rcState; const u8* pSrc = srcp; u32 dstCnt = 0; u32 dstSize = 0; RCInitInfo_( &infoVal, LENGTH_BITS, work ); RCInitInfo_( &infoOfs, OFFSET_BITS, (u8*)work + (1 << LENGTH_BITS) * sizeof(u32) * 2 ); RCInitState_( &rcState ); // Load the header dstSize = CXiConvertEndian_( *(u32*)pSrc ) >> 8; pSrc += 4; if ( dstSize == 0 ) { dstSize = CXiConvertEndian_( *(u32*)pSrc ); pSrc += 4; } // Load the initial code rcState.code = (u32)( (*pSrc << 24) | (*(pSrc + 1) << 16) | (*(pSrc + 2) << 8) | (*(pSrc + 3) ) ); pSrc += 4; // Continue to get values from the range coder and perform LZ decompression while ( dstCnt < dstSize ) { u32 cnt; u16 val = (u16)( RCGetData_( pSrc, &infoVal, &rcState, &cnt ) ); pSrc += cnt; if ( val < 0x100 ) // Uncompressed data { dstp[ dstCnt++ ] = (u8)val; } else // Compressed data { u16 length = (u16)( (val & 0xFF) + 3 ); val = (u16)( RCGetData_( pSrc, &infoOfs, &rcState, &cnt ) + 1 ); pSrc += cnt; // Check for a buffer overrun if ( dstCnt + length > dstSize ) { return; } if ( dstCnt < val ) { return; } while ( length-- > 0 ) { dstp[ dstCnt ] = dstp[ dstCnt - val ]; ++dstCnt; } } } #undef LENGTH_BITS #undef OFFSET_BITS } /*---------------------------------------------------------------------------* Name: CXUnfilterDiff Description: 8-bit decompression to restore differential filter conversion. - Restores a differential filter, writing in 8 bit units. - Use 4 byte alignment for the source address. Arguments: *srcp Source address *destp Destination address Returns: None. *---------------------------------------------------------------------------*/ void CXUnfilterDiff( register const void *srcp, register void *destp ) { const u8* pSrc = srcp; u8* pDst = destp; u32 bitSize = *pSrc & 0xFU; s32 destCount = (s32)( CXiConvertEndian_( *(u32*)pSrc ) >> 8 ); u32 sum = 0; pSrc += 4; if ( bitSize != 1 ) { // Difference calculation in units of 8 bits do { u8 tmp = *(pSrc++); destCount--; sum += tmp; *(pDst++) = (u8)sum; } while ( destCount > 0 ); } else { // Difference calculation in units of 16 bits do { u16 tmp = CXiConvertEndian16_( *(u16*)pSrc ); pSrc += 2; destCount -= 2; sum += tmp; *(u16*)pDst = CXiConvertEndian16_( (u16)sum ); pDst += 2; } while ( destCount > 0 ); } } /*---------------------------------------------------------------------------* Name: CXGetCompressionHeader Description: Gets header information from the first four bytes in the compression data. Arguments: data First 8 bytes of compressed data *---------------------------------------------------------------------------*/ CXCompressionHeader CXGetCompressionHeader( const void* data ) { CXCompressionHeader ret; ret.compType = (u8)( (*(u8*)data & 0xF0) >> 4 ); ret.compParam = (u8)( *(u8*)data & 0x0F ); ret.destSize = CXiConvertEndian_( *(u32*)data ) >> 8; if ( ret.destSize == 0 ) { ret.destSize = CXiConvertEndian_( *((u32*)data + 1) ); } return ret; }