CharsetDecoder.cpp

Go to the documentation of this file.

#if 0
#include <errno.h>
#endif
#include <dcl/Config.h>
 
#include <string.h>     // memcmp, memcpy
#include <wchar.h>      // mbrtowc
 
#include <dcl/Charset.h>
 
#if __DCL_HAVE_ALLOC_DEBUG
#undef __DCL_ALLOC_LEVEL
#define __DCL_ALLOC_LEVEL   __DCL_ALLOC_INTERNAL
#endif
 
#if __DCL_DEBUG
#undef __THIS_FILE__
static const char_t __THIS_FILE__[] = __T("dcl/CharsetDecoder.cpp");
#endif
 
__DCL_BEGIN_NAMESPACE
 
#define ILLEGAL_UCS2        -1  // decode
#define ILLEGAL_SEQUENCE    -2  // decode
#define SOURCE_FEW          -3  // decode
#define SOURCE_FEW_N(_mbslen)       (-3-(_mbslen))
 
IMPLEMENT_CLASSINFO(CharsetDecoder, Object)
 
CharsetDecoder::CharsetDecoder()
{
#if __DCL_DEBUG
    __DCL_ASSERT(sizeof(ucs4_t) == 4);
#if __SIZEOF_WCHAR_T__ == 4
    __DCL_ASSERT(sizeof(wchar_t) == 4);
#endif
#if __SIZEOF_WCHAR_T__ == 2
    __DCL_ASSERT(sizeof(wchar_t) == 2);
#endif
    ucs4_t uc = 0x31323334;
#if __BYTE_ORDER == __BIG_ENDIAN
    __DCL_ASSERT(memcmp(&uc, "1234", sizeof(uc)) == 0);
#else
    __DCL_ASSERT(memcmp(&uc, "4321", sizeof(uc)) == 0);
#endif
#endif
 
}
 
int CharsetDecoder::decode(           // MB, UTF ==> UCS
        const byte_t* _in,        // in: input bytes
        size_t&         _inCount,   // in: count of input bytes, out: count of processed bytes
        wchar_t*        _out,       // out: output buffer
        size_t&         _outCount   // in: buffer size (countof(_out[], wchar_t)), out: count of converted wchars
        )
{
    __DCL_ASSERT(_in != NULL && _out != NULL);
 
    wchar_t* dst = _out;
    wchar_t* dstend = dst + _outCount;
 
    const byte_t* src = _in;
    const byte_t* srcend = src + _inCount;
    size_t srclen;
 
    int _mbslen = 0;
    while((srclen = srcend - src) > 0 && dst < dstend) {
        ucs4_t uc;
        _mbslen = toWideChar(src, srclen, &uc);
 
        if (_mbslen <= 0)
            break;
 
#if __SIZEOF_WCHAR_T__ == 2
        if (uc > 0xFFFF || (0xD800 <= uc && uc < 0xE000)) {
            _mbslen = ILLEGAL_UCS2;
            break;
        }
#endif
        *dst = (wchar_t)uc;
 
        src += (size_t)_mbslen;
        dst++;
    }
 
    _inCount = src - _in;
    _outCount = dst - _out;
    
    if (_mbslen >= 0)
        return CS_NOERROR;
 
    switch (_mbslen) {
        case ILLEGAL_UCS2 :
            return CS_ILLEGAL_UCS2;
        case ILLEGAL_SEQUENCE :
            return CS_ILLEGAL_SEQUENCE;
    }
    return CS_SOURCE_FEW;
}
 
size_t CharsetDecoder::getDecodedLength(const char* _mbs, size_t _mbslen)
        __DCL_THROWS1(CharsetConvertException*)
{
    const byte_t* _in = (const byte_t*)_mbs;
    size_t inTotal = _mbslen;
    size_t outTotal = 0;
    wchar_t buf[6];
 
    for ( ; ; ) {
        size_t _inCount = inTotal;
        size_t _outCount = __countof(buf, wchar_t);
        int r = decode(_in, _inCount, buf, _outCount);
        outTotal += _outCount;
        if (r == CS_NOERROR) {
            inTotal -= _inCount;
            if (inTotal == 0)
                break;
            // else buffer small
        }
        else {
            throw(new CharsetConvertException(r));
        }
        _in += _inCount;        
    }
    return outTotal;
}
 
String CharsetDecoder::decode(const char* _mbs, size_t _mbslen)
        __DCL_THROWS1(CharsetConvertException*)
{
    __DCL_ASSERT(_mbs != NULL);
    if (_mbslen == (size_t) -1)
        _mbslen = ByteString::length(_mbs);
 
    String rstr;
    if (_mbslen) {
        const byte_t* _in = (const byte_t*) _mbs;
        size_t _inCount = _mbslen;
#if 0
        size_t _outCount = getDecodedLength(_mbs, _mbslen);
#else
        size_t _outCount = _mbslen;
#endif
 
        CharBuffer* buf = CharBuffer::create(_outCount);
        int rn = decode(_in, _inCount, buf->data(), _outCount);
        __DCL_ASSERT(_outCount <= buf->__allocLength);
        if (rn != CS_NOERROR) {
            buf->release();
            throw new CharsetConvertException(rn);
        }
        buf->data()[_outCount] = L'\0';
        buf->__dataLength = _outCount;
        CharBuffer::shrink(buf);
 
        rstr.assign(buf);
        buf->release();
    }
    return rstr;
}
 
String CharsetDecoder::decode(const ByteString& _str)
        __DCL_THROWS1(CharsetConvertException*)
{
    return decode(_str, _str.length());
}
 
IMPLEMENT_CLASSINFO(UTF8Decoder, CharsetDecoder)
 
UTF8Decoder::UTF8Decoder()
{
    reset();
}
 
void UTF8Decoder::reset()
{
    __hasBOM = false;
}
 
int UTF8Decoder::toWideChar(const byte_t* _mbs, size_t _mbslen, ucs4_t* _uc)
{
    int count = 0;
    while (_mbslen > 0) {
        byte_t c = _mbs[0];
 
        if (c < 0x80) {
            *_uc = c;
            return count + 1;
        }
        else if (c < 0xC2) {
            return ILLEGAL_SEQUENCE;
        }
        else if (c < 0xE0) {
            if (_mbslen < 2)
                return SOURCE_FEW_N(0);
 
            if (!((_mbs[1] ^ 0x80) < 0x40))
                return ILLEGAL_SEQUENCE;
 
            *_uc = ((ucs4_t)(c & 0x1F) << 6)
                | (ucs4_t)(_mbs[1] ^ 0x80);
            
            return count + 2;
        }
        else if (c < 0xF0) {
            if (_mbslen < 3)
                return SOURCE_FEW_N(0);
 
            if (!((_mbs[1] ^ 0x80) < 0x40 && (_mbs[2] ^ 0x80) < 0x40
                && (c >= 0xE1 || _mbs[1] >= 0xA0)))
                return ILLEGAL_SEQUENCE;
 
            ucs4_t ucTemp = ((ucs4_t) (c & 0x0F) << 12)
                | ((ucs4_t) (_mbs[1] ^ 0x80) << 6)
                | (ucs4_t) (_mbs[2] ^ 0x80);
            if (ucTemp == 0xFEFF) {
                // BOM
                __hasBOM = true;
                _mbslen -= 3;
                _mbs += 3;
            }
            else {
                *_uc = ucTemp;
                return count + 3;
            }
        }
        else if (c < 0xF8) {
            if (_mbslen < 4)
                return SOURCE_FEW_N(0);
 
            if (!((_mbs[1] ^ 0x80) < 0x40 && (_mbs[2] ^ 0x80) < 0x40
                && (_mbs[3] ^ 0x80) < 0x40 && (c >= 0xF1 || _mbs[1] >= 0x90)))
                return ILLEGAL_SEQUENCE;
 
            *_uc = ((ucs4_t) (c & 0x07) << 18)
                | ((ucs4_t) (_mbs[1] ^ 0x80) << 12)
                | ((ucs4_t) (_mbs[2] ^ 0x80) << 6)
                | (ucs4_t) (_mbs[3] ^ 0x80);
 
            return count + 4;
        }
        else if (c < 0xFC) {
            if (_mbslen < 5)
                return SOURCE_FEW_N(0);
 
            if (!((_mbs[1] ^ 0x80) < 0x40 && (_mbs[2] ^ 0x80) < 0x40
                && (_mbs[3] ^ 0x80) < 0x40 && (_mbs[4] ^ 0x80) < 0x40
                && (c >= 0xF9 || _mbs[1] >= 0x88)))
                return ILLEGAL_SEQUENCE;
 
            *_uc = ((ucs4_t) (c & 0x03) << 24)
                | ((ucs4_t) (_mbs[1] ^ 0x80) << 18)
                | ((ucs4_t) (_mbs[2] ^ 0x80) << 12)
                | ((ucs4_t) (_mbs[3] ^ 0x80) << 6)
                | (ucs4_t) (_mbs[4] ^ 0x80);
 
            return count + 5;
        }
        else if (c < 0xFE) {
            if (_mbslen < 6)
                return SOURCE_FEW_N(0);
 
            if (!((_mbs[1] ^ 0x80) < 0x40 && (_mbs[2] ^ 0x80) < 0x40
                && (_mbs[3] ^ 0x80) < 0x40 && (_mbs[4] ^ 0x80) < 0x40
                && (_mbs[5] ^ 0x80) < 0x40
                && (c >= 0xFD || _mbs[1] >= 0x84)))
                return ILLEGAL_SEQUENCE;
 
            *_uc = ((ucs4_t) (c & 0x01) << 30)
                | ((ucs4_t) (_mbs[1] ^ 0x80) << 24)
                | ((ucs4_t) (_mbs[2] ^ 0x80) << 18)
                | ((ucs4_t) (_mbs[3] ^ 0x80) << 12)
                | ((ucs4_t) (_mbs[4] ^ 0x80) << 6)
                | (ucs4_t) (_mbs[5] ^ 0x80);
 
                return count + 6;
        }
        else
            return ILLEGAL_SEQUENCE;
    }
    return SOURCE_FEW_N(0);
}
 
IMPLEMENT_CLASSINFO(UTF16Decoder, CharsetDecoder)
 
UTF16Decoder::UTF16Decoder(
        int nDefaultByteOrder // = CS_DEFAULT_ENDIAN
        )
{
    __DCL_ASSERT(nDefaultByteOrder == CS_LITTLE_ENDIAN
        || nDefaultByteOrder == CS_BIG_ENDIAN);
 
    __defaultBigEndian = nDefaultByteOrder == CS_BIG_ENDIAN;
    reset();
}
 
void UTF16Decoder::reset()
{
    __hasBOM = false;
    __bigEndian = __defaultBigEndian;
}
 
int UTF16Decoder::toWideChar(const byte_t* _mbs, size_t _mbslen, ucs4_t* _uc)
{
    int count = 0;
    while(_mbslen >= 2) {
        ucs4_t uc = __bigEndian ? (_mbs[0] << 8) + _mbs[1] : _mbs[0] + (_mbs[1] << 8);
        if (uc == 0xFEFF) {
            __hasBOM = true;
        }
        else if (uc == 0xFFFE) {
            __hasBOM = true;
            __bigEndian = !__bigEndian;
        }
        else if (uc >= 0xD800 && uc < 0xDC00) {
            if (_mbslen >= 4) {
                ucs4_t uc2 = __bigEndian ? (_mbs[2] << 8) + _mbs[3] : _mbs[2] + (_mbs[3] << 8);
                if (!(uc2 >= 0xdc00 && uc2 < 0xe000))
                    return ILLEGAL_SEQUENCE;
                *_uc = 0x10000 + ((uc - 0xd800) << 10) + (uc2 - 0xdc00);
                return count + 4;
            }
            else
                break;
        }
        else if (uc >= 0xdc00 && uc < 0xe000) {
          return ILLEGAL_SEQUENCE;
        }
        else {
            *_uc = uc;
            return count + 2;
        }
        _mbs += 2; _mbslen -= 2; count += 2;
    }
    return SOURCE_FEW_N(count);
}
 
IMPLEMENT_CLASSINFO(UTF32Decoder, CharsetDecoder)
 
UTF32Decoder::UTF32Decoder(
        int nDefaultByteOrder // = CS_DEFAULT_ENDIAN
        )
{
    __DCL_ASSERT(nDefaultByteOrder == CS_LITTLE_ENDIAN
        || nDefaultByteOrder == CS_BIG_ENDIAN);
 
    __defaultBigEndian = nDefaultByteOrder == CS_BIG_ENDIAN;
}
 
void UTF32Decoder::reset()
{
    __hasBOM = false;
    __bigEndian = __defaultBigEndian;
}
 
int UTF32Decoder::toWideChar(const byte_t* _mbs, size_t _mbslen, ucs4_t* _uc)
{
    int count = 0;
    while(_mbslen >= 4) {
        ucs4_t uc = __bigEndian ? (_mbs[0] << 24) + (_mbs[1] << 16) + (_mbs[2] << 8) + _mbs[3]
            : _mbs[0] + (_mbs[1] << 8) + (_mbs[2] << 16) + (_mbs[3] << 24);
 
        count += 4;
 
        if (uc == 0x0000FEFF) {
            __hasBOM = true;
        }
        else if (uc == 0xFFFE0000U) {
            __hasBOM = true;
            __bigEndian = !__bigEndian;
        }
        else {
            if (uc < 0x110000 && !(uc >= 0xD800 && uc < 0xE000)) {
                *_uc = uc;
                return count;
            }
            else
                return ILLEGAL_SEQUENCE;
        }
        _mbs += 4; _mbslen -= 4;
    }
    return SOURCE_FEW_N(count);
}
 
IMPLEMENT_CLASSINFO(AsciiDecoder, CharsetEncoder)
 
AsciiDecoder::AsciiDecoder()
{
}
 
int AsciiDecoder::toWideChar(const byte_t* _mbs, size_t _mbslen, ucs4_t* _uc)
{
    byte_t c = *_mbs;
    if (c < 0x80) {
        *_uc = c;
        return 1;
    }
    return ILLEGAL_SEQUENCE;
}
 
String AsciiDecoder::decode(const char* _mbs, size_t _mbslen)
{
    if (_mbslen == (size_t)-1)
        _mbslen = ByteString::length(_mbs);
 
    CharBuffer* buf = CharBuffer::create(_mbslen);
    wchar_t* p = buf->data();
    for(size_t i = 0; i < _mbslen; i++)
        p[i] = (wchar_t)(byte_t)_mbs[i];
 
    p[_mbslen] = L'\0';
    buf->__dataLength = _mbslen;
    __DCL_ASSERT(buf->data()[buf->__dataLength] == L'\0');
 
    String r = buf;
    buf->release();
    return r;
}
 
IMPLEMENT_CLASSINFO(Latin1Decoder, CharsetEncoder)
 
Latin1Decoder::Latin1Decoder()
{
}
 
int Latin1Decoder::toWideChar(const byte_t* _mbs, size_t _mbslen, ucs4_t* _uc)
{
    *_uc = (byte_t)*_mbs;
    return 1;
}
 
String Latin1Decoder::decode(const char* _mbs, size_t _mbslen)
{
    if (_mbslen == (size_t)-1)
        _mbslen = ByteString::length(_mbs);
 
    CharBuffer* buf = CharBuffer::create(_mbslen);
    wchar_t* p = buf->data();
    for (size_t i = 0; i < _mbslen; i++)
        p[i] = (wchar_t)(byte_t)_mbs[i];
 
    p[_mbslen] = L'\0';
    buf->__dataLength = _mbslen;
    __DCL_ASSERT(buf->data()[buf->__dataLength] == L'\0');
 
    String r = buf;
    buf->release();
    return r;
}
 
IMPLEMENT_CLASSINFO(LocaleDecoder, CharsetDecoder)
 
LocaleDecoder::LocaleDecoder()
{
    reset();
}
 
void LocaleDecoder::reset()
{
    memset(&__mbstate, 0, sizeof(__mbstate));
}
 
// mbsrtowcs
int LocaleDecoder::decode(          // MB ==> UCS
        const byte_t* _in,        // in: input bytes
        size_t&         _inCount,   // in: count of input bytes, out: count of processed bytes
        wchar_t*        _out,       // out: output buffer
        size_t&         _outCount   // in: buffer size (countof(_out[], wchar_t)), out: count of converted wchars
        )
{
    __DCL_ASSERT(_in != NULL && _out != NULL);
 
    wchar_t* dst = _out;
    wchar_t* dstend = dst + _outCount;
 
    const byte_t* src = _in;
    const byte_t* srcend = src + _inCount;
    size_t srclen;
    size_t _mbslen = 0;
 
    while(dst < dstend && (srclen = srcend - src) > 0) {
        _mbslen = mbrtowc(dst, (const char*) src, srclen, &__mbstate);
        if (_mbslen == 0 || _mbslen == (size_t) -1 || _mbslen == (size_t) -2)
            break;
 
        src += _mbslen;
        dst++;
    }
 
    _inCount = src - _in;
    _outCount = dst - _out;
 
    switch(_mbslen) {
        case (size_t) -2 :
            return CS_SOURCE_FEW;
        case (size_t) -1 :
            return CS_ILLEGAL_SEQUENCE;
    }
    return CS_NOERROR;
}
 
int LocaleDecoder::toWideChar(const byte_t* _mbs, size_t _mbslen, ucs4_t* _uc)
{
    __DCL_ASSERT(false);
    return 0;
}
 
__DCL_END_NAMESPACE