C++ 11의 UTF 변환 함수

Question

C++ 11에서 UTF 문자 변환을 수행하는 함수 모음을 찾고 있습니다. utf8, utf16 및 utf32와의 변환이 포함되어야합니다. 바이트 순서 표시를 인식하는 기능도 도움이 될 것입니다.

Answer 1

업데이트 : 여기에 나열된 기능을하는 GitHub의의의 repo에 .hpp, .cpp 및 tests을 유지하고 있습니다. 일부 UTF-16 함수는 올바르게 작동하지 않으므로 사용 불가능하게 설정되었습니다. utf.test.cpp 파일의 "banana"테스트는 문제를 보여줍니다.

바이트 순서 표시를 인식하는 "read_with_bom"기능도 포함되었습니다.

#if _MSC_VER == 1900 //work around for bug in MS Visual C++ 2015 https://social.msdn.microsoft.com/Forums/en-US/8f40dcd8-c67f-4eba-9134-a19b9178e481/vs-2015-rc-linker-stdcodecvt-error?forum=vcgeneral 

std::string to_utf8(const std::u16string &s) 
{ 
    std::wstring_convert<std::codecvt_utf8<int16_t>, int16_t> convert; 
    auto p = reinterpret_cast<const int16_t *>(s.data()); 
    return convert.to_bytes(p, p + s.size()); 
} 

std::string to_utf8(const std::u32string &s) 
{ 
    std::wstring_convert<std::codecvt_utf8<int32_t>, int32_t> convert; 
    auto p = reinterpret_cast<const int32_t *>(s.data()); 
    return convert.to_bytes(p, p + s.size()); 
} 

std::u16string to_utf16(const std::string &s) 
{ 
    std::wstring_convert<std::codecvt_utf8<int16_t>, int16_t> convert; 
    auto asInt = convert.from_bytes(s); 
    return std::u16string(reinterpret_cast<char16_t const *>(asInt.data()), asInt.length()); 
} 

std::u32string to_utf32(const std::string &s) 
{ 
    std::wstring_convert<std::codecvt_utf8<int32_t>, int32_t> convert; 
    auto asInt = convert.from_bytes(s); 
    return std::u32string(reinterpret_cast<char32_t const *>(asInt.data()), asInt.length()); 
} 

#else 

std::string to_utf8(const std::u16string &s) 
{ 
    std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> conv; 
    return conv.to_bytes(s); 
} 

std::string to_utf8(const std::u32string &s) 
{ 
    std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv; 
    return conv.to_bytes(s); 
} 

std::u16string to_utf16(const std::string &s) 
{ 
    std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> convert; 
    return convert.from_bytes(s); 
} 

std::u32string to_utf32(const std::string &s) 
{ 
    std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv; 
    return conv.from_bytes(s); 
} 

#endif 

std::u16string to_utf16(const std::u32string &s) 
{ 
    return to_utf16(to_utf8(s)); 
} 

std::u32string to_utf32(const std::u16string &s) { 
    return to_utf32(to_utf8(s)); 
} 

std::u32string read_with_bom(std::istream & src) 
{ 

    enum encoding { 
     encoding_utf32be = 0, 
     encoding_utf32le, 
     encoding_utf16be, 
     encoding_utf16le, 
     encoding_utf8, 
     encoding_ascii, 
    }; 

    std::vector<std::string> boms = { 
     std::string("\x00\x00\xFE\xFF", 4), 
     std::string("\xFF\xFE\x00\x00", 4), 
     std::string("\xFE\xFF", 2), 
     std::string("\xFF\xFE", 2), 
     std::string("\xEF\xBB\xBF", 3) 
    }; 

    std::string buffer((std::istreambuf_iterator<char>(src)), std::istreambuf_iterator<char>()); 

    encoding enc = encoding_ascii; 

    for (unsigned int i = 0; i < boms.size(); ++i) { 
     std::string testBom = boms[i]; 
     if (buffer.compare(0, testBom.length(), testBom) == 0) { 
      enc = encoding(i); 
      buffer = buffer.substr(testBom.length()); 
      break; 
     } 
    } 

    switch (enc) { 
    case encoding_utf32be: 
    { 
     if (buffer.length() % 4 != 0) { 
      throw std::logic_error("size in bytes must be a multiple of 4"); 
     } 
     int count = buffer.length()/4; 
     std::u32string temp = std::u32string(count, 0); 
     for (int i = 0; i < count; ++i) { 
      temp[i] = static_cast<char32_t>(buffer[i * 4 + 3] << 0 | buffer[i * 4 + 2] << 8 | buffer[i * 4 + 1] << 16 | buffer[i * 4 + 0] << 24); 
     } 
     return temp; 
    } 
    case encoding_utf32le: 
    { 
     if (buffer.length() % 4 != 0) { 
      throw std::logic_error("size in bytes must be a multiple of 4"); 
     } 
     int count = buffer.length()/4; 
     std::u32string temp = std::u32string(count, 0); 
     for (int i = 0; i < count; ++i) { 
      temp[i] = static_cast<char32_t>(buffer[i * 4 + 0] << 0 | buffer[i * 4 + 1] << 8 | buffer[i * 4 + 2] << 16 | buffer[i * 4 + 3] << 24); 
     } 
     return temp; 
    } 
    case encoding_utf16be: 
    { 
     if (buffer.length() % 2 != 0) { 
      throw std::logic_error("size in bytes must be a multiple of 2"); 
     } 
     int count = buffer.length()/2; 
     std::u16string temp = std::u16string(count, 0); 
     for (int i = 0; i < count; ++i) { 
      temp[i] = static_cast<char16_t>(buffer[i * 2 + 1] << 0 | buffer[i * 2 + 0] << 8); 
     } 
     return to_utf32(temp); 
    } 
    case encoding_utf16le: 
    { 
     if (buffer.length() % 2 != 0) { 
      throw std::logic_error("size in bytes must be a multiple of 2"); 
     } 
     int count = buffer.length()/2; 
     std::u16string temp = std::u16string(count, 0); 
     for (int i = 0; i < count; ++i) { 
      temp[i] = static_cast<char16_t>(buffer[i * 2 + 0] << 0 | buffer[i * 2 + 1] << 8); 
     } 
     return to_utf32(temp); 
    } 
    default: 
     return to_utf32(buffer); 
    } 
} 

Answer 2

나는 이것을 위해 약간의 utf_ranges 라이브러리를 작성했습니다. Range-V3 및 C++ 14를 사용합니다.

세 가지 기본 UTF 인코딩 중 하나를 변환하기위한보기 및 동작 (Range-V3 용어에 익숙한 경우)이 있으며, 바이트 순서 표시를 생성 및 생성하고 bom을 기반으로 엔디안 변환을 수행 할 수 있습니다. 예를 들어, \n에 일곱 유니 코드 라인 엔딩 중 하나를 변환하는 UTF-8 std::string에 알 수없는 엔디안 UTF-16에서 파일을 읽기는 다음과 같습니다

여기

std::ifstream source{path, std::ios::binary}; 

std::string str = utf::istreambuf<char16_t>(source) 
        | utf::view::consume_bom 
        | utf::view::utf8 
        | utf::view::line_end_convert; 

Answer 3

아기에서 내 UTF-8 코드입니다 X (https://github.com/MalcolmMcLean/babyx) 모든 것이 제대로 작동하는지

static const unsigned int offsetsFromUTF8[6] = 
{ 
    0x00000000UL, 0x00003080UL, 0x000E2080UL, 
    0x03C82080UL, 0xFA082080UL, 0x82082080UL 
}; 

static const unsigned char trailingBytesForUTF8[256] = { 
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 
    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 
}; 

int bbx_isutf8z(const char *str) 
{ 
    int len = 0; 
    int pos = 0; 
    int nb; 
    int i; 
    int ch; 

    while(str[len]) 
    len++; 
    while(pos < len && *str) 
    { 
    nb = bbx_utf8_skip(str); 
    if(nb < 1 || nb > 4) 
     return 0; 
    if(pos + nb > len) 
     return 0; 
    for(i=1;i<nb;i++) 
     if((str[i] & 0xC0) != 0x80) 
     return 0; 
    ch = bbx_utf8_getch(str); 
    if(ch < 0x80) 
    { 
     if(nb != 1) 
     return 0; 
    } 
    else if(ch < 0x8000) 
    { 
     if(nb != 2) 
     return 0; 
    } 
    else if(ch < 0x10000) 
    { 
     if(nb != 3) 
     return 0; 
    } 
    else if(ch < 0x110000) 
    { 
     if(nb != 4) 
     return 0; 
    } 
    pos += nb; 
    str += nb;  
    } 

    return 1; 
} 

int bbx_utf8_skip(const char *utf8) 
{ 
    return trailingBytesForUTF8[(unsigned char) *utf8] + 1; 
} 

int bbx_utf8_getch(const char *utf8) 
{ 
    int ch; 
    int nb; 

    nb = trailingBytesForUTF8[(unsigned char)*utf8]; 
    ch = 0; 
    switch (nb) 
    { 
      /* these fall through deliberately */ 
     case 3: ch += (unsigned char)*utf8++; ch <<= 6; 
     case 2: ch += (unsigned char)*utf8++; ch <<= 6; 
     case 1: ch += (unsigned char)*utf8++; ch <<= 6; 
     case 0: ch += (unsigned char)*utf8++; 
    } 
    ch -= offsetsFromUTF8[nb]; 

    return ch; 
} 

int bbx_utf8_putch(char *out, int ch) 
{ 
    char *dest = out; 
    if (ch < 0x80) 
    { 
    *dest++ = (char)ch; 
    } 
    else if (ch < 0x800) 
    { 
    *dest++ = (ch>>6) | 0xC0; 
    *dest++ = (ch & 0x3F) | 0x80; 
    } 
    else if (ch < 0x10000) 
    { 
    *dest++ = (ch>>12) | 0xE0; 
    *dest++ = ((ch>>6) & 0x3F) | 0x80; 
    *dest++ = (ch & 0x3F) | 0x80; 
    } 
    else if (ch < 0x110000) 
    { 
    *dest++ = (ch>>18) | 0xF0; 
    *dest++ = ((ch>>12) & 0x3F) | 0x80; 
    *dest++ = ((ch>>6) & 0x3F) | 0x80; 
    *dest++ = (ch & 0x3F) | 0x80; 
    } 
    else 
    return 0; 
    return dest - out; 
} 

int bbx_utf8_charwidth(int ch) 
{ 
    if (ch < 0x80) 
    { 
     return 1; 
    } 
    else if (ch < 0x800) 
    { 
     return 2; 
    } 
    else if (ch < 0x10000) 
    { 
     return 3; 
    } 
    else if (ch < 0x110000) 
    { 
     return 4; 
    } 
    else 
     return 0; 
} 

int bbx_utf8_Nchars(const char *utf8) 
{ 
    int answer = 0; 

    while(*utf8) 
    { 
    utf8 += bbx_utf8_skip(utf8); 
    answer++; 
    } 

    return answer; 
}