UTF-8 part 2: Encoding wide-character strings into UTF-8
All checks were successful
continuous-integration/drone/push Build is passing

We now have Utf8StringEncoder and Utf8Encoder (no state this time)
This commit is contained in:
apio 2022-12-18 14:34:50 +01:00
parent 9c1c6bb320
commit 6389099808
Signed by: apio
GPG Key ID: B8A7D06E42258954
2 changed files with 120 additions and 4 deletions

View File

@ -15,7 +15,7 @@ class Utf8StringDecoder
Result<usize> code_points() const;
// The caller must ensure that 'buf' is at least wide_length() + a NULL wide.
// The caller must ensure that 'buf' is at least code_points() + a NULL wide.
Result<void> decode(wchar_t* buf) const;
private:
@ -23,6 +23,26 @@ class Utf8StringDecoder
usize m_byte_length;
};
class Utf8StringEncoder
{
public:
Utf8StringEncoder(const wchar_t* str);
usize code_points() const
{
return m_code_points;
}
Result<usize> byte_length() const;
// The caller must ensure that 'buf' is at least byte_length() + a NULL wide.
Result<void> encode(char* buf) const;
private:
const wchar_t* m_str;
usize m_code_points;
};
class Utf8StateDecoder
{
public:
@ -35,4 +55,11 @@ class Utf8StateDecoder
char m_state[4];
usize m_state_len = 0;
usize m_state_index = 0;
};
class Utf8Encoder
{
public:
// Does not null-terminate. Returns the number of bytes written.
Result<usize> encode(wchar_t c, char buf[4]);
};

View File

@ -10,7 +10,52 @@ static Result<usize> utf8_char_length(char c)
return err(EILSEQ);
}
static Result<wchar_t> encode_utf8_char_into_wide_char(const char* beg, usize& len)
static Result<usize> wide_char_length_as_utf8(wchar_t c)
{
if (c <= 0x7f) return 1;
if (c <= 0x7ff) return 2;
if (c <= 0xffff) return 3;
if (c <= 0x10ffff) return 4;
return err(EILSEQ);
}
static Result<void> encode_wide_char_as_utf8(wchar_t c, char* result, usize& len)
{
len = TRY(wide_char_length_as_utf8(c));
u8* buf = (u8*)result;
if (len == 1)
{
buf[0] = c & 0x7f;
return {};
}
if (len == 2)
{
buf[0] = 0b11000000 | ((c & 0x7c0) >> 6);
buf[1] = 0b10000000 | (c & 0x3f);
return {};
}
if (len == 3)
{
buf[0] = 0b11100000 | ((c & 0xf000) >> 12);
buf[1] = 0b10000000 | ((c & 0xfc0) >> 6);
buf[2] = 0b10000000 | (c & 0x3f);
return {};
}
if (len == 4)
{
buf[0] = 0b11110000 | ((c & 0x1c0000) >> 18);
buf[1] = 0b10000000 | ((c & 0x3f000) >> 12);
buf[2] = 0b10000000 | ((c & 0xfc0) >> 6);
buf[3] = 0b10000000 | (c & 0x3f);
return {};
}
unreachable();
}
static Result<wchar_t> encode_utf8_as_wide_char(const char* beg, usize& len)
{
usize utf8_len = TRY(utf8_char_length(*beg));
if (utf8_len > len) return err(EILSEQ); // Unterminated sequence
@ -82,7 +127,7 @@ Result<void> Utf8StringDecoder::decode(wchar_t* buf) const
while ((usize)(it - m_str) < m_byte_length)
{
usize len = m_byte_length - (usize)(it - m_str); // Remaining space
*buf = TRY(encode_utf8_char_into_wide_char(it, len));
*buf = TRY(encode_utf8_as_wide_char(it, len));
it += len;
buf++;
}
@ -92,6 +137,41 @@ Result<void> Utf8StringDecoder::decode(wchar_t* buf) const
return {};
}
Utf8StringEncoder::Utf8StringEncoder(const wchar_t* str) : m_str(str), m_code_points(wcslen(str))
{
}
Result<usize> Utf8StringEncoder::byte_length() const
{
const wchar_t* it = m_str;
usize len = 0;
while (*it)
{
len += TRY(wide_char_length_as_utf8(*it));
it++;
}
return len;
}
Result<void> Utf8StringEncoder::encode(char* buf) const
{
const wchar_t* it = m_str;
while (*it)
{
usize len = 0;
TRY(encode_wide_char_as_utf8(*it, buf, len));
buf += len;
it++;
}
*buf = 0;
return {};
}
Utf8StateDecoder::Utf8StateDecoder() : m_state_len(0), m_state_index(0)
{
}
@ -117,7 +197,7 @@ Result<Option<wchar_t>> Utf8StateDecoder::feed(char c)
if (m_state_index == m_state_len - 1)
{
usize len = m_state_len;
wchar_t wc = TRY(encode_utf8_char_into_wide_char(m_state, len));
wchar_t wc = TRY(encode_utf8_as_wide_char(m_state, len));
m_state_len = 0;
return Option<wchar_t>{wc};
}
@ -128,4 +208,13 @@ Result<Option<wchar_t>> Utf8StateDecoder::feed(char c)
void Utf8StateDecoder::reset()
{
m_state_index = m_state_len = 0;
}
Result<usize> Utf8Encoder::encode(wchar_t c, char buf[4])
{
usize len = 0;
TRY(encode_wide_char_as_utf8(c, buf, len));
return len;
}