utf8 Namespace Reference

Namespaces
namespace	internal
namespace	unchecked

Classes
class	exception
class	invalid_code_point
class	invalid_utf16
class	invalid_utf8
class	iterator
class	not_enough_room

Functions
template<typename octet_iterator >
octet_iterator	append (uint32_t cp, octet_iterator result)
	The library API - functions intended to be called by the users.
template<typename octet_iterator , typename output_iterator >
output_iterator	replace_invalid (octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
template<typename octet_iterator , typename output_iterator >
output_iterator	replace_invalid (octet_iterator start, octet_iterator end, output_iterator out)
template<typename octet_iterator >
uint32_t	next (octet_iterator &it, octet_iterator end)
template<typename octet_iterator >
uint32_t	peek_next (octet_iterator it, octet_iterator end)
template<typename octet_iterator >
uint32_t	prior (octet_iterator &it, octet_iterator start)
template<typename octet_iterator >
uint32_t	previous (octet_iterator &it, octet_iterator pass_start)
	Deprecated in versions that include "prior".
template<typename octet_iterator , typename distance_type >
void	advance (octet_iterator &it, distance_type n, octet_iterator end)
template<typename octet_iterator >
std::iterator_traits< octet_iterator >::difference_type	distance (octet_iterator first, octet_iterator last)
template<typename u16bit_iterator , typename octet_iterator >
octet_iterator	utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
template<typename u16bit_iterator , typename octet_iterator >
u16bit_iterator	utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
template<typename octet_iterator , typename u32bit_iterator >
octet_iterator	utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
template<typename octet_iterator , typename u32bit_iterator >
u32bit_iterator	utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
template<typename octet_iterator >
octet_iterator	find_invalid (octet_iterator start, octet_iterator end)
template<typename octet_iterator >
bool	is_valid (octet_iterator start, octet_iterator end)
template<typename octet_iterator >
bool	starts_with_bom (octet_iterator it, octet_iterator end)
template<typename octet_iterator >
bool	is_bom (octet_iterator it)

Variables
const uint8_t	bom [] = {0xef, 0xbb, 0xbf}
	The library API - functions intended to be called by the users.

Function Documentation

advance()

template<typename octet_iterator , typename distance_type >

void utf8::advance	(	octet_iterator &	it,
		distance_type	n,
		octet_iterator	end )

Definition at line 190 of file checked.h.

    {
        for (distance_type i = 0; i < n; ++i)
            utf8::next(it, end);
    }

Here is the call graph for this function:

append()

template<typename octet_iterator >

octet_iterator utf8::append	(	uint32_t	cp,
		octet_iterator	result )

Definition at line 73 of file checked.h.

    {
        if (!utf8::internal::is_code_point_valid(cp))
            throw invalid_code_point(cp);
 
        if (cp < 0x80)                        // one octet
            *(result++) = static_cast<uint8_t>(cp);
        else if (cp < 0x800) {                // two octets
            *(result++) = static_cast<uint8_t>((cp >> 6)            | 0xc0);
            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
        }
        else if (cp < 0x10000) {              // three octets
            *(result++) = static_cast<uint8_t>((cp >> 12)           | 0xe0);
            *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f)   | 0x80);
            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
        }
        else {                                // four octets
            *(result++) = static_cast<uint8_t>((cp >> 18)           | 0xf0);
            *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)  | 0x80);
            *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f)   | 0x80);
            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
        }
        return result;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

distance()

template<typename octet_iterator >

std::iterator_traits< octet_iterator >::difference_type utf8::distance	(	octet_iterator	first,
		octet_iterator	last )

Definition at line 198 of file checked.h.

    {
        typename std::iterator_traits<octet_iterator>::difference_type dist;
        for (dist = 0; first < last; ++dist)
            utf8::next(first, last);
        return dist;
    }

Here is the call graph for this function:

find_invalid()

template<typename octet_iterator >

octet_iterator utf8::find_invalid	(	octet_iterator	start,
		octet_iterator	end )

Definition at line 288 of file core.h.

    {
        octet_iterator result = start;
        while (result != end) {
            utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end);
            if (err_code != internal::UTF8_OK)
                return result;
        }
        return result;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

is_bom()

template<typename octet_iterator >

bool utf8::is_bom ( octet_iterator it )

inline

Definition at line 317 of file core.h.

    {
        return (
            (utf8::internal::mask8(*it++)) == bom[0] &&
            (utf8::internal::mask8(*it++)) == bom[1] &&
            (utf8::internal::mask8(*it))   == bom[2]
           );
    }

Here is the call graph for this function:

is_valid()

template<typename octet_iterator >

bool utf8::is_valid ( octet_iterator start, octet_iterator end )

inline

Definition at line 300 of file core.h.

    {
        return (utf8::find_invalid(start, end) == end);
    }

Here is the call graph for this function:

Here is the caller graph for this function:

next()

template<typename octet_iterator >

uint32_t utf8::next	(	octet_iterator &	it,
		octet_iterator	end )

Definition at line 137 of file checked.h.

    {
        uint32_t cp = 0;
        internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
        switch (err_code) {
            case internal::UTF8_OK :
                break;
            case internal::NOT_ENOUGH_ROOM :
                throw not_enough_room();
            case internal::INVALID_LEAD :
            case internal::INCOMPLETE_SEQUENCE :
            case internal::OVERLONG_SEQUENCE :
                throw invalid_utf8(*it);
            case internal::INVALID_CODE_POINT :
                throw invalid_code_point(cp);
        }
        return cp;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

peek_next()

template<typename octet_iterator >

uint32_t utf8::peek_next	(	octet_iterator	it,
		octet_iterator	end )

Definition at line 157 of file checked.h.

    {
        return utf8::next(it, end);
    }

Here is the call graph for this function:

Here is the caller graph for this function:

previous()

template<typename octet_iterator >

uint32_t utf8::previous	(	octet_iterator &	it,
		octet_iterator	pass_start )

Definition at line 179 of file checked.h.

    {
        octet_iterator end = it;
        while (utf8::internal::is_trail(*(--it)))
            if (it == pass_start)
                throw invalid_utf8(*it); // error - no lead byte in the sequence
        octet_iterator temp = it;
        return utf8::next(temp, end);
    }

Here is the call graph for this function:

prior()

template<typename octet_iterator >

uint32_t utf8::prior	(	octet_iterator &	it,
		octet_iterator	start )

Definition at line 163 of file checked.h.

    {
        // can't do much if it == start
        if (it == start)
            throw not_enough_room();
 
        octet_iterator end = it;
        // Go back until we hit either a lead octet or start
        while (utf8::internal::is_trail(*(--it)))
            if (it == start)
                throw invalid_utf8(*it); // error - no lead byte in the sequence
        return utf8::peek_next(it, end);
    }

Here is the call graph for this function:

Here is the caller graph for this function:

replace_invalid() [1/2]

template<typename octet_iterator , typename output_iterator >

output_iterator utf8::replace_invalid ( octet_iterator start, octet_iterator end, output_iterator out )

inline

Definition at line 130 of file checked.h.

    {
        static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
        return utf8::replace_invalid(start, end, out, replacement_marker);
    }

Here is the call graph for this function:

replace_invalid() [2/2]

template<typename octet_iterator , typename output_iterator >

output_iterator utf8::replace_invalid	(	octet_iterator	start,
		octet_iterator	end,
		output_iterator	out,
		uint32_t	replacement )

Definition at line 99 of file checked.h.

    {
        while (start != end) {
            octet_iterator sequence_start = start;
            internal::utf_error err_code = utf8::internal::validate_next(start, end);
            switch (err_code) {
                case internal::UTF8_OK :
                    for (octet_iterator it = sequence_start; it != start; ++it)
                        *out++ = *it;
                    break;
                case internal::NOT_ENOUGH_ROOM:
                    throw not_enough_room();
                case internal::INVALID_LEAD:
                    out = utf8::append (replacement, out);
                    ++start;
                    break;
                case internal::INCOMPLETE_SEQUENCE:
                case internal::OVERLONG_SEQUENCE:
                case internal::INVALID_CODE_POINT:
                    out = utf8::append (replacement, out);
                    ++start;
                    // just one replacement mark for the sequence
                    while (start != end && utf8::internal::is_trail(*start))
                        ++start;
                    break;
            }
        }
        return out;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

starts_with_bom()

template<typename octet_iterator >

bool utf8::starts_with_bom ( octet_iterator it, octet_iterator end )

inline

Definition at line 306 of file core.h.

    {
        return (
            ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) &&
            ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
            ((it != end) && (utf8::internal::mask8(*it))   == bom[2])
           );
    }

Here is the call graph for this function:

utf16to8()

template<typename u16bit_iterator , typename octet_iterator >

octet_iterator utf8::utf16to8	(	u16bit_iterator	start,
		u16bit_iterator	end,
		octet_iterator	result )

Definition at line 207 of file checked.h.

    {
        while (start != end) {
            uint32_t cp = utf8::internal::mask16(*start++);
            // Take care of surrogate pairs first
            if (utf8::internal::is_lead_surrogate(cp)) {
                if (start != end) {
                    uint32_t trail_surrogate = utf8::internal::mask16(*start++);
                    if (utf8::internal::is_trail_surrogate(trail_surrogate))
                        cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
                    else
                        throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
                }
                else
                    throw invalid_utf16(static_cast<uint16_t>(cp));
 
            }
            // Lone trail surrogate
            else if (utf8::internal::is_trail_surrogate(cp))
                throw invalid_utf16(static_cast<uint16_t>(cp));
 
            result = utf8::append(cp, result);
        }
        return result;
    }

Here is the call graph for this function:

utf32to8()

template<typename octet_iterator , typename u32bit_iterator >

octet_iterator utf8::utf32to8	(	u32bit_iterator	start,
		u32bit_iterator	end,
		octet_iterator	result )

Definition at line 249 of file checked.h.

    {
        while (start != end)
            result = utf8::append(*(start++), result);
 
        return result;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

utf8to16()

template<typename u16bit_iterator , typename octet_iterator >

u16bit_iterator utf8::utf8to16	(	octet_iterator	start,
		octet_iterator	end,
		u16bit_iterator	result )

Definition at line 234 of file checked.h.

    {
        while (start != end) {
            uint32_t cp = utf8::next(start, end);
            if (cp > 0xffff) { //make a surrogate pair
                *result++ = static_cast<uint16_t>((cp >> 10)   + internal::LEAD_OFFSET);
                *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
            }
            else
                *result++ = static_cast<uint16_t>(cp);
        }
        return result;
    }

Here is the call graph for this function:

utf8to32()

template<typename octet_iterator , typename u32bit_iterator >

u32bit_iterator utf8::utf8to32	(	octet_iterator	start,
		octet_iterator	end,
		u32bit_iterator	result )

Definition at line 258 of file checked.h.

    {
        while (start != end)
            (*result++) = utf8::next(start, end);
 
        return result;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

bom

const uint8_t utf8::bom[] = {0xef, 0xbb, 0xbf}

Definition at line 285 of file core.h.

{0xef, 0xbb, 0xbf};