utf8::internal Namespace Reference

Enumerations
enum	utf_error {   UTF8_OK , NOT_ENOUGH_ROOM , INVALID_LEAD , INCOMPLETE_SEQUENCE ,   OVERLONG_SEQUENCE , INVALID_CODE_POINT }

Functions
template<typename octet_type >
uint8_t	mask8 (octet_type oc)
template<typename u16_type >
uint16_t	mask16 (u16_type oc)
template<typename octet_type >
bool	is_trail (octet_type oc)
template<typename u16 >
bool	is_lead_surrogate (u16 cp)
template<typename u16 >
bool	is_trail_surrogate (u16 cp)
template<typename u16 >
bool	is_surrogate (u16 cp)
template<typename u32 >
bool	is_code_point_valid (u32 cp)
template<typename octet_iterator >
std::iterator_traits< octet_iterator >::difference_type	sequence_length (octet_iterator lead_it)
template<typename octet_difference_type >
bool	is_overlong_sequence (uint32_t cp, octet_difference_type length)
template<typename octet_iterator >
utf_error	increase_safely (octet_iterator &it, octet_iterator end)
	Helper for get_sequence_x.
template<typename octet_iterator >
utf_error	get_sequence_1 (octet_iterator &it, octet_iterator end, uint32_t &code_point)
	get_sequence_x functions decode utf-8 sequences of the length x
template<typename octet_iterator >
utf_error	get_sequence_2 (octet_iterator &it, octet_iterator end, uint32_t &code_point)
template<typename octet_iterator >
utf_error	get_sequence_3 (octet_iterator &it, octet_iterator end, uint32_t &code_point)
template<typename octet_iterator >
utf_error	get_sequence_4 (octet_iterator &it, octet_iterator end, uint32_t &code_point)
template<typename octet_iterator >
utf_error	validate_next (octet_iterator &it, octet_iterator end, uint32_t &code_point)
template<typename octet_iterator >
utf_error	validate_next (octet_iterator &it, octet_iterator end)

Variables
const uint16_t	LEAD_SURROGATE_MIN = 0xd800u
const uint16_t	LEAD_SURROGATE_MAX = 0xdbffu
const uint16_t	TRAIL_SURROGATE_MIN = 0xdc00u
const uint16_t	TRAIL_SURROGATE_MAX = 0xdfffu
const uint16_t	LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10)
const uint32_t	SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN
const uint32_t	CODE_POINT_MAX = 0x0010ffffu

Enumeration Type Documentation

utf_error

enum utf8::internal::utf_error

Enumerator
UTF8_OK
NOT_ENOUGH_ROOM
INVALID_LEAD
INCOMPLETE_SEQUENCE
OVERLONG_SEQUENCE
INVALID_CODE_POINT

Definition at line 134 of file core.h.

{UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};

Function Documentation

get_sequence_1()

template<typename octet_iterator >

utf_error utf8::internal::get_sequence_1	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point )

Definition at line 153 of file core.h.

    {
        if (it == end)
            return NOT_ENOUGH_ROOM;
 
        code_point = utf8::internal::mask8(*it);
 
        return UTF8_OK;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

get_sequence_2()

template<typename octet_iterator >

utf_error utf8::internal::get_sequence_2	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point )

Definition at line 164 of file core.h.

    {
        if (it == end) 
            return NOT_ENOUGH_ROOM;
        
        code_point = utf8::internal::mask8(*it);
 
        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
 
        code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);
 
        return UTF8_OK;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

get_sequence_3()

template<typename octet_iterator >

utf_error utf8::internal::get_sequence_3	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point )

Definition at line 179 of file core.h.

    {
        if (it == end)
            return NOT_ENOUGH_ROOM;
            
        code_point = utf8::internal::mask8(*it);
 
        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
 
        code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
 
        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
 
        code_point += (*it) & 0x3f;
 
        return UTF8_OK;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

get_sequence_4()

template<typename octet_iterator >

utf_error utf8::internal::get_sequence_4	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point )

Definition at line 198 of file core.h.

    {
        if (it == end)
           return NOT_ENOUGH_ROOM;
 
        code_point = utf8::internal::mask8(*it);
 
        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
 
        code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
 
        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
 
        code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
 
        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
 
        code_point += (*it) & 0x3f;
 
        return UTF8_OK;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

increase_safely()

template<typename octet_iterator >

utf_error utf8::internal::increase_safely	(	octet_iterator &	it,
		octet_iterator	end )

Definition at line 138 of file core.h.

    {
        if (++it == end)
            return NOT_ENOUGH_ROOM;
 
        if (!utf8::internal::is_trail(*it))
            return INCOMPLETE_SEQUENCE;
        
        return UTF8_OK;
    }

Here is the call graph for this function:

is_code_point_valid()

template<typename u32 >

bool utf8::internal::is_code_point_valid ( u32 cp )

inline

Definition at line 93 of file core.h.

    {
        return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
    }

Here is the call graph for this function:

Here is the caller graph for this function:

is_lead_surrogate()

template<typename u16 >

bool utf8::internal::is_lead_surrogate ( u16 cp )

inline

Definition at line 75 of file core.h.

    {
        return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
    }

Here is the caller graph for this function:

is_overlong_sequence()

template<typename octet_difference_type >

bool utf8::internal::is_overlong_sequence ( uint32_t cp, octet_difference_type length )

inline

Definition at line 116 of file core.h.

    {
        if (cp < 0x80) {
            if (length != 1) 
                return true;
        }
        else if (cp < 0x800) {
            if (length != 2) 
                return true;
        }
        else if (cp < 0x10000) {
            if (length != 3) 
                return true;
        }
 
        return false;
    }

Here is the caller graph for this function:

is_surrogate()

template<typename u16 >

bool utf8::internal::is_surrogate ( u16 cp )

inline

Definition at line 87 of file core.h.

    {
        return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
    }

Here is the caller graph for this function:

is_trail()

template<typename octet_type >

bool utf8::internal::is_trail ( octet_type oc )

inline

Definition at line 69 of file core.h.

    {
        return ((utf8::internal::mask8(oc) >> 6) == 0x2);
    }

Here is the call graph for this function:

Here is the caller graph for this function:

is_trail_surrogate()

template<typename u16 >

bool utf8::internal::is_trail_surrogate ( u16 cp )

inline

Definition at line 81 of file core.h.

    {
        return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
    }

Here is the caller graph for this function:

mask16()

template<typename u16_type >

uint16_t utf8::internal::mask16 ( u16_type oc )

inline

Definition at line 64 of file core.h.

    {
        return static_cast<uint16_t>(0xffff & oc);
    }

Here is the caller graph for this function:

mask8()

template<typename octet_type >

uint8_t utf8::internal::mask8 ( octet_type oc )

inline

Definition at line 59 of file core.h.

    {
        return static_cast<uint8_t>(0xff & oc);
    }

Here is the caller graph for this function:

sequence_length()

template<typename octet_iterator >

std::iterator_traits< octet_iterator >::difference_type utf8::internal::sequence_length ( octet_iterator lead_it )

inline

Definition at line 100 of file core.h.

    {
        uint8_t lead = utf8::internal::mask8(*lead_it);
        if (lead < 0x80)
            return 1;
        else if ((lead >> 5) == 0x6)
            return 2;
        else if ((lead >> 4) == 0xe)
            return 3;
        else if ((lead >> 3) == 0x1e)
            return 4;
        else
            return 0;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

validate_next() [1/2]

template<typename octet_iterator >

utf_error utf8::internal::validate_next ( octet_iterator & it, octet_iterator end )

inline

Definition at line 275 of file core.h.

                                                                           {
        uint32_t ignored;
        return utf8::internal::validate_next(it, end, ignored);
    }

Here is the call graph for this function:

validate_next() [2/2]

template<typename octet_iterator >

utf_error utf8::internal::validate_next	(	octet_iterator &	it,
		octet_iterator	end,
		uint32_t &	code_point )

Definition at line 223 of file core.h.

    {
        // Save the original value of it so we can go back in case of failure
        // Of course, it does not make much sense with i.e. stream iterators
        octet_iterator original_it = it;
 
        uint32_t cp = 0;
        // Determine the sequence length based on the lead octet
        typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
        const octet_difference_type length = utf8::internal::sequence_length(it);
 
        // Get trail octets and calculate the code point
        utf_error err = UTF8_OK;
        switch (length) {
            case 0: 
                return INVALID_LEAD;
            case 1:
                err = utf8::internal::get_sequence_1(it, end, cp);
                break;
            case 2:
                err = utf8::internal::get_sequence_2(it, end, cp);
            break;
            case 3:
                err = utf8::internal::get_sequence_3(it, end, cp);
            break;
            case 4:
                err = utf8::internal::get_sequence_4(it, end, cp);
            break;
        }
 
        if (err == UTF8_OK) {
            // Decoding succeeded. Now, security checks...
            if (utf8::internal::is_code_point_valid(cp)) {
                if (!utf8::internal::is_overlong_sequence(cp, length)){
                    // Passed! Return here.
                    code_point = cp;
                    ++it;
                    return UTF8_OK;
                }
                else
                    err = OVERLONG_SEQUENCE;
            }
            else 
                err = INVALID_CODE_POINT;
        }
 
        // Failure branch - restore the original value of the iterator
        it = original_it;
        return err;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

CODE_POINT_MAX

const uint32_t utf8::internal::CODE_POINT_MAX = 0x0010ffffu

Definition at line 56 of file core.h.

LEAD_OFFSET

const uint16_t utf8::internal::LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10)

Definition at line 52 of file core.h.

LEAD_SURROGATE_MAX

const uint16_t utf8::internal::LEAD_SURROGATE_MAX = 0xdbffu

Definition at line 49 of file core.h.

LEAD_SURROGATE_MIN

const uint16_t utf8::internal::LEAD_SURROGATE_MIN = 0xd800u

Definition at line 48 of file core.h.

SURROGATE_OFFSET

const uint32_t utf8::internal::SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN

Definition at line 53 of file core.h.

TRAIL_SURROGATE_MAX

const uint16_t utf8::internal::TRAIL_SURROGATE_MAX = 0xdfffu

Definition at line 51 of file core.h.

TRAIL_SURROGATE_MIN

const uint16_t utf8::internal::TRAIL_SURROGATE_MIN = 0xdc00u

Definition at line 50 of file core.h.