/**
 * pugixml parser - version 1.2
 * --------------------------------------------------------
 * Copyright (C) 2006-2012, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
 * Report bugs and download new versions at http://pugixml.org/
 *
 * This library is distributed under the MIT License. See notice at the end
 * of this file.
 *
 * This work is based on the pugxml parser, which is:
 * Copyright (C) 2003, by Kristen Wegner (kristen@tima.net)
 */

//#ifndef SOURCE_PUGIXML_CPP
//#define SOURCE_PUGIXML_CPP

#include "Precompiled.h"

#include "pugixml.hpp"

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <wchar.h>

#ifndef PUGIXML_NO_XPATH
	#include <math.h>
	#include <float.h>
	#ifdef PUGIXML_NO_EXCEPTIONS
		#include <setjmp.h>
	#endif
#endif

#ifndef PUGIXML_NO_STL
	#include <istream>
	#include <ostream>
	#include <string>
#endif

// For placement new
#include <new>

#ifdef MYGUI_CHECK_MEMORY_LEAKS
	#undef new
	#undef delete
#endif

#ifdef _MSC_VER
	#pragma warning(push)
	#pragma warning(disable : 4127) // conditional expression is constant
	#pragma warning(disable : 4324) // structure was padded due to __declspec(align())
	#pragma warning(disable : 4611) // interaction between '_setjmp' and C++ object destruction is non-portable
	#pragma warning(disable : 4702) // unreachable code
	#pragma warning(disable : 4996) // this function or variable may be unsafe
	#pragma warning(disable : 4793) // function compiled as native: presence of '_setjmp' makes a function unmanaged
#endif

#ifdef __INTEL_COMPILER
	#pragma warning(disable : 177) // function was declared but never referenced
	#pragma warning(disable : 279) // controlling expression is constant
	#pragma warning(disable : 1478 1786) // function was declared "deprecated"
	#pragma warning(disable : 1684) // conversion from pointer to same-sized integral type
#endif

#if defined(__BORLANDC__) && defined(PUGIXML_HEADER_ONLY)
	#pragma warn - \
			8080 // symbol is declared but never used; disabling this inside push/pop bracket does not make the warning go away
#endif

#ifdef __BORLANDC__
	#pragma option push
	#pragma warn - 8008 // condition is always false
	#pragma warn - 8066 // unreachable code
#endif

#ifdef __SNC__
	// Using diag_push/diag_pop does not disable the warnings inside templates due to a compiler bug
	#pragma diag_suppress = 178 // function was declared but never referenced
	#pragma diag_suppress = 237 // controlling expression is constant
#endif

// Inlining controls
#if defined(_MSC_VER) && _MSC_VER >= 1300
	#define PUGI__NO_INLINE __declspec(noinline)
#elif defined(__GNUC__)
	#define PUGI__NO_INLINE __attribute__((noinline))
#else
	#define PUGI__NO_INLINE
#endif

// Simple static assertion
#define PUGI__STATIC_ASSERT(cond) \
	{ \
		static const char condition_failed[(cond) ? 1 : -1] = {0}; \
		(void)condition_failed[0]; \
	}

// Digital Mars C++ bug workaround for passing char loaded from memory via stack
#ifdef __DMC__
	#define PUGI__DMC_VOLATILE volatile
#else
	#define PUGI__DMC_VOLATILE
#endif

// Borland C++ bug workaround for not defining ::memcpy depending on header include order (can't always use std::memcpy because some compilers don't have it at all)
#if defined(__BORLANDC__) && !defined(__MEM_H_USING_LIST)
using std::memcpy;
using std::memmove;
#endif

// In some environments MSVC is a compiler but the CRT lacks certain MSVC-specific features
#if defined(_MSC_VER) && !defined(__S3E__)
	#define PUGI__MSVC_CRT_VERSION _MSC_VER
#endif

#ifdef PUGIXML_HEADER_ONLY
	#define PUGI__NS_BEGIN \
		namespace pugi \
		{ \
			namespace impl \
			{
	#define PUGI__NS_END \
		} \
		}
	#define PUGI__FN inline
	#define PUGI__FN_NO_INLINE inline
#else
	#if defined(_MSC_VER) && \
		_MSC_VER < 1300 // MSVC6 seems to have an amusing bug with anonymous namespaces inside namespaces
		#define PUGI__NS_BEGIN \
			namespace pugi \
			{ \
				namespace impl \
				{
		#define PUGI__NS_END \
			} \
			}
	#else
		#define PUGI__NS_BEGIN \
			namespace pugi \
			{ \
				namespace impl \
				{ \
					namespace \
					{
		#define PUGI__NS_END \
			} \
			} \
			}
	#endif
	#define PUGI__FN
	#define PUGI__FN_NO_INLINE PUGI__NO_INLINE
#endif

// uintptr_t
#if !defined(_MSC_VER) || _MSC_VER >= 1600
	#include <stdint.h>
#else
	#ifndef _UINTPTR_T_DEFINED
// No native uintptr_t in MSVC6 and in some WinCE versions
typedef size_t uintptr_t;
		#define _UINTPTR_T_DEFINED
	#endif
PUGI__NS_BEGIN
typedef unsigned __int8 uint8_t;
typedef unsigned __int16 uint16_t;
typedef unsigned __int32 uint32_t;
PUGI__NS_END
#endif

// Memory allocation
PUGI__NS_BEGIN
PUGI__FN void* default_allocate(size_t size)
{
	return malloc(size);
}

PUGI__FN void default_deallocate(void* ptr)
{
	free(ptr);
}

template<typename T>
struct xml_memory_management_function_storage
{
	static allocation_function allocate;
	static deallocation_function deallocate;
};

template<typename T>
allocation_function xml_memory_management_function_storage<T>::allocate = default_allocate;
template<typename T>
deallocation_function xml_memory_management_function_storage<T>::deallocate = default_deallocate;

typedef xml_memory_management_function_storage<int> xml_memory;
PUGI__NS_END

// String utilities
PUGI__NS_BEGIN
// Get string length
PUGI__FN size_t strlength(const char_t* s)
{
	assert(s);

#ifdef PUGIXML_WCHAR_MODE
	return wcslen(s);
#else
	return strlen(s);
#endif
}

// Compare two strings
PUGI__FN bool strequal(const char_t* src, const char_t* dst)
{
	assert(src && dst);

#ifdef PUGIXML_WCHAR_MODE
	return wcscmp(src, dst) == 0;
#else
	return strcmp(src, dst) == 0;
#endif
}

// Compare lhs with [rhs_begin, rhs_end)
PUGI__FN bool strequalrange(const char_t* lhs, const char_t* rhs, size_t count)
{
	for (size_t i = 0; i < count; ++i)
		if (lhs[i] != rhs[i])
			return false;

	return lhs[count] == 0;
}

#ifdef PUGIXML_WCHAR_MODE
// Convert string to wide string, assuming all symbols are ASCII
PUGI__FN void widen_ascii(wchar_t* dest, const char* source)
{
	for (const char* i = source; *i; ++i)
		*dest++ = *i;
	*dest = 0;
}
#endif
PUGI__NS_END

#if !defined(PUGIXML_NO_STL) || !defined(PUGIXML_NO_XPATH)
// auto_ptr-like buffer holder for exception recovery
PUGI__NS_BEGIN
struct buffer_holder
{
	void* data;
	void (*deleter)(void*);

	buffer_holder(void* data_, void (*deleter_)(void*)) :
		data(data_),
		deleter(deleter_)
	{
	}

	~buffer_holder()
	{
		if (data)
			deleter(data);
	}

	void* release()
	{
		void* result = data;
		data = 0;
		return result;
	}
};
PUGI__NS_END
#endif

PUGI__NS_BEGIN
static const size_t xml_memory_page_size =
#ifdef PUGIXML_MEMORY_PAGE_SIZE
	PUGIXML_MEMORY_PAGE_SIZE
#else
	32768
#endif
	;

static const uintptr_t xml_memory_page_alignment = 32;
static const uintptr_t xml_memory_page_pointer_mask = ~(xml_memory_page_alignment - 1);
static const uintptr_t xml_memory_page_name_allocated_mask = 16;
static const uintptr_t xml_memory_page_value_allocated_mask = 8;
static const uintptr_t xml_memory_page_type_mask = 7;

struct xml_allocator;

struct xml_memory_page
{
	static xml_memory_page* construct(void* memory)
	{
		if (!memory)
			return 0; //$ redundant, left for performance

		xml_memory_page* result = static_cast<xml_memory_page*>(memory);

		result->allocator = 0;
		result->memory = 0;
		result->prev = 0;
		result->next = 0;
		result->busy_size = 0;
		result->freed_size = 0;

		return result;
	}

	xml_allocator* allocator;

	void* memory;

	xml_memory_page* prev;
	xml_memory_page* next;

	size_t busy_size;
	size_t freed_size;

	char data[1];
};

struct xml_memory_string_header
{
	uint16_t page_offset; // offset from page->data
	uint16_t full_size; // 0 if string occupies whole page
};

struct xml_allocator
{
	xml_allocator(xml_memory_page* root) :
		_root(root),
		_busy_size(root->busy_size)
	{
	}

	xml_memory_page* allocate_page(size_t data_size)
	{
		size_t size = offsetof(xml_memory_page, data) + data_size;

		// allocate block with some alignment, leaving memory for worst-case padding
		void* memory = xml_memory::allocate(size + xml_memory_page_alignment);
		if (!memory)
			return 0;

		// align upwards to page boundary
		void* page_memory = reinterpret_cast<void*>(
			(reinterpret_cast<uintptr_t>(memory) + (xml_memory_page_alignment - 1)) & ~(xml_memory_page_alignment - 1));

		// prepare page structure
		xml_memory_page* page = xml_memory_page::construct(page_memory);

		page->memory = memory;
		page->allocator = _root->allocator;

		return page;
	}

	static void deallocate_page(xml_memory_page* page)
	{
		xml_memory::deallocate(page->memory);
	}

	void* allocate_memory_oob(size_t size, xml_memory_page*& out_page);

	void* allocate_memory(size_t size, xml_memory_page*& out_page)
	{
		if (_busy_size + size > xml_memory_page_size)
			return allocate_memory_oob(size, out_page);

		void* buf = _root->data + _busy_size;

		_busy_size += size;

		out_page = _root;

		return buf;
	}

	void deallocate_memory(void* ptr, size_t size, xml_memory_page* page)
	{
		if (page == _root)
			page->busy_size = _busy_size;

		assert(ptr >= page->data && ptr < page->data + page->busy_size);
		(void)!ptr;

		page->freed_size += size;
		assert(page->freed_size <= page->busy_size);

		if (page->freed_size == page->busy_size)
		{
			if (page->next == 0)
			{
				assert(_root == page);

				// top page freed, just reset sizes
				page->busy_size = page->freed_size = 0;
				_busy_size = 0;
			}
			else
			{
				assert(_root != page);
				assert(page->prev);

				// remove from the list
				page->prev->next = page->next;
				page->next->prev = page->prev;

				// deallocate
				deallocate_page(page);
			}
		}
	}

	char_t* allocate_string(size_t length)
	{
		// allocate memory for string and header block
		size_t size = sizeof(xml_memory_string_header) + length * sizeof(char_t);

		// round size up to pointer alignment boundary
		size_t full_size = (size + (sizeof(void*) - 1)) & ~(sizeof(void*) - 1);

		xml_memory_page* page;
		xml_memory_string_header* header = static_cast<xml_memory_string_header*>(allocate_memory(full_size, page));

		if (!header)
			return 0;

		// setup header
		ptrdiff_t page_offset = reinterpret_cast<char*>(header) - page->data;

		assert(page_offset >= 0 && page_offset < (1 << 16));
		header->page_offset = static_cast<uint16_t>(page_offset);

		// full_size == 0 for large strings that occupy the whole page
		assert(full_size < (1 << 16) || (page->busy_size == full_size && page_offset == 0));
		header->full_size = static_cast<uint16_t>(full_size < (1 << 16) ? full_size : 0);

		// round-trip through void* to avoid 'cast increases required alignment of target type' warning
		// header is guaranteed a pointer-sized alignment, which should be enough for char_t
		return static_cast<char_t*>(static_cast<void*>(header + 1));
	}

	void deallocate_string(char_t* string)
	{
		// this function casts pointers through void* to avoid 'cast increases required alignment of target type' warnings
		// we're guaranteed the proper (pointer-sized) alignment on the input string if it was allocated via allocate_string

		// get header
		xml_memory_string_header* header = static_cast<xml_memory_string_header*>(static_cast<void*>(string)) - 1;

		// deallocate
		size_t page_offset = offsetof(xml_memory_page, data) + header->page_offset;
		xml_memory_page* page =
			reinterpret_cast<xml_memory_page*>(static_cast<void*>(reinterpret_cast<char*>(header) - page_offset));

		// if full_size == 0 then this string occupies the whole page
		size_t full_size = header->full_size == 0 ? page->busy_size : header->full_size;

		deallocate_memory(header, full_size, page);
	}

	xml_memory_page* _root;
	size_t _busy_size;
};

PUGI__FN_NO_INLINE void* xml_allocator::allocate_memory_oob(size_t size, xml_memory_page*& out_page)
{
	const size_t large_allocation_threshold = xml_memory_page_size / 4;

	xml_memory_page* page = allocate_page(size <= large_allocation_threshold ? xml_memory_page_size : size);
	out_page = page;

	if (!page)
		return 0;

	if (size <= large_allocation_threshold)
	{
		_root->busy_size = _busy_size;

		// insert page at the end of linked list
		page->prev = _root;
		_root->next = page;
		_root = page;

		_busy_size = size;
	}
	else
	{
		// insert page before the end of linked list, so that it is deleted as soon as possible
		// the last page is not deleted even if it's empty (see deallocate_memory)
		assert(_root->prev);

		page->prev = _root->prev;
		page->next = _root;

		_root->prev->next = page;
		_root->prev = page;
	}

	// allocate inside page
	page->busy_size = size;

	return page->data;
}
PUGI__NS_END

namespace pugi
{
	/// A 'name=value' XML attribute structure.
	struct xml_attribute_struct
	{
		/// Default ctor
		xml_attribute_struct(impl::xml_memory_page* page) :
			header(reinterpret_cast<uintptr_t>(page)),
			name(0),
			value(0),
			prev_attribute_c(0),
			next_attribute(0)
		{
		}

		uintptr_t header;

		char_t* name; ///< Pointer to attribute name.
		char_t* value; ///< Pointer to attribute value.

		xml_attribute_struct* prev_attribute_c; ///< Previous attribute (cyclic list)
		xml_attribute_struct* next_attribute; ///< Next attribute
	};

	/// An XML document tree node.
	struct xml_node_struct
	{
		/// Default ctor
		/// \param type - node type
		xml_node_struct(impl::xml_memory_page* page, xml_node_type type) :
			header(reinterpret_cast<uintptr_t>(page) | (type - 1)),
			parent(0),
			name(0),
			value(0),
			first_child(0),
			prev_sibling_c(0),
			next_sibling(0),
			first_attribute(0)
		{
		}

		uintptr_t header;

		xml_node_struct* parent; ///< Pointer to parent

		char_t* name; ///< Pointer to element name.
		char_t* value; ///< Pointer to any associated string data.

		xml_node_struct* first_child; ///< First child

		xml_node_struct* prev_sibling_c; ///< Left brother (cyclic list)
		xml_node_struct* next_sibling; ///< Right brother

		xml_attribute_struct* first_attribute; ///< First attribute
	};
}

PUGI__NS_BEGIN
struct xml_document_struct : public xml_node_struct, public xml_allocator
{
	xml_document_struct(xml_memory_page* page) :
		xml_node_struct(page, node_document),
		xml_allocator(page),
		buffer(0)
	{
	}

	const char_t* buffer;
};

inline xml_allocator& get_allocator(const xml_node_struct* node)
{
	assert(node);

	return *reinterpret_cast<xml_memory_page*>(node->header & xml_memory_page_pointer_mask)->allocator;
}
PUGI__NS_END

// Low-level DOM operations
PUGI__NS_BEGIN
inline xml_attribute_struct* allocate_attribute(xml_allocator& alloc)
{
	xml_memory_page* page;
	void* memory = alloc.allocate_memory(sizeof(xml_attribute_struct), page);

	return new (memory) xml_attribute_struct(page);
}

inline xml_node_struct* allocate_node(xml_allocator& alloc, xml_node_type type)
{
	xml_memory_page* page;
	void* memory = alloc.allocate_memory(sizeof(xml_node_struct), page);

	return new (memory) xml_node_struct(page, type);
}

inline void destroy_attribute(xml_attribute_struct* a, xml_allocator& alloc)
{
	uintptr_t header = a->header;

	if (header & impl::xml_memory_page_name_allocated_mask)
		alloc.deallocate_string(a->name);
	if (header & impl::xml_memory_page_value_allocated_mask)
		alloc.deallocate_string(a->value);

	alloc.deallocate_memory(
		a,
		sizeof(xml_attribute_struct),
		reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask));
}

inline void destroy_node(xml_node_struct* n, xml_allocator& alloc)
{
	uintptr_t header = n->header;

	if (header & impl::xml_memory_page_name_allocated_mask)
		alloc.deallocate_string(n->name);
	if (header & impl::xml_memory_page_value_allocated_mask)
		alloc.deallocate_string(n->value);

	for (xml_attribute_struct* attr = n->first_attribute; attr;)
	{
		xml_attribute_struct* next = attr->next_attribute;

		destroy_attribute(attr, alloc);

		attr = next;
	}

	for (xml_node_struct* child = n->first_child; child;)
	{
		xml_node_struct* next = child->next_sibling;

		destroy_node(child, alloc);

		child = next;
	}

	alloc.deallocate_memory(
		n,
		sizeof(xml_node_struct),
		reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask));
}

PUGI__FN_NO_INLINE xml_node_struct* append_node(
	xml_node_struct* node,
	xml_allocator& alloc,
	xml_node_type type = node_element)
{
	xml_node_struct* child = allocate_node(alloc, type);
	if (!child)
		return 0;

	child->parent = node;

	xml_node_struct* first_child = node->first_child;

	if (first_child)
	{
		xml_node_struct* last_child = first_child->prev_sibling_c;

		last_child->next_sibling = child;
		child->prev_sibling_c = last_child;
		first_child->prev_sibling_c = child;
	}
	else
	{
		node->first_child = child;
		child->prev_sibling_c = child;
	}

	return child;
}

PUGI__FN_NO_INLINE xml_attribute_struct* append_attribute_ll(xml_node_struct* node, xml_allocator& alloc)
{
	xml_attribute_struct* a = allocate_attribute(alloc);
	if (!a)
		return 0;

	xml_attribute_struct* first_attribute = node->first_attribute;

	if (first_attribute)
	{
		xml_attribute_struct* last_attribute = first_attribute->prev_attribute_c;

		last_attribute->next_attribute = a;
		a->prev_attribute_c = last_attribute;
		first_attribute->prev_attribute_c = a;
	}
	else
	{
		node->first_attribute = a;
		a->prev_attribute_c = a;
	}

	return a;
}
PUGI__NS_END

// Helper classes for code generation
PUGI__NS_BEGIN
struct opt_false
{
	enum
	{
		value = 0
	};
};

struct opt_true
{
	enum
	{
		value = 1
	};
};
PUGI__NS_END

// Unicode utilities
PUGI__NS_BEGIN
inline uint16_t endian_swap(uint16_t value)
{
	return static_cast<uint16_t>(((value & 0xff) << 8) | (value >> 8));
}

inline uint32_t endian_swap(uint32_t value)
{
	return ((value & 0xff) << 24) | ((value & 0xff00) << 8) | ((value & 0xff0000) >> 8) | (value >> 24);
}

struct utf8_counter
{
	typedef size_t value_type;

	static value_type low(value_type result, uint32_t ch)
	{
		// U+0000..U+007F
		if (ch < 0x80)
			return result + 1;
		// U+0080..U+07FF
		else if (ch < 0x800)
			return result + 2;
		// U+0800..U+FFFF
		else
			return result + 3;
	}

	static value_type high(value_type result, uint32_t)
	{
		// U+10000..U+10FFFF
		return result + 4;
	}
};

struct utf8_writer
{
	typedef uint8_t* value_type;

	static value_type low(value_type result, uint32_t ch)
	{
		// U+0000..U+007F
		if (ch < 0x80)
		{
			*result = static_cast<uint8_t>(ch);
			return result + 1;
		}
		// U+0080..U+07FF
		else if (ch < 0x800)
		{
			result[0] = static_cast<uint8_t>(0xC0 | (ch >> 6));
			result[1] = static_cast<uint8_t>(0x80 | (ch & 0x3F));
			return result + 2;
		}
		// U+0800..U+FFFF
		else
		{
			result[0] = static_cast<uint8_t>(0xE0 | (ch >> 12));
			result[1] = static_cast<uint8_t>(0x80 | ((ch >> 6) & 0x3F));
			result[2] = static_cast<uint8_t>(0x80 | (ch & 0x3F));
			return result + 3;
		}
	}

	static value_type high(value_type result, uint32_t ch)
	{
		// U+10000..U+10FFFF
		result[0] = static_cast<uint8_t>(0xF0 | (ch >> 18));
		result[1] = static_cast<uint8_t>(0x80 | ((ch >> 12) & 0x3F));
		result[2] = static_cast<uint8_t>(0x80 | ((ch >> 6) & 0x3F));
		result[3] = static_cast<uint8_t>(0x80 | (ch & 0x3F));
		return result + 4;
	}

	static value_type any(value_type result, uint32_t ch)
	{
		return (ch < 0x10000) ? low(result, ch) : high(result, ch);
	}
};

struct utf16_counter
{
	typedef size_t value_type;

	static value_type low(value_type result, uint32_t)
	{
		return result + 1;
	}

	static value_type high(value_type result, uint32_t)
	{
		return result + 2;
	}
};

struct utf16_writer
{
	typedef uint16_t* value_type;

	static value_type low(value_type result, uint32_t ch)
	{
		*result = static_cast<uint16_t>(ch);

		return result + 1;
	}

	static value_type high(value_type result, uint32_t ch)
	{
		uint32_t msh = static_cast<uint32_t>(ch - 0x10000) >> 10;
		uint32_t lsh = static_cast<uint32_t>(ch - 0x10000) & 0x3ff;

		result[0] = static_cast<uint16_t>(0xD800 + msh);
		result[1] = static_cast<uint16_t>(0xDC00 + lsh);

		return result + 2;
	}

	static value_type any(value_type result, uint32_t ch)
	{
		return (ch < 0x10000) ? low(result, ch) : high(result, ch);
	}
};

struct utf32_counter
{
	typedef size_t value_type;

	static value_type low(value_type result, uint32_t)
	{
		return result + 1;
	}

	static value_type high(value_type result, uint32_t)
	{
		return result + 1;
	}
};

struct utf32_writer
{
	typedef uint32_t* value_type;

	static value_type low(value_type result, uint32_t ch)
	{
		*result = ch;

		return result + 1;
	}

	static value_type high(value_type result, uint32_t ch)
	{
		*result = ch;

		return result + 1;
	}

	static value_type any(value_type result, uint32_t ch)
	{
		*result = ch;

		return result + 1;
	}
};

struct latin1_writer
{
	typedef uint8_t* value_type;

	static value_type low(value_type result, uint32_t ch)
	{
		*result = static_cast<uint8_t>(ch > 255 ? '?' : ch);

		return result + 1;
	}

	static value_type high(value_type result, uint32_t ch)
	{
		(void)ch;

		*result = '?';

		return result + 1;
	}
};

template<size_t size>
struct wchar_selector;

template<>
struct wchar_selector<2>
{
	typedef uint16_t type;
	typedef utf16_counter counter;
	typedef utf16_writer writer;
};

template<>
struct wchar_selector<4>
{
	typedef uint32_t type;
	typedef utf32_counter counter;
	typedef utf32_writer writer;
};

typedef wchar_selector<sizeof(wchar_t)>::counter wchar_counter;
typedef wchar_selector<sizeof(wchar_t)>::writer wchar_writer;

template<typename Traits, typename opt_swap = opt_false>
struct utf_decoder
{
	static inline typename Traits::value_type decode_utf8_block(
		const uint8_t* data,
		size_t size,
		typename Traits::value_type result)
	{
		const uint8_t utf8_byte_mask = 0x3f;

		while (size)
		{
			uint8_t lead = *data;

			// 0xxxxxxx -> U+0000..U+007F
			if (lead < 0x80)
			{
				result = Traits::low(result, lead);
				data += 1;
				size -= 1;

				// process aligned single-byte (ascii) blocks
				if ((reinterpret_cast<uintptr_t>(data) & 3) == 0)
				{
					// round-trip through void* to silence 'cast increases required alignment of target type' warnings
					while (size >= 4 &&
						   (*static_cast<const uint32_t*>(static_cast<const void*>(data)) & 0x80808080) == 0)
					{
						result = Traits::low(result, data[0]);
						result = Traits::low(result, data[1]);
						result = Traits::low(result, data[2]);
						result = Traits::low(result, data[3]);
						data += 4;
						size -= 4;
					}
				}
			}
			// 110xxxxx -> U+0080..U+07FF
			else if (static_cast<unsigned int>(lead - 0xC0) < 0x20 && size >= 2 && (data[1] & 0xc0) == 0x80)
			{
				result = Traits::low(result, ((lead & ~0xC0) << 6) | (data[1] & utf8_byte_mask));
				data += 2;
				size -= 2;
			}
			// 1110xxxx -> U+0800-U+FFFF
			else if (
				static_cast<unsigned int>(lead - 0xE0) < 0x10 && size >= 3 && (data[1] & 0xc0) == 0x80 &&
				(data[2] & 0xc0) == 0x80)
			{
				result = Traits::low(
					result,
					((lead & ~0xE0) << 12) | ((data[1] & utf8_byte_mask) << 6) | (data[2] & utf8_byte_mask));
				data += 3;
				size -= 3;
			}
			// 11110xxx -> U+10000..U+10FFFF
			else if (
				static_cast<unsigned int>(lead - 0xF0) < 0x08 && size >= 4 && (data[1] & 0xc0) == 0x80 &&
				(data[2] & 0xc0) == 0x80 && (data[3] & 0xc0) == 0x80)
			{
				result = Traits::high(
					result,
					((lead & ~0xF0) << 18) | ((data[1] & utf8_byte_mask) << 12) | ((data[2] & utf8_byte_mask) << 6) |
						(data[3] & utf8_byte_mask));
				data += 4;
				size -= 4;
			}
			// 10xxxxxx or 11111xxx -> invalid
			else
			{
				data += 1;
				size -= 1;
			}
		}

		return result;
	}

	static inline typename Traits::value_type decode_utf16_block(
		const uint16_t* data,
		size_t size,
		typename Traits::value_type result)
	{
		const uint16_t* end = data + size;

		while (data < end)
		{
			uint16_t lead = opt_swap::value ? endian_swap(*data) : *data;

			// U+0000..U+D7FF
			if (lead < 0xD800)
			{
				result = Traits::low(result, lead);
				data += 1;
			}
			// U+E000..U+FFFF
			else if (static_cast<unsigned int>(lead - 0xE000) < 0x2000)
			{
				result = Traits::low(result, lead);
				data += 1;
			}
			// surrogate pair lead
			else if (static_cast<unsigned int>(lead - 0xD800) < 0x400 && data + 1 < end)
			{
				uint16_t next = opt_swap::value ? endian_swap(data[1]) : data[1];

				if (static_cast<unsigned int>(next - 0xDC00) < 0x400)
				{
					result = Traits::high(result, 0x10000 + ((lead & 0x3ff) << 10) + (next & 0x3ff));
					data += 2;
				}
				else
				{
					data += 1;
				}
			}
			else
			{
				data += 1;
			}
		}

		return result;
	}

	static inline typename Traits::value_type decode_utf32_block(
		const uint32_t* data,
		size_t size,
		typename Traits::value_type result)
	{
		const uint32_t* end = data + size;

		while (data < end)
		{
			uint32_t lead = opt_swap::value ? endian_swap(*data) : *data;

			// U+0000..U+FFFF
			if (lead < 0x10000)
			{
				result = Traits::low(result, lead);
				data += 1;
			}
			// U+10000..U+10FFFF
			else
			{
				result = Traits::high(result, lead);
				data += 1;
			}
		}

		return result;
	}

	static inline typename Traits::value_type decode_latin1_block(
		const uint8_t* data,
		size_t size,
		typename Traits::value_type result)
	{
		for (size_t i = 0; i < size; ++i)
		{
			result = Traits::low(result, data[i]);
		}

		return result;
	}

	static inline typename Traits::value_type decode_wchar_block_impl(
		const uint16_t* data,
		size_t size,
		typename Traits::value_type result)
	{
		return decode_utf16_block(data, size, result);
	}

	static inline typename Traits::value_type decode_wchar_block_impl(
		const uint32_t* data,
		size_t size,
		typename Traits::value_type result)
	{
		return decode_utf32_block(data, size, result);
	}

	static inline typename Traits::value_type decode_wchar_block(
		const wchar_t* data,
		size_t size,
		typename Traits::value_type result)
	{
		return decode_wchar_block_impl(
			reinterpret_cast<const wchar_selector<sizeof(wchar_t)>::type*>(data),
			size,
			result);
	}
};

template<typename T>
PUGI__FN void convert_utf_endian_swap(T* result, const T* data, size_t length)
{
	for (size_t i = 0; i < length; ++i)
		result[i] = endian_swap(data[i]);
}

#ifdef PUGIXML_WCHAR_MODE
PUGI__FN void convert_wchar_endian_swap(wchar_t* result, const wchar_t* data, size_t length)
{
	for (size_t i = 0; i < length; ++i)
		result[i] = static_cast<wchar_t>(endian_swap(static_cast<wchar_selector<sizeof(wchar_t)>::type>(data[i])));
}
#endif
PUGI__NS_END

PUGI__NS_BEGIN
enum chartype_t
{
	ct_parse_pcdata = 1, // \0, &, \r, <
	ct_parse_attr = 2, // \0, &, \r, ', "
	ct_parse_attr_ws = 4, // \0, &, \r, ', ", \n, tab
	ct_space = 8, // \r, \n, space, tab
	ct_parse_cdata = 16, // \0, ], >, \r
	ct_parse_comment = 32, // \0, -, >, \r
	ct_symbol = 64, // Any symbol > 127, a-z, A-Z, 0-9, _, :, -, .
	ct_start_symbol = 128 // Any symbol > 127, a-z, A-Z, _, :
};

static const unsigned char chartype_table[256] = {
	55,	 0,	  0,   0,	0,	 0,	  0,   0,	0,	 12,  12,  0,	0,	 63,  0,   0, // 0-15
	0,	 0,	  0,   0,	0,	 0,	  0,   0,	0,	 0,	  0,   0,	0,	 0,	  0,   0, // 16-31
	8,	 0,	  6,   0,	0,	 0,	  7,   6,	0,	 0,	  0,   0,	0,	 96,  64,  0, // 32-47
	64,	 64,  64,  64,	64,	 64,  64,  64,	64,	 64,  192, 0,	1,	 0,	  48,  0, // 48-63
	0,	 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, // 64-79
	192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 0,	0,	 16,  0,   192, // 80-95
	0,	 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, // 96-111
	192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 0,	0,	 0,	  0,   0, // 112-127

	192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, // 128+
	192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
	192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
	192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
	192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
	192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192};

enum chartypex_t
{
	ctx_special_pcdata = 1, // Any symbol >= 0 and < 32 (except \t, \r, \n), &, <, >
	ctx_special_attr = 2, // Any symbol >= 0 and < 32 (except \t), &, <, >, "
	ctx_start_symbol = 4, // Any symbol > 127, a-z, A-Z, _
	ctx_digit = 8, // 0-9
	ctx_symbol = 16 // Any symbol > 127, a-z, A-Z, 0-9, _, -, .
};

static const unsigned char chartypex_table[256] = {
	3,	3,	3,	3,	3,	3,	3,	3,	3,	0,	2,	3,	3,	2,	3,	3, // 0-15
	3,	3,	3,	3,	3,	3,	3,	3,	3,	3,	3,	3,	3,	3,	3,	3, // 16-31
	0,	0,	2,	0,	0,	0,	3,	0,	0,	0,	0,	0,	0,	16, 16, 0, // 32-47
	24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0,	0,	3,	0,	3,	0, // 48-63

	0,	20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, // 64-79
	20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 0,	0,	0,	0,	20, // 80-95
	0,	20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, // 96-111
	20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 0,	0,	0,	0,	0, // 112-127

	20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, // 128+
	20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
	20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
	20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
	20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20};

#ifdef PUGIXML_WCHAR_MODE
	#define PUGI__IS_CHARTYPE_IMPL(c, ct, table) \
		((static_cast<unsigned int>(c) < 128 ? table[static_cast<unsigned int>(c)] : table[128]) & (ct))
#else
	#define PUGI__IS_CHARTYPE_IMPL(c, ct, table) (table[static_cast<unsigned char>(c)] & (ct))
#endif

#define PUGI__IS_CHARTYPE(c, ct) PUGI__IS_CHARTYPE_IMPL(c, ct, chartype_table)
#define PUGI__IS_CHARTYPEX(c, ct) PUGI__IS_CHARTYPE_IMPL(c, ct, chartypex_table)

PUGI__FN bool is_little_endian()
{
	unsigned int ui = 1;

	return *reinterpret_cast<unsigned char*>(&ui) == 1;
}

PUGI__FN xml_encoding get_wchar_encoding()
{
	PUGI__STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);

	if (sizeof(wchar_t) == 2)
		return is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
	else
		return is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
}

PUGI__FN xml_encoding guess_buffer_encoding(uint8_t d0, uint8_t d1, uint8_t d2, uint8_t d3)
{
	// look for BOM in first few bytes
	if (d0 == 0 && d1 == 0 && d2 == 0xfe && d3 == 0xff)
		return encoding_utf32_be;
	if (d0 == 0xff && d1 == 0xfe && d2 == 0 && d3 == 0)
		return encoding_utf32_le;
	if (d0 == 0xfe && d1 == 0xff)
		return encoding_utf16_be;
	if (d0 == 0xff && d1 == 0xfe)
		return encoding_utf16_le;
	if (d0 == 0xef && d1 == 0xbb && d2 == 0xbf)
		return encoding_utf8;

	// look for <, <? or <?xm in various encodings
	if (d0 == 0 && d1 == 0 && d2 == 0 && d3 == 0x3c)
		return encoding_utf32_be;
	if (d0 == 0x3c && d1 == 0 && d2 == 0 && d3 == 0)
		return encoding_utf32_le;
	if (d0 == 0 && d1 == 0x3c && d2 == 0 && d3 == 0x3f)
		return encoding_utf16_be;
	if (d0 == 0x3c && d1 == 0 && d2 == 0x3f && d3 == 0)
		return encoding_utf16_le;
	if (d0 == 0x3c && d1 == 0x3f && d2 == 0x78 && d3 == 0x6d)
		return encoding_utf8;

	// look for utf16 < followed by node name (this may fail, but is better than utf8 since it's zero terminated so early)
	if (d0 == 0 && d1 == 0x3c)
		return encoding_utf16_be;
	if (d0 == 0x3c && d1 == 0)
		return encoding_utf16_le;

	// no known BOM detected, assume utf8
	return encoding_utf8;
}

PUGI__FN xml_encoding get_buffer_encoding(xml_encoding encoding, const void* contents, size_t size)
{
	// replace wchar encoding with utf implementation
	if (encoding == encoding_wchar)
		return get_wchar_encoding();

	// replace utf16 encoding with utf16 with specific endianness
	if (encoding == encoding_utf16)
		return is_little_endian() ? encoding_utf16_le : encoding_utf16_be;

	// replace utf32 encoding with utf32 with specific endianness
	if (encoding == encoding_utf32)
		return is_little_endian() ? encoding_utf32_le : encoding_utf32_be;

	// only do autodetection if no explicit encoding is requested
	if (encoding != encoding_auto)
		return encoding;

	// skip encoding autodetection if input buffer is too small
	if (size < 4)
		return encoding_utf8;

	// try to guess encoding (based on XML specification, Appendix F.1)
	const uint8_t* data = static_cast<const uint8_t*>(contents);

	PUGI__DMC_VOLATILE uint8_t d0 = data[0], d1 = data[1], d2 = data[2], d3 = data[3];

	return guess_buffer_encoding(d0, d1, d2, d3);
}

PUGI__FN bool get_mutable_buffer(
	char_t*& out_buffer,
	size_t& out_length,
	const void* contents,
	size_t size,
	bool is_mutable)
{
	if (is_mutable)
	{
		out_buffer = static_cast<char_t*>(const_cast<void*>(contents));
	}
	else
	{
		void* buffer = xml_memory::allocate(size > 0 ? size : 1);
		if (!buffer)
			return false;

		memcpy(buffer, contents, size);

		out_buffer = static_cast<char_t*>(buffer);
	}

	out_length = size / sizeof(char_t);

	return true;
}

#ifdef PUGIXML_WCHAR_MODE
PUGI__FN bool need_endian_swap_utf(xml_encoding le, xml_encoding re)
{
	return (le == encoding_utf16_be && re == encoding_utf16_le) ||
		(le == encoding_utf16_le && re == encoding_utf16_be) || (le == encoding_utf32_be && re == encoding_utf32_le) ||
		(le == encoding_utf32_le && re == encoding_utf32_be);
}

PUGI__FN bool convert_buffer_endian_swap(
	char_t*& out_buffer,
	size_t& out_length,
	const void* contents,
	size_t size,
	bool is_mutable)
{
	const char_t* data = static_cast<const char_t*>(contents);

	if (is_mutable)
	{
		out_buffer = const_cast<char_t*>(data);
	}
	else
	{
		out_buffer = static_cast<char_t*>(xml_memory::allocate(size > 0 ? size : 1));
		if (!out_buffer)
			return false;
	}

	out_length = size / sizeof(char_t);

	convert_wchar_endian_swap(out_buffer, data, out_length);

	return true;
}

PUGI__FN bool convert_buffer_utf8(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size)
{
	const uint8_t* data = static_cast<const uint8_t*>(contents);

	// first pass: get length in wchar_t units
	out_length = utf_decoder<wchar_counter>::decode_utf8_block(data, size, 0);

	// allocate buffer of suitable length
	out_buffer = static_cast<char_t*>(xml_memory::allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
	if (!out_buffer)
		return false;

	// second pass: convert utf8 input to wchar_t
	wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
	wchar_writer::value_type out_end = utf_decoder<wchar_writer>::decode_utf8_block(data, size, out_begin);

	assert(out_end == out_begin + out_length);
	(void)!out_end;

	return true;
}

template<typename opt_swap>
PUGI__FN bool convert_buffer_utf16(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
{
	const uint16_t* data = static_cast<const uint16_t*>(contents);
	size_t length = size / sizeof(uint16_t);

	// first pass: get length in wchar_t units
	out_length = utf_decoder<wchar_counter, opt_swap>::decode_utf16_block(data, length, 0);

	// allocate buffer of suitable length
	out_buffer = static_cast<char_t*>(xml_memory::allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
	if (!out_buffer)
		return false;

	// second pass: convert utf16 input to wchar_t
	wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
	wchar_writer::value_type out_end = utf_decoder<wchar_writer, opt_swap>::decode_utf16_block(data, length, out_begin);

	assert(out_end == out_begin + out_length);
	(void)!out_end;

	return true;
}

template<typename opt_swap>
PUGI__FN bool convert_buffer_utf32(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
{
	const uint32_t* data = static_cast<const uint32_t*>(contents);
	size_t length = size / sizeof(uint32_t);

	// first pass: get length in wchar_t units
	out_length = utf_decoder<wchar_counter, opt_swap>::decode_utf32_block(data, length, 0);

	// allocate buffer of suitable length
	out_buffer = static_cast<char_t*>(xml_memory::allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
	if (!out_buffer)
		return false;

	// second pass: convert utf32 input to wchar_t
	wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
	wchar_writer::value_type out_end = utf_decoder<wchar_writer, opt_swap>::decode_utf32_block(data, length, out_begin);

	assert(out_end == out_begin + out_length);
	(void)!out_end;

	return true;
}

PUGI__FN bool convert_buffer_latin1(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size)
{
	const uint8_t* data = static_cast<const uint8_t*>(contents);

	// get length in wchar_t units
	out_length = size;

	// allocate buffer of suitable length
	out_buffer = static_cast<char_t*>(xml_memory::allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
	if (!out_buffer)
		return false;

	// convert latin1 input to wchar_t
	wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
	wchar_writer::value_type out_end = utf_decoder<wchar_writer>::decode_latin1_block(data, size, out_begin);

	assert(out_end == out_begin + out_length);
	(void)!out_end;

	return true;
}

PUGI__FN bool convert_buffer(
	char_t*& out_buffer,
	size_t& out_length,
	xml_encoding encoding,
	const void* contents,
	size_t size,
	bool is_mutable)
{
	// get native encoding
	xml_encoding wchar_encoding = get_wchar_encoding();

	// fast path: no conversion required
	if (encoding == wchar_encoding)
		return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);

	// only endian-swapping is required
	if (need_endian_swap_utf(encoding, wchar_encoding))
		return convert_buffer_endian_swap(out_buffer, out_length, contents, size, is_mutable);

	// source encoding is utf8
	if (encoding == encoding_utf8)
		return convert_buffer_utf8(out_buffer, out_length, contents, size);

	// source encoding is utf16
	if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
	{
		xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;

		return (native_encoding == encoding)
			? convert_buffer_utf16(out_buffer, out_length, contents, size, opt_false())
			: convert_buffer_utf16(out_buffer, out_length, contents, size, opt_true());
	}

	// source encoding is utf32
	if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
	{
		xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;

		return (native_encoding == encoding)
			? convert_buffer_utf32(out_buffer, out_length, contents, size, opt_false())
			: convert_buffer_utf32(out_buffer, out_length, contents, size, opt_true());
	}

	// source encoding is latin1
	if (encoding == encoding_latin1)
		return convert_buffer_latin1(out_buffer, out_length, contents, size);

	assert(!"Invalid encoding");
	return false;
}
#else
template<typename opt_swap>
PUGI__FN bool convert_buffer_utf16(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
{
	const uint16_t* data = static_cast<const uint16_t*>(contents);
	size_t length = size / sizeof(uint16_t);

	// first pass: get length in utf8 units
	out_length = utf_decoder<utf8_counter, opt_swap>::decode_utf16_block(data, length, 0);

	// allocate buffer of suitable length
	out_buffer = static_cast<char_t*>(xml_memory::allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
	if (!out_buffer)
		return false;

	// second pass: convert utf16 input to utf8
	uint8_t* out_begin = reinterpret_cast<uint8_t*>(out_buffer);
	uint8_t* out_end = utf_decoder<utf8_writer, opt_swap>::decode_utf16_block(data, length, out_begin);

	assert(out_end == out_begin + out_length);
	(void)!out_end;

	return true;
}

template<typename opt_swap>
PUGI__FN bool convert_buffer_utf32(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
{
	const uint32_t* data = static_cast<const uint32_t*>(contents);
	size_t length = size / sizeof(uint32_t);

	// first pass: get length in utf8 units
	out_length = utf_decoder<utf8_counter, opt_swap>::decode_utf32_block(data, length, 0);

	// allocate buffer of suitable length
	out_buffer = static_cast<char_t*>(xml_memory::allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
	if (!out_buffer)
		return false;

	// second pass: convert utf32 input to utf8
	uint8_t* out_begin = reinterpret_cast<uint8_t*>(out_buffer);
	uint8_t* out_end = utf_decoder<utf8_writer, opt_swap>::decode_utf32_block(data, length, out_begin);

	assert(out_end == out_begin + out_length);
	(void)!out_end;

	return true;
}

PUGI__FN size_t get_latin1_7bit_prefix_length(const uint8_t* data, size_t size)
{
	for (size_t i = 0; i < size; ++i)
		if (data[i] > 127)
			return i;

	return size;
}

PUGI__FN bool convert_buffer_latin1(
	char_t*& out_buffer,
	size_t& out_length,
	const void* contents,
	size_t size,
	bool is_mutable)
{
	const uint8_t* data = static_cast<const uint8_t*>(contents);

	// get size of prefix that does not need utf8 conversion
	size_t prefix_length = get_latin1_7bit_prefix_length(data, size);
	assert(prefix_length <= size);

	const uint8_t* postfix = data + prefix_length;
	size_t postfix_length = size - prefix_length;

	// if no conversion is needed, just return the original buffer
	if (postfix_length == 0)
		return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);

	// first pass: get length in utf8 units
	out_length = prefix_length + utf_decoder<utf8_counter>::decode_latin1_block(postfix, postfix_length, 0);

	// allocate buffer of suitable length
	out_buffer = static_cast<char_t*>(xml_memory::allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
	if (!out_buffer)
		return false;

	// second pass: convert latin1 input to utf8
	memcpy(out_buffer, data, prefix_length);

	uint8_t* out_begin = reinterpret_cast<uint8_t*>(out_buffer);
	uint8_t* out_end =
		utf_decoder<utf8_writer>::decode_latin1_block(postfix, postfix_length, out_begin + prefix_length);

	assert(out_end == out_begin + out_length);
	(void)!out_end;

	return true;
}

PUGI__FN bool convert_buffer(
	char_t*& out_buffer,
	size_t& out_length,
	xml_encoding encoding,
	const void* contents,
	size_t size,
	bool is_mutable)
{
	// fast path: no conversion required
	if (encoding == encoding_utf8)
		return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);

	// source encoding is utf16
	if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
	{
		xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;

		return (native_encoding == encoding)
			? convert_buffer_utf16(out_buffer, out_length, contents, size, opt_false())
			: convert_buffer_utf16(out_buffer, out_length, contents, size, opt_true());
	}

	// source encoding is utf32
	if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
	{
		xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;

		return (native_encoding == encoding)
			? convert_buffer_utf32(out_buffer, out_length, contents, size, opt_false())
			: convert_buffer_utf32(out_buffer, out_length, contents, size, opt_true());
	}

	// source encoding is latin1
	if (encoding == encoding_latin1)
		return convert_buffer_latin1(out_buffer, out_length, contents, size, is_mutable);

	assert(!"Invalid encoding");
	return false;
}
#endif

PUGI__FN size_t as_utf8_begin(const wchar_t* str, size_t length)
{
	// get length in utf8 characters
	return utf_decoder<utf8_counter>::decode_wchar_block(str, length, 0);
}

PUGI__FN void as_utf8_end(char* buffer, size_t size, const wchar_t* str, size_t length)
{
	// convert to utf8
	uint8_t* begin = reinterpret_cast<uint8_t*>(buffer);
	uint8_t* end = utf_decoder<utf8_writer>::decode_wchar_block(str, length, begin);

	assert(begin + size == end);
	(void)!end;

	// zero-terminate
	buffer[size] = 0;
}

#ifndef PUGIXML_NO_STL
PUGI__FN std::string as_utf8_impl(const wchar_t* str, size_t length)
{
	// first pass: get length in utf8 characters
	size_t size = as_utf8_begin(str, length);

	// allocate resulting string
	std::string result;
	result.resize(size);

	// second pass: convert to utf8
	if (size > 0)
		as_utf8_end(&result[0], size, str, length);

	return result;
}

PUGI__FN std::basic_string<wchar_t> as_wide_impl(const char* str, size_t size)
{
	const uint8_t* data = reinterpret_cast<const uint8_t*>(str);

	// first pass: get length in wchar_t units
	size_t length = utf_decoder<wchar_counter>::decode_utf8_block(data, size, 0);

	// allocate resulting string
	std::basic_string<wchar_t> result;
	result.resize(length);

	// second pass: convert to wchar_t
	if (length > 0)
	{
		wchar_writer::value_type begin = reinterpret_cast<wchar_writer::value_type>(&result[0]);
		wchar_writer::value_type end = utf_decoder<wchar_writer>::decode_utf8_block(data, size, begin);

		assert(begin + length == end);
		(void)!end;
	}

	return result;
}
#endif

inline bool strcpy_insitu_allow(size_t length, uintptr_t allocated, char_t* target)
{
	assert(target);
	size_t target_length = strlength(target);

	// always reuse document buffer memory if possible
	if (!allocated)
		return target_length >= length;

	// reuse heap memory if waste is not too great
	const size_t reuse_threshold = 32;

	return target_length >= length && (target_length < reuse_threshold || target_length - length < target_length / 2);
}

PUGI__FN bool strcpy_insitu(char_t*& dest, uintptr_t& header, uintptr_t header_mask, const char_t* source)
{
	size_t source_length = strlength(source);

	if (source_length == 0)
	{
		// empty string and null pointer are equivalent, so just deallocate old memory
		xml_allocator* alloc = reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask)->allocator;

		if (header & header_mask)
			alloc->deallocate_string(dest);

		// mark the string as not allocated
		dest = 0;
		header &= ~header_mask;

		return true;
	}
	else if (dest && strcpy_insitu_allow(source_length, header & header_mask, dest))
	{
		// we can reuse old buffer, so just copy the new data (including zero terminator)
		memcpy(dest, source, (source_length + 1) * sizeof(char_t));

		return true;
	}
	else
	{
		xml_allocator* alloc = reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask)->allocator;

		// allocate new buffer
		char_t* buf = alloc->allocate_string(source_length + 1);
		if (!buf)
			return false;

		// copy the string (including zero terminator)
		memcpy(buf, source, (source_length + 1) * sizeof(char_t));

		// deallocate old buffer (*after* the above to protect against overlapping memory and/or allocation failures)
		if (header & header_mask)
			alloc->deallocate_string(dest);

		// the string is now allocated, so set the flag
		dest = buf;
		header |= header_mask;

		return true;
	}
}

struct gap
{
	char_t* end;
	size_t size;

	gap() :
		end(0),
		size(0)
	{
	}

	// Push new gap, move s count bytes further (skipping the gap).
	// Collapse previous gap.
	void push(char_t*& s, size_t count)
	{
		if (end) // there was a gap already; collapse it
		{
			// Move [old_gap_end, new_gap_start) to [old_gap_start, ...)
			assert(s >= end);
			memmove(end - size, end, reinterpret_cast<char*>(s) - reinterpret_cast<char*>(end));
		}

		s += count; // end of current gap

		// "merge" two gaps
		end = s;
		size += count;
	}

	// Collapse all gaps, return past-the-end pointer
	char_t* flush(char_t* s)
	{
		if (end)
		{
			// Move [old_gap_end, current_pos) to [old_gap_start, ...)
			assert(s >= end);
			memmove(end - size, end, reinterpret_cast<char*>(s) - reinterpret_cast<char*>(end));

			return s - size;
		}
		else
			return s;
	}
};

PUGI__FN char_t* strconv_escape(char_t* s, gap& g)
{
	char_t* stre = s + 1;

	switch (*stre)
	{
	case '#': // &#...
	{
		unsigned int ucsc = 0;

		if (stre[1] == 'x') // &#x... (hex code)
		{
			stre += 2;

			char_t ch = *stre;

			if (ch == ';')
				return stre;

			for (;;)
			{
				if (static_cast<unsigned int>(ch - '0') <= 9)
					ucsc = 16 * ucsc + (ch - '0');
				else if (static_cast<unsigned int>((ch | ' ') - 'a') <= 5)
					ucsc = 16 * ucsc + ((ch | ' ') - 'a' + 10);
				else if (ch == ';')
					break;
				else // cancel
					return stre;

				ch = *++stre;
			}

			++stre;
		}
		else // &#... (dec code)
		{
			char_t ch = *++stre;

			if (ch == ';')
				return stre;

			for (;;)
			{
				if (static_cast<unsigned int>(ch - '0') <= 9)
					ucsc = 10 * ucsc + (ch - '0');
				else if (ch == ';')
					break;
				else // cancel
					return stre;

				ch = *++stre;
			}

			++stre;
		}

#ifdef PUGIXML_WCHAR_MODE
		s = reinterpret_cast<char_t*>(wchar_writer::any(reinterpret_cast<wchar_writer::value_type>(s), ucsc));
#else
		s = reinterpret_cast<char_t*>(utf8_writer::any(reinterpret_cast<uint8_t*>(s), ucsc));
#endif

		g.push(s, stre - s);
		return stre;
	}

	case 'a': // &a
	{
		++stre;

		if (*stre == 'm') // &am
		{
			if (*++stre == 'p' && *++stre == ';') // &amp;
			{
				*s++ = '&';
				++stre;

				g.push(s, stre - s);
				return stre;
			}
		}
		else if (*stre == 'p') // &ap
		{
			if (*++stre == 'o' && *++stre == 's' && *++stre == ';') // &apos;
			{
				*s++ = '\'';
				++stre;

				g.push(s, stre - s);
				return stre;
			}
		}
		break;
	}

	case 'g': // &g
	{
		if (*++stre == 't' && *++stre == ';') // &gt;
		{
			*s++ = '>';
			++stre;

			g.push(s, stre - s);
			return stre;
		}
		break;
	}

	case 'l': // &l
	{
		if (*++stre == 't' && *++stre == ';') // &lt;
		{
			*s++ = '<';
			++stre;

			g.push(s, stre - s);
			return stre;
		}
		break;
	}

	case 'q': // &q
	{
		if (*++stre == 'u' && *++stre == 'o' && *++stre == 't' && *++stre == ';') // &quot;
		{
			*s++ = '"';
			++stre;

			g.push(s, stre - s);
			return stre;
		}
		break;
	}

	default: break;
	}

	return stre;
}

// Utility macro for last character handling
#define ENDSWITH(c, e) ((c) == (e) || ((c) == 0 && endch == (e)))

PUGI__FN char_t* strconv_comment(char_t* s, char_t endch)
{
	gap g;

	while (true)
	{
		while (!PUGI__IS_CHARTYPE(*s, ct_parse_comment))
			++s;

		if (*s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
		{
			*s++ = '\n'; // replace first one with 0x0a

			if (*s == '\n')
				g.push(s, 1);
		}
		else if (s[0] == '-' && s[1] == '-' && ENDSWITH(s[2], '>')) // comment ends here
		{
			*g.flush(s) = 0;

			return s + (s[2] == '>' ? 3 : 2);
		}
		else if (*s == 0)
		{
			return 0;
		}
		else
			++s;
	}
}

PUGI__FN char_t* strconv_cdata(char_t* s, char_t endch)
{
	gap g;

	while (true)
	{
		while (!PUGI__IS_CHARTYPE(*s, ct_parse_cdata))
			++s;

		if (*s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
		{
			*s++ = '\n'; // replace first one with 0x0a

			if (*s == '\n')
				g.push(s, 1);
		}
		else if (s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>')) // CDATA ends here
		{
			*g.flush(s) = 0;

			return s + 1;
		}
		else if (*s == 0)
		{
			return 0;
		}
		else
			++s;
	}
}

typedef char_t* (*strconv_pcdata_t)(char_t*);

template<typename opt_eol, typename opt_escape>
struct strconv_pcdata_impl
{
	static char_t* parse(char_t* s)
	{
		gap g;

		while (true)
		{
			while (!PUGI__IS_CHARTYPE(*s, ct_parse_pcdata))
				++s;

			if (*s == '<') // PCDATA ends here
			{
				*g.flush(s) = 0;

				return s + 1;
			}
			else if (opt_eol::value && *s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
			{
				*s++ = '\n'; // replace first one with 0x0a

				if (*s == '\n')
					g.push(s, 1);
			}
			else if (opt_escape::value && *s == '&')
			{
				s = strconv_escape(s, g);
			}
			else if (*s == 0)
			{
				return s;
			}
			else
				++s;
		}
	}
};

PUGI__FN strconv_pcdata_t get_strconv_pcdata(unsigned int optmask)
{
	PUGI__STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20);

	switch ((optmask >> 4) & 3) // get bitmask for flags (eol escapes)
	{
	case 0: return strconv_pcdata_impl<opt_false, opt_false>::parse;
	case 1: return strconv_pcdata_impl<opt_false, opt_true>::parse;
	case 2: return strconv_pcdata_impl<opt_true, opt_false>::parse;
	case 3: return strconv_pcdata_impl<opt_true, opt_true>::parse;
	default: return 0; // should not get here
	}
}

typedef char_t* (*strconv_attribute_t)(char_t*, char_t);

template<typename opt_escape>
struct strconv_attribute_impl
{
	static char_t* parse_wnorm(char_t* s, char_t end_quote)
	{
		gap g;

		// trim leading whitespaces
		if (PUGI__IS_CHARTYPE(*s, ct_space))
		{
			char_t* str = s;

			do
				++str;
			while (PUGI__IS_CHARTYPE(*str, ct_space));

			g.push(s, str - s);
		}

		while (true)
		{
			while (!PUGI__IS_CHARTYPE(*s, ct_parse_attr_ws | ct_space))
				++s;

			if (*s == end_quote)
			{
				char_t* str = g.flush(s);

				do
					*str-- = 0;
				while (PUGI__IS_CHARTYPE(*str, ct_space));

				return s + 1;
			}
			else if (PUGI__IS_CHARTYPE(*s, ct_space))
			{
				*s++ = ' ';

				if (PUGI__IS_CHARTYPE(*s, ct_space))
				{
					char_t* str = s + 1;
					while (PUGI__IS_CHARTYPE(*str, ct_space))
						++str;

					g.push(s, str - s);
				}
			}
			else if (opt_escape::value && *s == '&')
			{
				s = strconv_escape(s, g);
			}
			else if (!*s)
			{
				return 0;
			}
			else
				++s;
		}
	}

	static char_t* parse_wconv(char_t* s, char_t end_quote)
	{
		gap g;

		while (true)
		{
			while (!PUGI__IS_CHARTYPE(*s, ct_parse_attr_ws))
				++s;

			if (*s == end_quote)
			{
				*g.flush(s) = 0;

				return s + 1;
			}
			else if (PUGI__IS_CHARTYPE(*s, ct_space))
			{
				if (*s == '\r')
				{
					*s++ = ' ';

					if (*s == '\n')
						g.push(s, 1);
				}
				else
					*s++ = ' ';
			}
			else if (opt_escape::value && *s == '&')
			{
				s = strconv_escape(s, g);
			}
			else if (!*s)
			{
				return 0;
			}
			else
				++s;
		}
	}

	static char_t* parse_eol(char_t* s, char_t end_quote)
	{
		gap g;

		while (true)
		{
			while (!PUGI__IS_CHARTYPE(*s, ct_parse_attr))
				++s;

			if (*s == end_quote)
			{
				*g.flush(s) = 0;

				return s + 1;
			}
			else if (*s == '\r')
			{
				*s++ = '\n';

				if (*s == '\n')
					g.push(s, 1);
			}
			else if (opt_escape::value && *s == '&')
			{
				s = strconv_escape(s, g);
			}
			else if (!*s)
			{
				return 0;
			}
			else
				++s;
		}
	}

	static char_t* parse_simple(char_t* s, char_t end_quote)
	{
		gap g;

		while (true)
		{
			while (!PUGI__IS_CHARTYPE(*s, ct_parse_attr))
				++s;

			if (*s == end_quote)
			{
				*g.flush(s) = 0;

				return s + 1;
			}
			else if (opt_escape::value && *s == '&')
			{
				s = strconv_escape(s, g);
			}
			else if (!*s)
			{
				return 0;
			}
			else
				++s;
		}
	}
};

PUGI__FN strconv_attribute_t get_strconv_attribute(unsigned int optmask)
{
	PUGI__STATIC_ASSERT(
		parse_escapes == 0x10 && parse_eol == 0x20 && parse_wconv_attribute == 0x40 && parse_wnorm_attribute == 0x80);

	switch ((optmask >> 4) & 15) // get bitmask for flags (wconv wnorm eol escapes)
	{
	case 0: return strconv_attribute_impl<opt_false>::parse_simple;
	case 1: return strconv_attribute_impl<opt_true>::parse_simple;
	case 2: return strconv_attribute_impl<opt_false>::parse_eol;
	case 3: return strconv_attribute_impl<opt_true>::parse_eol;
	case 4: return strconv_attribute_impl<opt_false>::parse_wconv;
	case 5: return strconv_attribute_impl<opt_true>::parse_wconv;
	case 6: return strconv_attribute_impl<opt_false>::parse_wconv;
	case 7: return strconv_attribute_impl<opt_true>::parse_wconv;
	case 8: return strconv_attribute_impl<opt_false>::parse_wnorm;
	case 9: return strconv_attribute_impl<opt_true>::parse_wnorm;
	case 10: return strconv_attribute_impl<opt_false>::parse_wnorm;
	case 11: return strconv_attribute_impl<opt_true>::parse_wnorm;
	case 12: return strconv_attribute_impl<opt_false>::parse_wnorm;
	case 13: return strconv_attribute_impl<opt_true>::parse_wnorm;
	case 14: return strconv_attribute_impl<opt_false>::parse_wnorm;
	case 15: return strconv_attribute_impl<opt_true>::parse_wnorm;
	default: return 0; // should not get here
	}
}

inline xml_parse_result make_parse_result(xml_parse_status status, ptrdiff_t offset = 0)
{
	xml_parse_result result;
	result.status = status;
	result.offset = offset;

	return result;
}

struct xml_parser
{
	xml_allocator alloc;
	char_t* error_offset;
	xml_parse_status error_status;

// Parser utilities.
#define PUGI__SKIPWS() \
	{ \
		while (PUGI__IS_CHARTYPE(*s, ct_space)) \
			++s; \
	}
#define PUGI__OPTSET(OPT) (optmsk & (OPT))
#define PUGI__PUSHNODE(TYPE) \
	{ \
		cursor = append_node(cursor, alloc, TYPE); \
		if (!cursor) \
			PUGI__THROW_ERROR(status_out_of_memory, s); \
	}
#define PUGI__POPNODE() \
	{ \
		cursor = cursor->parent; \
	}
#define PUGI__SCANFOR(X) \
	{ \
		while (*s != 0 && !(X)) \
			++s; \
	}
#define PUGI__SCANWHILE(X) \
	{ \
		while ((X)) \
			++s; \
	}
#define PUGI__ENDSEG() \
	{ \
		ch = *s; \
		*s = 0; \
		++s; \
	}
#define PUGI__THROW_ERROR(err, m) return error_offset = m, error_status = err, static_cast<char_t*>(0)
#define PUGI__CHECK_ERROR(err, m) \
	{ \
		if (*s == 0) \
			PUGI__THROW_ERROR(err, m); \
	}

	xml_parser(const xml_allocator& alloc_) :
		alloc(alloc_),
		error_offset(0),
		error_status(status_ok)
	{
	}

	// DOCTYPE consists of nested sections of the following possible types:
	// <!-- ... -->, <? ... ?>, "...", '...'
	// <![...]]>
	// <!...>
	// First group can not contain nested groups
	// Second group can contain nested groups of the same type
	// Third group can contain all other groups
	char_t* parse_doctype_primitive(char_t* s)
	{
		if (*s == '"' || *s == '\'')
		{
			// quoted string
			char_t ch = *s++;
			PUGI__SCANFOR(*s == ch);
			if (!*s)
				PUGI__THROW_ERROR(status_bad_doctype, s);

			s++;
		}
		else if (s[0] == '<' && s[1] == '?')
		{
			// <? ... ?>
			s += 2;
			PUGI__SCANFOR(s[0] == '?' && s[1] == '>'); // no need for ENDSWITH because ?> can't terminate proper doctype
			if (!*s)
				PUGI__THROW_ERROR(status_bad_doctype, s);

			s += 2;
		}
		else if (s[0] == '<' && s[1] == '!' && s[2] == '-' && s[3] == '-')
		{
			s += 4;
			PUGI__SCANFOR(
				s[0] == '-' && s[1] == '-' &&
				s[2] == '>'); // no need for ENDSWITH because --> can't terminate proper doctype
			if (!*s)
				PUGI__THROW_ERROR(status_bad_doctype, s);

			s += 4;
		}
		else
			PUGI__THROW_ERROR(status_bad_doctype, s);

		return s;
	}

	char_t* parse_doctype_ignore(char_t* s)
	{
		assert(s[0] == '<' && s[1] == '!' && s[2] == '[');
		s++;

		while (*s)
		{
			if (s[0] == '<' && s[1] == '!' && s[2] == '[')
			{
				// nested ignore section
				s = parse_doctype_ignore(s);
				if (!s)
					return s;
			}
			else if (s[0] == ']' && s[1] == ']' && s[2] == '>')
			{
				// ignore section end
				s += 3;

				return s;
			}
			else
				s++;
		}

		PUGI__THROW_ERROR(status_bad_doctype, s);
	}

	char_t* parse_doctype_group(char_t* s, char_t endch, bool toplevel)
	{
		assert(s[0] == '<' && s[1] == '!');
		s++;

		while (*s)
		{
			if (s[0] == '<' && s[1] == '!' && s[2] != '-')
			{
				if (s[2] == '[')
				{
					// ignore
					s = parse_doctype_ignore(s);
					if (!s)
						return s;
				}
				else
				{
					// some control group
					s = parse_doctype_group(s, endch, false);
					if (!s)
						return s;
				}
			}
			else if (s[0] == '<' || s[0] == '"' || s[0] == '\'')
			{
				// unknown tag (forbidden), or some primitive group
				s = parse_doctype_primitive(s);
				if (!s)
					return s;
			}
			else if (*s == '>')
			{
				s++;

				return s;
			}
			else
				s++;
		}

		if (!toplevel || endch != '>')
			PUGI__THROW_ERROR(status_bad_doctype, s);

		return s;
	}

	char_t* parse_exclamation(char_t* s, xml_node_struct* cursor, unsigned int optmsk, char_t endch)
	{
		// parse node contents, starting with exclamation mark
		++s;

		if (*s == '-') // '<!-...'
		{
			++s;

			if (*s == '-') // '<!--...'
			{
				++s;

				if (PUGI__OPTSET(parse_comments))
				{
					PUGI__PUSHNODE(node_comment); // Append a new node on the tree.
					cursor->value = s; // Save the offset.
				}

				if (PUGI__OPTSET(parse_eol) && PUGI__OPTSET(parse_comments))
				{
					s = strconv_comment(s, endch);

					if (!s)
						PUGI__THROW_ERROR(status_bad_comment, cursor->value);
				}
				else
				{
					// Scan for terminating '-->'.
					PUGI__SCANFOR(s[0] == '-' && s[1] == '-' && ENDSWITH(s[2], '>'));
					PUGI__CHECK_ERROR(status_bad_comment, s);

					if (PUGI__OPTSET(parse_comments))
						*s = 0; // Zero-terminate this segment at the first terminating '-'.

					s += (s[2] == '>' ? 3 : 2); // Step over the '\0->'.
				}
			}
			else
				PUGI__THROW_ERROR(status_bad_comment, s);
		}
		else if (*s == '[')
		{
			// '<![CDATA[...'
			if (*++s == 'C' && *++s == 'D' && *++s == 'A' && *++s == 'T' && *++s == 'A' && *++s == '[')
			{
				++s;

				if (PUGI__OPTSET(parse_cdata))
				{
					PUGI__PUSHNODE(node_cdata); // Append a new node on the tree.
					cursor->value = s; // Save the offset.

					if (PUGI__OPTSET(parse_eol))
					{
						s = strconv_cdata(s, endch);

						if (!s)
							PUGI__THROW_ERROR(status_bad_cdata, cursor->value);
					}
					else
					{
						// Scan for terminating ']]>'.
						PUGI__SCANFOR(s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>'));
						PUGI__CHECK_ERROR(status_bad_cdata, s);

						*s++ = 0; // Zero-terminate this segment.
					}
				}
				else // Flagged for discard, but we still have to scan for the terminator.
				{
					// Scan for terminating ']]>'.
					PUGI__SCANFOR(s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>'));
					PUGI__CHECK_ERROR(status_bad_cdata, s);

					++s;
				}

				s += (s[1] == '>' ? 2 : 1); // Step over the last ']>'.
			}
			else
				PUGI__THROW_ERROR(status_bad_cdata, s);
		}
		else if (
			s[0] == 'D' && s[1] == 'O' && s[2] == 'C' && s[3] == 'T' && s[4] == 'Y' && s[5] == 'P' &&
			ENDSWITH(s[6], 'E'))
		{
			s -= 2;

			if (cursor->parent)
				PUGI__THROW_ERROR(status_bad_doctype, s);

			char_t* mark = s + 9;

			s = parse_doctype_group(s, endch, true);
			if (!s)
				return s;

			if (PUGI__OPTSET(parse_doctype))
			{
				while (PUGI__IS_CHARTYPE(*mark, ct_space))
					++mark;

				PUGI__PUSHNODE(node_doctype);

				cursor->value = mark;

				assert((s[0] == 0 && endch == '>') || s[-1] == '>');
				s[*s == 0 ? 0 : -1] = 0;

				PUGI__POPNODE();
			}
		}
		else if (*s == 0 && endch == '-')
			PUGI__THROW_ERROR(status_bad_comment, s);
		else if (*s == 0 && endch == '[')
			PUGI__THROW_ERROR(status_bad_cdata, s);
		else
			PUGI__THROW_ERROR(status_unrecognized_tag, s);

		return s;
	}

	char_t* parse_question(char_t* s, xml_node_struct*& ref_cursor, unsigned int optmsk, char_t endch)
	{
		// load into registers
		xml_node_struct* cursor = ref_cursor;
		char_t ch = 0;

		// parse node contents, starting with question mark
		++s;

		// read PI target
		char_t* target = s;

		if (!PUGI__IS_CHARTYPE(*s, ct_start_symbol))
			PUGI__THROW_ERROR(status_bad_pi, s);

		PUGI__SCANWHILE(PUGI__IS_CHARTYPE(*s, ct_symbol));
		PUGI__CHECK_ERROR(status_bad_pi, s);

		// determine node type; stricmp / strcasecmp is not portable
		bool declaration =
			(target[0] | ' ') == 'x' && (target[1] | ' ') == 'm' && (target[2] | ' ') == 'l' && target + 3 == s;

		if (declaration ? PUGI__OPTSET(parse_declaration) : PUGI__OPTSET(parse_pi))
		{
			if (declaration)
			{
				// disallow non top-level declarations
				if (cursor->parent)
					PUGI__THROW_ERROR(status_bad_pi, s);

				PUGI__PUSHNODE(node_declaration);
			}
			else
			{
				PUGI__PUSHNODE(node_pi);
			}

			cursor->name = target;

			PUGI__ENDSEG();

			// parse value/attributes
			if (ch == '?')
			{
				// empty node
				if (!ENDSWITH(*s, '>'))
					PUGI__THROW_ERROR(status_bad_pi, s);
				s += (*s == '>');

				PUGI__POPNODE();
			}
			else if (PUGI__IS_CHARTYPE(ch, ct_space))
			{
				PUGI__SKIPWS();

				// scan for tag end
				char_t* value = s;

				PUGI__SCANFOR(s[0] == '?' && ENDSWITH(s[1], '>'));
				PUGI__CHECK_ERROR(status_bad_pi, s);

				if (declaration)
				{
					// replace ending ? with / so that 'element' terminates properly
					*s = '/';

					// we exit from this function with cursor at node_declaration, which is a signal to parse() to go to LOC_ATTRIBUTES
					s = value;
				}
				else
				{
					// store value and step over >
					cursor->value = value;
					PUGI__POPNODE();

					PUGI__ENDSEG();

					s += (*s == '>');
				}
			}
			else
				PUGI__THROW_ERROR(status_bad_pi, s);
		}
		else
		{
			// scan for tag end
			PUGI__SCANFOR(s[0] == '?' && ENDSWITH(s[1], '>'));
			PUGI__CHECK_ERROR(status_bad_pi, s);

			s += (s[1] == '>' ? 2 : 1);
		}

		// store from registers
		ref_cursor = cursor;

		return s;
	}

	char_t* parse(char_t* s, xml_node_struct* xmldoc, unsigned int optmsk, char_t endch)
	{
		strconv_attribute_t strconv_attribute = get_strconv_attribute(optmsk);
		strconv_pcdata_t strconv_pcdata = get_strconv_pcdata(optmsk);

		char_t ch = 0;
		xml_node_struct* cursor = xmldoc;
		char_t* mark = s;

		while (*s != 0)
		{
			if (*s == '<')
			{
				++s;

			LOC_TAG:
				if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) // '<#...'
				{
					PUGI__PUSHNODE(node_element); // Append a new node to the tree.

					cursor->name = s;

					PUGI__SCANWHILE(PUGI__IS_CHARTYPE(*s, ct_symbol)); // Scan for a terminator.
					PUGI__ENDSEG(); // Save char in 'ch', terminate & step over.

					if (ch == '>')
					{
						// end of tag
					}
					else if (PUGI__IS_CHARTYPE(ch, ct_space))
					{
					LOC_ATTRIBUTES:
						while (true)
						{
							PUGI__SKIPWS(); // Eat any whitespace.

							if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) // <... #...
							{
								xml_attribute_struct* a =
									append_attribute_ll(cursor, alloc); // Make space for this attribute.
								if (!a)
									PUGI__THROW_ERROR(status_out_of_memory, s);

								a->name = s; // Save the offset.

								PUGI__SCANWHILE(PUGI__IS_CHARTYPE(*s, ct_symbol)); // Scan for a terminator.
								PUGI__CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance

								PUGI__ENDSEG(); // Save char in 'ch', terminate & step over.
								PUGI__CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance

								if (PUGI__IS_CHARTYPE(ch, ct_space))
								{
									PUGI__SKIPWS(); // Eat any whitespace.
									PUGI__CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance

									ch = *s;
									++s;
								}

								if (ch == '=') // '<... #=...'
								{
									PUGI__SKIPWS(); // Eat any whitespace.

									if (*s == '"' || *s == '\'') // '<... #="...'
									{
										ch = *s; // Save quote char to avoid breaking on "''" -or- '""'.
										++s; // Step over the quote.
										a->value = s; // Save the offset.

										s = strconv_attribute(s, ch);

										if (!s)
											PUGI__THROW_ERROR(status_bad_attribute, a->value);

										// After this line the loop continues from the start;
										// Whitespaces, / and > are ok, symbols and EOF are wrong,
										// everything else will be detected
										if (PUGI__IS_CHARTYPE(*s, ct_start_symbol))
											PUGI__THROW_ERROR(status_bad_attribute, s);
									}
									else
										PUGI__THROW_ERROR(status_bad_attribute, s);
								}
								else
									PUGI__THROW_ERROR(status_bad_attribute, s);
							}
							else if (*s == '/')
							{
								++s;

								if (*s == '>')
								{
									PUGI__POPNODE();
									s++;
									break;
								}
								else if (*s == 0 && endch == '>')
								{
									PUGI__POPNODE();
									break;
								}
								else
									PUGI__THROW_ERROR(status_bad_start_element, s);
							}
							else if (*s == '>')
							{
								++s;

								break;
							}
							else if (*s == 0 && endch == '>')
							{
								break;
							}
							else
								PUGI__THROW_ERROR(status_bad_start_element, s);
						}

						// !!!
					}
					else if (ch == '/') // '<#.../'
					{
						if (!ENDSWITH(*s, '>'))
							PUGI__THROW_ERROR(status_bad_start_element, s);

						PUGI__POPNODE(); // Pop.

						s += (*s == '>');
					}
					else if (ch == 0)
					{
						// we stepped over null terminator, backtrack & handle closing tag
						--s;

						if (endch != '>')
							PUGI__THROW_ERROR(status_bad_start_element, s);
					}
					else
						PUGI__THROW_ERROR(status_bad_start_element, s);
				}
				else if (*s == '/')
				{
					++s;

					char_t* name = cursor->name;
					if (!name)
						PUGI__THROW_ERROR(status_end_element_mismatch, s);

					while (PUGI__IS_CHARTYPE(*s, ct_symbol))
					{
						if (*s++ != *name++)
							PUGI__THROW_ERROR(status_end_element_mismatch, s);
					}

					if (*name)
					{
						if (*s == 0 && name[0] == endch && name[1] == 0)
							PUGI__THROW_ERROR(status_bad_end_element, s);
						else
							PUGI__THROW_ERROR(status_end_element_mismatch, s);
					}

					PUGI__POPNODE(); // Pop.

					PUGI__SKIPWS();

					if (*s == 0)
					{
						if (endch != '>')
							PUGI__THROW_ERROR(status_bad_end_element, s);
					}
					else
					{
						if (*s != '>')
							PUGI__THROW_ERROR(status_bad_end_element, s);
						++s;
					}
				}
				else if (*s == '?') // '<?...'
				{
					s = parse_question(s, cursor, optmsk, endch);
					if (!s)
						return s;

					assert(cursor);
					if ((cursor->header & xml_memory_page_type_mask) + 1 == node_declaration)
						goto LOC_ATTRIBUTES;
				}
				else if (*s == '!') // '<!...'
				{
					s = parse_exclamation(s, cursor, optmsk, endch);
					if (!s)
						return s;
				}
				else if (*s == 0 && endch == '?')
					PUGI__THROW_ERROR(status_bad_pi, s);
				else
					PUGI__THROW_ERROR(status_unrecognized_tag, s);
			}
			else
			{
				mark = s; // Save this offset while searching for a terminator.

				PUGI__SKIPWS(); // Eat whitespace if no genuine PCDATA here.

				if (*s == '<')
				{
					// We skipped some whitespace characters because otherwise we would take the tag branch instead of PCDATA one
					assert(mark != s);

					if (!PUGI__OPTSET(parse_ws_pcdata | parse_ws_pcdata_single))
					{
						continue;
					}
					else if (PUGI__OPTSET(parse_ws_pcdata_single))
					{
						if (s[1] != '/' || cursor->first_child)
							continue;
					}
				}

				s = mark;

				if (cursor->parent)
				{
					PUGI__PUSHNODE(node_pcdata); // Append a new node on the tree.
					cursor->value = s; // Save the offset.

					s = strconv_pcdata(s);

					PUGI__POPNODE(); // Pop since this is a standalone.

					if (!*s)
						break;
				}
				else
				{
					PUGI__SCANFOR(*s == '<'); // '...<'
					if (!*s)
						break;

					++s;
				}

				// We're after '<'
				goto LOC_TAG;
			}
		}

		// check that last tag is closed
		if (cursor != xmldoc)
			PUGI__THROW_ERROR(status_end_element_mismatch, s);

		return s;
	}

	static xml_parse_result parse(char_t* buffer, size_t length, xml_node_struct* root, unsigned int optmsk)
	{
		xml_document_struct* xmldoc = static_cast<xml_document_struct*>(root);

		// store buffer for offset_debug
		xmldoc->buffer = buffer;

		// early-out for empty documents
		if (length == 0)
			return make_parse_result(status_ok);

		// create parser on stack
		xml_parser parser(*xmldoc);

		// save last character and make buffer zero-terminated (speeds up parsing)
		char_t endch = buffer[length - 1];
		buffer[length - 1] = 0;

		// perform actual parsing
		parser.parse(buffer, xmldoc, optmsk, endch);

		xml_parse_result result =
			make_parse_result(parser.error_status, parser.error_offset ? parser.error_offset - buffer : 0);
		assert(result.offset >= 0 && static_cast<size_t>(result.offset) <= length);

		// update allocator state
		*static_cast<xml_allocator*>(xmldoc) = parser.alloc;

		// since we removed last character, we have to handle the only possible false positive
		if (result && endch == '<')
		{
			// there's no possible well-formed document with < at the end
			return make_parse_result(status_unrecognized_tag, length);
		}

		return result;
	}
};

// Output facilities
PUGI__FN xml_encoding get_write_native_encoding()
{
#ifdef PUGIXML_WCHAR_MODE
	return get_wchar_encoding();
#else
	return encoding_utf8;
#endif
}

PUGI__FN xml_encoding get_write_encoding(xml_encoding encoding)
{
	// replace wchar encoding with utf implementation
	if (encoding == encoding_wchar)
		return get_wchar_encoding();

	// replace utf16 encoding with utf16 with specific endianness
	if (encoding == encoding_utf16)
		return is_little_endian() ? encoding_utf16_le : encoding_utf16_be;

	// replace utf32 encoding with utf32 with specific endianness
	if (encoding == encoding_utf32)
		return is_little_endian() ? encoding_utf32_le : encoding_utf32_be;

	// only do autodetection if no explicit encoding is requested
	if (encoding != encoding_auto)
		return encoding;

	// assume utf8 encoding
	return encoding_utf8;
}

#ifdef PUGIXML_WCHAR_MODE
PUGI__FN size_t get_valid_length(const char_t* data, size_t length)
{
	assert(length > 0);

	// discard last character if it's the lead of a surrogate pair
	return (sizeof(wchar_t) == 2 && static_cast<unsigned int>(static_cast<uint16_t>(data[length - 1]) - 0xD800) < 0x400)
		? length - 1
		: length;
}

PUGI__FN size_t convert_buffer(
	char_t* r_char,
	uint8_t* r_u8,
	uint16_t* r_u16,
	uint32_t* r_u32,
	const char_t* data,
	size_t length,
	xml_encoding encoding)
{
	// only endian-swapping is required
	if (need_endian_swap_utf(encoding, get_wchar_encoding()))
	{
		convert_wchar_endian_swap(r_char, data, length);

		return length * sizeof(char_t);
	}

	// convert to utf8
	if (encoding == encoding_utf8)
	{
		uint8_t* dest = r_u8;
		uint8_t* end = utf_decoder<utf8_writer>::decode_wchar_block(data, length, dest);

		return static_cast<size_t>(end - dest);
	}

	// convert to utf16
	if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
	{
		uint16_t* dest = r_u16;

		// convert to native utf16
		uint16_t* end = utf_decoder<utf16_writer>::decode_wchar_block(data, length, dest);

		// swap if necessary
		xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;

		if (native_encoding != encoding)
			convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));

		return static_cast<size_t>(end - dest) * sizeof(uint16_t);
	}

	// convert to utf32
	if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
	{
		uint32_t* dest = r_u32;

		// convert to native utf32
		uint32_t* end = utf_decoder<utf32_writer>::decode_wchar_block(data, length, dest);

		// swap if necessary
		xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;

		if (native_encoding != encoding)
			convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));

		return static_cast<size_t>(end - dest) * sizeof(uint32_t);
	}

	// convert to latin1
	if (encoding == encoding_latin1)
	{
		uint8_t* dest = r_u8;
		uint8_t* end = utf_decoder<latin1_writer>::decode_wchar_block(data, length, dest);

		return static_cast<size_t>(end - dest);
	}

	assert(!"Invalid encoding");
	return 0;
}
#else
PUGI__FN size_t get_valid_length(const char_t* data, size_t length)
{
	assert(length > 4);

	for (size_t i = 1; i <= 4; ++i)
	{
		uint8_t ch = static_cast<uint8_t>(data[length - i]);

		// either a standalone character or a leading one
		if ((ch & 0xc0) != 0x80)
			return length - i;
	}

	// there are four non-leading characters at the end, sequence tail is broken so might as well process the whole chunk
	return length;
}

PUGI__FN size_t convert_buffer(
	char_t* /* r_char */,
	uint8_t* r_u8,
	uint16_t* r_u16,
	uint32_t* r_u32,
	const char_t* data,
	size_t length,
	xml_encoding encoding)
{
	if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
	{
		uint16_t* dest = r_u16;

		// convert to native utf16
		uint16_t* end =
			utf_decoder<utf16_writer>::decode_utf8_block(reinterpret_cast<const uint8_t*>(data), length, dest);

		// swap if necessary
		xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;

		if (native_encoding != encoding)
			convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));

		return static_cast<size_t>(end - dest) * sizeof(uint16_t);
	}

	if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
	{
		uint32_t* dest = r_u32;

		// convert to native utf32
		uint32_t* end =
			utf_decoder<utf32_writer>::decode_utf8_block(reinterpret_cast<const uint8_t*>(data), length, dest);

		// swap if necessary
		xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;

		if (native_encoding != encoding)
			convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));

		return static_cast<size_t>(end - dest) * sizeof(uint32_t);
	}

	if (encoding == encoding_latin1)
	{
		uint8_t* dest = r_u8;
		uint8_t* end =
			utf_decoder<latin1_writer>::decode_utf8_block(reinterpret_cast<const uint8_t*>(data), length, dest);

		return static_cast<size_t>(end - dest);
	}

	assert(!"Invalid encoding");
	return 0;
}
#endif

class xml_buffered_writer
{
	xml_buffered_writer(const xml_buffered_writer&) = delete;
	xml_buffered_writer& operator=(const xml_buffered_writer&) = delete;

public:
	xml_buffered_writer(xml_writer& writer_, xml_encoding user_encoding) :
		writer(writer_),
		bufsize(0),
		encoding(get_write_encoding(user_encoding))
	{
		PUGI__STATIC_ASSERT(bufcapacity >= 8);
	}

	~xml_buffered_writer()
	{
		flush();
	}

	void flush()
	{
		flush(buffer, bufsize);
		bufsize = 0;
	}

	void flush(const char_t* data, size_t size)
	{
		if (size == 0)
			return;

		// fast path, just write data
		if (encoding == get_write_native_encoding())
			writer.write(data, size * sizeof(char_t));
		else
		{
			// convert chunk
			size_t result = convert_buffer(
				scratch.data_char,
				scratch.data_u8,
				scratch.data_u16,
				scratch.data_u32,
				data,
				size,
				encoding);
			assert(result <= sizeof(scratch));

			// write data
			writer.write(scratch.data_u8, result);
		}
	}

	void write(const char_t* data, size_t length)
	{
		if (bufsize + length > bufcapacity)
		{
			// flush the remaining buffer contents
			flush();

			// handle large chunks
			if (length > bufcapacity)
			{
				if (encoding == get_write_native_encoding())
				{
					// fast path, can just write data chunk
					writer.write(data, length * sizeof(char_t));
					return;
				}

				// need to convert in suitable chunks
				while (length > bufcapacity)
				{
					// get chunk size by selecting such number of characters that are guaranteed to fit into scratch buffer
					// and form a complete codepoint sequence (i.e. discard start of last codepoint if necessary)
					size_t chunk_size = get_valid_length(data, bufcapacity);

					// convert chunk and write
					flush(data, chunk_size);

					// iterate
					data += chunk_size;
					length -= chunk_size;
				}

				// small tail is copied below
				bufsize = 0;
			}
		}

		memcpy(buffer + bufsize, data, length * sizeof(char_t));
		bufsize += length;
	}

	void write(const char_t* data)
	{
		write(data, strlength(data));
	}

	void write(char_t d0)
	{
		if (bufsize + 1 > bufcapacity)
			flush();

		buffer[bufsize + 0] = d0;
		bufsize += 1;
	}

	void write(char_t d0, char_t d1)
	{
		if (bufsize + 2 > bufcapacity)
			flush();

		buffer[bufsize + 0] = d0;
		buffer[bufsize + 1] = d1;
		bufsize += 2;
	}

	void write(char_t d0, char_t d1, char_t d2)
	{
		if (bufsize + 3 > bufcapacity)
			flush();

		buffer[bufsize + 0] = d0;
		buffer[bufsize + 1] = d1;
		buffer[bufsize + 2] = d2;
		bufsize += 3;
	}

	void write(char_t d0, char_t d1, char_t d2, char_t d3)
	{
		if (bufsize + 4 > bufcapacity)
			flush();

		buffer[bufsize + 0] = d0;
		buffer[bufsize + 1] = d1;
		buffer[bufsize + 2] = d2;
		buffer[bufsize + 3] = d3;
		bufsize += 4;
	}

	void write(char_t d0, char_t d1, char_t d2, char_t d3, char_t d4)
	{
		if (bufsize + 5 > bufcapacity)
			flush();

		buffer[bufsize + 0] = d0;
		buffer[bufsize + 1] = d1;
		buffer[bufsize + 2] = d2;
		buffer[bufsize + 3] = d3;
		buffer[bufsize + 4] = d4;
		bufsize += 5;
	}

	void write(char_t d0, char_t d1, char_t d2, char_t d3, char_t d4, char_t d5)
	{
		if (bufsize + 6 > bufcapacity)
			flush();

		buffer[bufsize + 0] = d0;
		buffer[bufsize + 1] = d1;
		buffer[bufsize + 2] = d2;
		buffer[bufsize + 3] = d3;
		buffer[bufsize + 4] = d4;
		buffer[bufsize + 5] = d5;
		bufsize += 6;
	}

	// utf8 maximum expansion: x4 (-> utf32)
	// utf16 maximum expansion: x2 (-> utf32)
	// utf32 maximum expansion: x1
	enum
	{
		bufcapacitybytes =
#ifdef PUGIXML_MEMORY_OUTPUT_STACK
			PUGIXML_MEMORY_OUTPUT_STACK
#else
			10240
#endif
			,
		bufcapacity = bufcapacitybytes / (sizeof(char_t) + 4)
	};

	char_t buffer[bufcapacity];

	union
	{
		uint8_t data_u8[4 * bufcapacity];
		uint16_t data_u16[2 * bufcapacity];
		uint32_t data_u32[bufcapacity];
		char_t data_char[bufcapacity];
	} scratch;

	xml_writer& writer;
	size_t bufsize;
	xml_encoding encoding;
};

PUGI__FN void text_output_escaped(xml_buffered_writer& writer, const char_t* s, chartypex_t type)
{
	while (*s)
	{
		const char_t* prev = s;

		// While *s is a usual symbol
		while (!PUGI__IS_CHARTYPEX(*s, type))
			++s;

		writer.write(prev, static_cast<size_t>(s - prev));

		switch (*s)
		{
		case 0: break;
		case '&':
			writer.write('&', 'a', 'm', 'p', ';');
			++s;
			break;
		case '<':
			writer.write('&', 'l', 't', ';');
			++s;
			break;
		case '>':
			writer.write('&', 'g', 't', ';');
			++s;
			break;
		case '"':
			writer.write('&', 'q', 'u', 'o', 't', ';');
			++s;
			break;
		default: // s is not a usual symbol
		{
			unsigned int ch = static_cast<unsigned int>(*s++);
			assert(ch < 32);

			writer.write('&', '#', static_cast<char_t>((ch / 10) + '0'), static_cast<char_t>((ch % 10) + '0'), ';');
		}
		}
	}
}

PUGI__FN void text_output(xml_buffered_writer& writer, const char_t* s, chartypex_t type, unsigned int flags)
{
	if (flags & format_no_escapes)
		writer.write(s);
	else
		text_output_escaped(writer, s, type);
}

PUGI__FN void text_output_cdata(xml_buffered_writer& writer, const char_t* s)
{
	do
	{
		writer.write('<', '!', '[', 'C', 'D');
		writer.write('A', 'T', 'A', '[');

		const char_t* prev = s;

		// look for ]]> sequence - we can't output it as is since it terminates CDATA
		while (*s && !(s[0] == ']' && s[1] == ']' && s[2] == '>'))
			++s;

		// skip ]] if we stopped at ]]>, > will go to the next CDATA section
		if (*s)
			s += 2;

		writer.write(prev, static_cast<size_t>(s - prev));

		writer.write(']', ']', '>');
	} while (*s);
}

PUGI__FN void node_output_attributes(xml_buffered_writer& writer, const xml_node& node, unsigned int flags)
{
	const char_t* default_name = PUGIXML_TEXT(":anonymous");

	for (xml_attribute a = node.first_attribute(); a; a = a.next_attribute())
	{
		writer.write(' ');
		writer.write(a.name()[0] ? a.name() : default_name);
		writer.write('=', '"');

		text_output(writer, a.value(), ctx_special_attr, flags);

		writer.write('"');
	}
}

PUGI__FN void node_output(
	xml_buffered_writer& writer,
	const xml_node& node,
	const char_t* indent,
	unsigned int flags,
	unsigned int depth)
{
	const char_t* default_name = PUGIXML_TEXT(":anonymous");

	if ((flags & format_indent) != 0 && (flags & format_raw) == 0)
		for (unsigned int i = 0; i < depth; ++i)
			writer.write(indent);

	switch (node.type())
	{
	case node_document:
	{
		for (xml_node n = node.first_child(); n; n = n.next_sibling())
			node_output(writer, n, indent, flags, depth);
		break;
	}

	case node_element:
	{
		const char_t* name = node.name()[0] ? node.name() : default_name;

		writer.write('<');
		writer.write(name);

		node_output_attributes(writer, node, flags);

		if (flags & format_raw)
		{
			if (!node.first_child())
			{
				if (flags & format_space_before_slash)
					writer.write(' ');
				writer.write('/', '>');
			}
			else
			{
				writer.write('>');

				for (xml_node n = node.first_child(); n; n = n.next_sibling())
					node_output(writer, n, indent, flags, depth + 1);

				writer.write('<', '/');
				writer.write(name);
				writer.write('>');
			}
		}
		else if (!node.first_child())
		{
			if (flags & format_space_before_slash)
				writer.write(' ');
			writer.write('/', '>');
			if (flags & format_win_new_line)
				writer.write('\r');
			writer.write('\n');
		}
		else if (
			node.first_child() == node.last_child() &&
			(node.first_child().type() == node_pcdata || node.first_child().type() == node_cdata))
		{
			writer.write('>');

			if (node.first_child().type() == node_pcdata)
				text_output(writer, node.first_child().value(), ctx_special_pcdata, flags);
			else
				text_output_cdata(writer, node.first_child().value());

			writer.write('<', '/');
			writer.write(name);
			writer.write('>');
			if (flags & format_win_new_line)
				writer.write('\r');
			writer.write('\n');
		}
		else
		{
			writer.write('>');
			if (flags & format_win_new_line)
				writer.write('\r');
			writer.write('\n');

			for (xml_node n = node.first_child(); n; n = n.next_sibling())
				node_output(writer, n, indent, flags, depth + 1);

			if ((flags & format_indent) != 0 && (flags & format_raw) == 0)
				for (unsigned int i = 0; i < depth; ++i)
					writer.write(indent);

			writer.write('<', '/');
			writer.write(name);
			writer.write('>');
			if (flags & format_win_new_line)
				writer.write('\r');
			writer.write('\n');
		}

		break;
	}

	case node_pcdata:
		text_output(writer, node.value(), ctx_special_pcdata, flags);
		if ((flags & format_raw) == 0)
		{
			if (flags & format_win_new_line)
				writer.write('\r');
			writer.write('\n');
		}
		break;

	case node_cdata:
		text_output_cdata(writer, node.value());
		if ((flags & format_raw) == 0)
		{
			if (flags & format_win_new_line)
				writer.write('\r');
			writer.write('\n');
		}
		break;

	case node_comment:
		writer.write('<', '!', '-', '-');
		writer.write(node.value());
		writer.write('-', '-', '>');
		if ((flags & format_raw) == 0)
		{
			if (flags & format_win_new_line)
				writer.write('\r');
			writer.write('\n');
		}
		break;

	case node_pi:
	case node_declaration:
		writer.write('<', '?');
		writer.write(node.name()[0] ? node.name() : default_name);

		if (node.type() == node_declaration)
		{
			node_output_attributes(writer, node, flags);
		}
		else if (node.value()[0])
		{
			writer.write(' ');
			writer.write(node.value());
		}

		writer.write('?', '>');
		if ((flags & format_raw) == 0)
		{
			if (flags & format_win_new_line)
				writer.write('\r');
			writer.write('\n');
		}
		break;

	case node_doctype:
		writer.write('<', '!', 'D', 'O', 'C');
		writer.write('T', 'Y', 'P', 'E');

		if (node.value()[0])
		{
			writer.write(' ');
			writer.write(node.value());
		}

		writer.write('>');
		if ((flags & format_raw) == 0)
		{
			if (flags & format_win_new_line)
				writer.write('\r');
			writer.write('\n');
		}
		break;

	default: assert(!"Invalid node type");
	}
}

inline bool has_declaration(const xml_node& node)
{
	for (xml_node child = node.first_child(); child; child = child.next_sibling())
	{
		xml_node_type type = child.type();

		if (type == node_declaration)
			return true;
		if (type == node_element)
			return false;
	}

	return false;
}

inline bool allow_insert_child(xml_node_type parent, xml_node_type child)
{
	if (parent != node_document && parent != node_element)
		return false;
	if (child == node_document || child == node_null)
		return false;
	if (parent != node_document && (child == node_declaration || child == node_doctype))
		return false;

	return true;
}

PUGI__FN void recursive_copy_skip(xml_node& dest, const xml_node& source, const xml_node& skip)
{
	assert(dest.type() == source.type());

	switch (source.type())
	{
	case node_element:
	{
		dest.set_name(source.name());

		for (xml_attribute a = source.first_attribute(); a; a = a.next_attribute())
			dest.append_attribute(a.name()).set_value(a.value());

		for (xml_node c = source.first_child(); c; c = c.next_sibling())
		{
			if (c == skip)
				continue;

			xml_node cc = dest.append_child(c.type());
			assert(cc);

			recursive_copy_skip(cc, c, skip);
		}

		break;
	}

	case node_pcdata:
	case node_cdata:
	case node_comment:
	case node_doctype: dest.set_value(source.value()); break;

	case node_pi:
		dest.set_name(source.name());
		dest.set_value(source.value());
		break;

	case node_declaration:
	{
		dest.set_name(source.name());

		for (xml_attribute a = source.first_attribute(); a; a = a.next_attribute())
			dest.append_attribute(a.name()).set_value(a.value());

		break;
	}

	default: assert(!"Invalid node type");
	}
}

inline bool is_text_node(xml_node_struct* node)
{
	xml_node_type type = static_cast<xml_node_type>((node->header & impl::xml_memory_page_type_mask) + 1);

	return type == node_pcdata || type == node_cdata;
}

// get value with conversion functions
PUGI__FN int get_value_int(const char_t* value, int def)
{
	if (!value)
		return def;

#ifdef PUGIXML_WCHAR_MODE
	return static_cast<int>(wcstol(value, 0, 10));
#else
	return static_cast<int>(strtol(value, 0, 10));
#endif
}

PUGI__FN unsigned int get_value_uint(const char_t* value, unsigned int def)
{
	if (!value)
		return def;

#ifdef PUGIXML_WCHAR_MODE
	return static_cast<unsigned int>(wcstoul(value, 0, 10));
#else
	return static_cast<unsigned int>(strtoul(value, 0, 10));
#endif
}

PUGI__FN double get_value_double(const char_t* value, double def)
{
	if (!value)
		return def;

#ifdef PUGIXML_WCHAR_MODE
	return wcstod(value, 0);
#else
	return strtod(value, 0);
#endif
}

PUGI__FN float get_value_float(const char_t* value, float def)
{
	if (!value)
		return def;

#ifdef PUGIXML_WCHAR_MODE
	return static_cast<float>(wcstod(value, 0));
#else
	return static_cast<float>(strtod(value, 0));
#endif
}

PUGI__FN bool get_value_bool(const char_t* value, bool def)
{
	if (!value)
		return def;

	// only look at first char
	char_t first = *value;

	// 1*, t* (true), T* (True), y* (yes), Y* (YES)
	return (first == '1' || first == 't' || first == 'T' || first == 'y' || first == 'Y');
}

// set value with conversion functions
PUGI__FN bool set_value_buffer(char_t*& dest, uintptr_t& header, uintptr_t header_mask, char (&buf)[128])
{
#ifdef PUGIXML_WCHAR_MODE
	char_t wbuf[128];
	impl::widen_ascii(wbuf, buf);

	return strcpy_insitu(dest, header, header_mask, wbuf);
#else
	return strcpy_insitu(dest, header, header_mask, buf);
#endif
}

PUGI__FN bool set_value_convert(char_t*& dest, uintptr_t& header, uintptr_t header_mask, int value)
{
	char buf[128];
	sprintf(buf, "%d", value);

	return set_value_buffer(dest, header, header_mask, buf);
}

PUGI__FN bool set_value_convert(char_t*& dest, uintptr_t& header, uintptr_t header_mask, unsigned int value)
{
	char buf[128];
	sprintf(buf, "%u", value);

	return set_value_buffer(dest, header, header_mask, buf);
}

PUGI__FN bool set_value_convert(char_t*& dest, uintptr_t& header, uintptr_t header_mask, double value)
{
	char buf[128];
	sprintf(buf, "%g", value);

	return set_value_buffer(dest, header, header_mask, buf);
}

PUGI__FN bool set_value_convert(char_t*& dest, uintptr_t& header, uintptr_t header_mask, bool value)
{
	return strcpy_insitu(dest, header, header_mask, value ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));
}

// we need to get length of entire file to load it in memory; the only (relatively) sane way to do it is via seek/tell trick
PUGI__FN xml_parse_status get_file_size(FILE* file, size_t& out_result)
{
#if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 && !defined(_WIN32_WCE)
	// there are 64-bit versions of fseek/ftell, let's use them
	typedef __int64 length_type;

	_fseeki64(file, 0, SEEK_END);
	length_type length = _ftelli64(file);
	_fseeki64(file, 0, SEEK_SET);
#elif defined(__MINGW32__) && !defined(__NO_MINGW_LFS) && !defined(__STRICT_ANSI__)
	// there are 64-bit versions of fseek/ftell, let's use them
	typedef off64_t length_type;

	fseeko64(file, 0, SEEK_END);
	length_type length = ftello64(file);
	fseeko64(file, 0, SEEK_SET);
#else
	// if this is a 32-bit OS, long is enough; if this is a unix system, long is 64-bit, which is enough; otherwise we can't do anything anyway.
	typedef long length_type;

	fseek(file, 0, SEEK_END);
	length_type length = ftell(file);
	fseek(file, 0, SEEK_SET);
#endif

	// check for I/O errors
	if (length < 0)
		return status_io_error;

	// check for overflow
	size_t result = static_cast<size_t>(length);

	if (static_cast<length_type>(result) != length)
		return status_out_of_memory;

	// finalize
	out_result = result;

	return status_ok;
}

PUGI__FN xml_parse_result load_file_impl(xml_document& doc, FILE* file, unsigned int options, xml_encoding encoding)
{
	if (!file)
		return make_parse_result(status_file_not_found);

	// get file size (can result in I/O errors)
	size_t size = 0;
	xml_parse_status size_status = get_file_size(file, size);

	if (size_status != status_ok)
	{
		fclose(file);
		return make_parse_result(size_status);
	}

	// allocate buffer for the whole file
	char* contents = static_cast<char*>(xml_memory::allocate(size > 0 ? size : 1));

	if (!contents)
	{
		fclose(file);
		return make_parse_result(status_out_of_memory);
	}

	// read file in memory
	size_t read_size = fread(contents, 1, size, file);
	fclose(file);

	if (read_size != size)
	{
		xml_memory::deallocate(contents);
		return make_parse_result(status_io_error);
	}

	return doc.load_buffer_inplace_own(contents, size, options, encoding);
}

#ifndef PUGIXML_NO_STL
template<typename T>
struct xml_stream_chunk
{
	static xml_stream_chunk* create()
	{
		void* memory = xml_memory::allocate(sizeof(xml_stream_chunk));

		return new (memory) xml_stream_chunk();
	}

	static void destroy(void* ptr)
	{
		xml_stream_chunk* chunk = static_cast<xml_stream_chunk*>(ptr);

		// free chunk chain
		while (chunk)
		{
			xml_stream_chunk* next = chunk->next;
			xml_memory::deallocate(chunk);
			chunk = next;
		}
	}

	xml_stream_chunk() :
		next(0),
		size(0)
	{
	}

	xml_stream_chunk* next;
	size_t size;

	T data[xml_memory_page_size / sizeof(T)];
};

template<typename T>
PUGI__FN xml_parse_status load_stream_data_noseek(std::basic_istream<T>& stream, void** out_buffer, size_t* out_size)
{
	buffer_holder chunks(0, xml_stream_chunk<T>::destroy);

	// read file to a chunk list
	size_t total = 0;
	xml_stream_chunk<T>* last = nullptr;

	while (!stream.eof())
	{
		// allocate new chunk
		xml_stream_chunk<T>* chunk = xml_stream_chunk<T>::create();
		if (!chunk)
			return status_out_of_memory;

		// append chunk to list
		if (last)
			last = last->next = chunk;
		else
			chunks.data = last = chunk;

		// read data to chunk
		stream.read(chunk->data, static_cast<std::streamsize>(sizeof(chunk->data) / sizeof(T)));
		chunk->size = static_cast<size_t>(stream.gcount()) * sizeof(T);

		// read may set failbit | eofbit in case gcount() is less than read length, so check for other I/O errors
		if (stream.bad() || (!stream.eof() && stream.fail()))
			return status_io_error;

		// guard against huge files (chunk size is small enough to make this overflow check work)
		if (total + chunk->size < total)
			return status_out_of_memory;
		total += chunk->size;
	}

	// copy chunk list to a contiguous buffer
	char* buffer = static_cast<char*>(xml_memory::allocate(total));
	if (!buffer)
		return status_out_of_memory;

	char* write = buffer;

	for (xml_stream_chunk<T>* chunk = static_cast<xml_stream_chunk<T>*>(chunks.data); chunk; chunk = chunk->next)
	{
		assert(write + chunk->size <= buffer + total);
		memcpy(write, chunk->data, chunk->size);
		write += chunk->size;
	}

	assert(write == buffer + total);

	// return buffer
	*out_buffer = buffer;
	*out_size = total;

	return status_ok;
}

template<typename T>
PUGI__FN xml_parse_status load_stream_data_seek(std::basic_istream<T>& stream, void** out_buffer, size_t* out_size)
{
	// get length of remaining data in stream
	typename std::basic_istream<T>::pos_type pos = stream.tellg();
	stream.seekg(0, std::ios::end);
	std::streamoff length = stream.tellg() - pos;
	stream.seekg(pos);

	if (stream.fail() || pos < 0)
		return status_io_error;

	// guard against huge files
	size_t read_length = static_cast<size_t>(length);

	if (static_cast<std::streamsize>(read_length) != length || length < 0)
		return status_out_of_memory;

	// read stream data into memory (guard against stream exceptions with buffer holder)
	buffer_holder buffer(xml_memory::allocate((read_length > 0 ? read_length : 1) * sizeof(T)), xml_memory::deallocate);
	if (!buffer.data)
		return status_out_of_memory;

	stream.read(static_cast<T*>(buffer.data), static_cast<std::streamsize>(read_length));

	// read may set failbit | eofbit in case gcount() is less than read_length (i.e. line ending conversion), so check for other I/O errors
	if (stream.bad() || (!stream.eof() && stream.fail()))
		return status_io_error;

	// return buffer
	size_t actual_length = static_cast<size_t>(stream.gcount());
	assert(actual_length <= read_length);

	*out_buffer = buffer.release();
	*out_size = actual_length * sizeof(T);

	return status_ok;
}

template<typename T>
PUGI__FN xml_parse_result
load_stream_impl(xml_document& doc, std::basic_istream<T>& stream, unsigned int options, xml_encoding encoding)
{
	void* buffer = nullptr;
	size_t size = 0;

	// load stream to memory (using seek-based implementation if possible, since it's faster and takes less memory)
	xml_parse_status status = (stream.tellg() < 0)
		? load_stream_data_noseek(stream, &buffer, &size)
		: load_stream_data_seek(stream, &buffer, &size);
	if (status != status_ok)
		return make_parse_result(status);

	return doc.load_buffer_inplace_own(buffer, size, options, encoding);
}
#endif

#if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__) || (defined(__MINGW32__) && !defined(__STRICT_ANSI__))
PUGI__FN FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
{
	return _wfopen(path, mode);
}
#else
PUGI__FN char* convert_path_heap(const wchar_t* str)
{
	assert(str);

	// first pass: get length in utf8 characters
	size_t length = wcslen(str);
	size_t size = as_utf8_begin(str, length);

	// allocate resulting string
	char* result = static_cast<char*>(xml_memory::allocate(size + 1));
	if (!result)
		return 0;

	// second pass: convert to utf8
	as_utf8_end(result, size, str, length);

	return result;
}

PUGI__FN FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
{
	// there is no standard function to open wide paths, so our best bet is to try utf8 path
	char* path_utf8 = convert_path_heap(path);
	if (!path_utf8)
		return 0;

	// convert mode to ASCII (we mirror _wfopen interface)
	char mode_ascii[4] = {0};
	for (size_t i = 0; mode[i]; ++i)
		mode_ascii[i] = static_cast<char>(mode[i]);

	// try to open the utf8 path
	FILE* result = fopen(path_utf8, mode_ascii);

	// free dummy buffer
	xml_memory::deallocate(path_utf8);

	return result;
}
#endif

PUGI__FN bool save_file_impl(
	const xml_document& doc,
	FILE* file,
	const char_t* indent,
	unsigned int flags,
	xml_encoding encoding)
{
	if (!file)
		return false;

	xml_writer_file writer(file);
	doc.save(writer, indent, flags, encoding);

	int result = ferror(file);

	fclose(file);

	return result == 0;
}
PUGI__NS_END

namespace pugi
{
	PUGI__FN xml_writer_file::xml_writer_file(void* file_) :
		file(file_)
	{
	}

	PUGI__FN void xml_writer_file::write(const void* data, size_t size)
	{
		size_t result = fwrite(data, 1, size, static_cast<FILE*>(file));
		(void)!result; // unfortunately we can't do proper error handling here
	}

#ifndef PUGIXML_NO_STL
	PUGI__FN xml_writer_stream::xml_writer_stream(std::basic_ostream<char, std::char_traits<char>>& stream) :
		narrow_stream(&stream),
		wide_stream(0)
	{
	}

	PUGI__FN xml_writer_stream::xml_writer_stream(std::basic_ostream<wchar_t, std::char_traits<wchar_t>>& stream) :
		narrow_stream(0),
		wide_stream(&stream)
	{
	}

	PUGI__FN void xml_writer_stream::write(const void* data, size_t size)
	{
		if (narrow_stream)
		{
			assert(!wide_stream);
			narrow_stream->write(reinterpret_cast<const char*>(data), static_cast<std::streamsize>(size));
		}
		else
		{
			assert(wide_stream);
			assert(size % sizeof(wchar_t) == 0);

			wide_stream->write(
				reinterpret_cast<const wchar_t*>(data),
				static_cast<std::streamsize>(size / sizeof(wchar_t)));
		}
	}
#endif

	PUGI__FN xml_tree_walker::xml_tree_walker() :
		_depth(0)
	{
	}

	PUGI__FN xml_tree_walker::~xml_tree_walker()
	{
	}

	PUGI__FN int xml_tree_walker::depth() const
	{
		return _depth;
	}

	PUGI__FN bool xml_tree_walker::begin(xml_node&)
	{
		return true;
	}

	PUGI__FN bool xml_tree_walker::end(xml_node&)
	{
		return true;
	}

	PUGI__FN xml_attribute::xml_attribute() :
		_attr(0)
	{
	}

	PUGI__FN xml_attribute::xml_attribute(xml_attribute_struct* attr) :
		_attr(attr)
	{
	}

	PUGI__FN static void unspecified_bool_xml_attribute(xml_attribute***)
	{
	}

	PUGI__FN xml_attribute::operator xml_attribute::unspecified_bool_type() const
	{
		return _attr ? unspecified_bool_xml_attribute : 0;
	}

	PUGI__FN bool xml_attribute::operator!() const
	{
		return !_attr;
	}

	PUGI__FN bool xml_attribute::operator==(const xml_attribute& r) const
	{
		return (_attr == r._attr);
	}

	PUGI__FN bool xml_attribute::operator!=(const xml_attribute& r) const
	{
		return (_attr != r._attr);
	}

	PUGI__FN bool xml_attribute::operator<(const xml_attribute& r) const
	{
		return (_attr < r._attr);
	}

	PUGI__FN bool xml_attribute::operator>(const xml_attribute& r) const
	{
		return (_attr > r._attr);
	}

	PUGI__FN bool xml_attribute::operator<=(const xml_attribute& r) const
	{
		return (_attr <= r._attr);
	}

	PUGI__FN bool xml_attribute::operator>=(const xml_attribute& r) const
	{
		return (_attr >= r._attr);
	}

	PUGI__FN xml_attribute xml_attribute::next_attribute() const
	{
		return _attr ? xml_attribute(_attr->next_attribute) : xml_attribute();
	}

	PUGI__FN xml_attribute xml_attribute::previous_attribute() const
	{
		return _attr && _attr->prev_attribute_c->next_attribute
			? xml_attribute(_attr->prev_attribute_c)
			: xml_attribute();
	}

	PUGI__FN const char_t* xml_attribute::as_string(const char_t* def) const
	{
		return (_attr && _attr->value) ? _attr->value : def;
	}

	PUGI__FN int xml_attribute::as_int(int def) const
	{
		return impl::get_value_int(_attr ? _attr->value : 0, def);
	}

	PUGI__FN unsigned int xml_attribute::as_uint(unsigned int def) const
	{
		return impl::get_value_uint(_attr ? _attr->value : 0, def);
	}

	PUGI__FN double xml_attribute::as_double(double def) const
	{
		return impl::get_value_double(_attr ? _attr->value : 0, def);
	}

	PUGI__FN float xml_attribute::as_float(float def) const
	{
		return impl::get_value_float(_attr ? _attr->value : 0, def);
	}

	PUGI__FN bool xml_attribute::as_bool(bool def) const
	{
		return impl::get_value_bool(_attr ? _attr->value : 0, def);
	}

	PUGI__FN bool xml_attribute::empty() const
	{
		return !_attr;
	}

	PUGI__FN const char_t* xml_attribute::name() const
	{
		return (_attr && _attr->name) ? _attr->name : PUGIXML_TEXT("");
	}

	PUGI__FN const char_t* xml_attribute::value() const
	{
		return (_attr && _attr->value) ? _attr->value : PUGIXML_TEXT("");
	}

	PUGI__FN size_t xml_attribute::hash_value() const
	{
		return static_cast<size_t>(reinterpret_cast<uintptr_t>(_attr) / sizeof(xml_attribute_struct));
	}

	PUGI__FN xml_attribute_struct* xml_attribute::internal_object() const
	{
		return _attr;
	}

	PUGI__FN xml_attribute& xml_attribute::operator=(const char_t* rhs)
	{
		set_value(rhs);
		return *this;
	}

	PUGI__FN xml_attribute& xml_attribute::operator=(int rhs)
	{
		set_value(rhs);
		return *this;
	}

	PUGI__FN xml_attribute& xml_attribute::operator=(unsigned int rhs)
	{
		set_value(rhs);
		return *this;
	}

	PUGI__FN xml_attribute& xml_attribute::operator=(double rhs)
	{
		set_value(rhs);
		return *this;
	}

	PUGI__FN xml_attribute& xml_attribute::operator=(bool rhs)
	{
		set_value(rhs);
		return *this;
	}

	PUGI__FN bool xml_attribute::set_name(const char_t* rhs)
	{
		if (!_attr)
			return false;

		return impl::strcpy_insitu(_attr->name, _attr->header, impl::xml_memory_page_name_allocated_mask, rhs);
	}

	PUGI__FN bool xml_attribute::set_value(const char_t* rhs)
	{
		if (!_attr)
			return false;

		return impl::strcpy_insitu(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
	}

	PUGI__FN bool xml_attribute::set_value(int rhs)
	{
		if (!_attr)
			return false;

		return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
	}

	PUGI__FN bool xml_attribute::set_value(unsigned int rhs)
	{
		if (!_attr)
			return false;

		return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
	}

	PUGI__FN bool xml_attribute::set_value(double rhs)
	{
		if (!_attr)
			return false;

		return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
	}

	PUGI__FN bool xml_attribute::set_value(bool rhs)
	{
		if (!_attr)
			return false;

		return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
	}

#ifdef __BORLANDC__
	PUGI__FN bool operator&&(const xml_attribute& lhs, bool rhs)
	{
		return (bool)lhs && rhs;
	}

	PUGI__FN bool operator||(const xml_attribute& lhs, bool rhs)
	{
		return (bool)lhs || rhs;
	}
#endif

	PUGI__FN xml_node::xml_node() :
		_root(0)
	{
	}

	PUGI__FN xml_node::xml_node(xml_node_struct* p) :
		_root(p)
	{
	}

	PUGI__FN static void unspecified_bool_xml_node(xml_node***)
	{
	}

	PUGI__FN xml_node::operator xml_node::unspecified_bool_type() const
	{
		return _root ? unspecified_bool_xml_node : 0;
	}

	PUGI__FN bool xml_node::operator!() const
	{
		return !_root;
	}

	PUGI__FN xml_node::iterator xml_node::begin() const
	{
		return iterator(_root ? _root->first_child : 0, _root);
	}

	PUGI__FN xml_node::iterator xml_node::end() const
	{
		return iterator(0, _root);
	}

	PUGI__FN xml_node::attribute_iterator xml_node::attributes_begin() const
	{
		return attribute_iterator(_root ? _root->first_attribute : 0, _root);
	}

	PUGI__FN xml_node::attribute_iterator xml_node::attributes_end() const
	{
		return attribute_iterator(0, _root);
	}

	PUGI__FN xml_object_range<xml_node_iterator> xml_node::children() const
	{
		return xml_object_range<xml_node_iterator>(begin(), end());
	}

	PUGI__FN xml_object_range<xml_named_node_iterator> xml_node::children(const char_t* name_) const
	{
		return xml_object_range<xml_named_node_iterator>(
			xml_named_node_iterator(child(name_), name_),
			xml_named_node_iterator());
	}

	PUGI__FN xml_object_range<xml_attribute_iterator> xml_node::attributes() const
	{
		return xml_object_range<xml_attribute_iterator>(attributes_begin(), attributes_end());
	}

	PUGI__FN bool xml_node::operator==(const xml_node& r) const
	{
		return (_root == r._root);
	}

	PUGI__FN bool xml_node::operator!=(const xml_node& r) const
	{
		return (_root != r._root);
	}

	PUGI__FN bool xml_node::operator<(const xml_node& r) const
	{
		return (_root < r._root);
	}

	PUGI__FN bool xml_node::operator>(const xml_node& r) const
	{
		return (_root > r._root);
	}

	PUGI__FN bool xml_node::operator<=(const xml_node& r) const
	{
		return (_root <= r._root);
	}

	PUGI__FN bool xml_node::operator>=(const xml_node& r) const
	{
		return (_root >= r._root);
	}

	PUGI__FN bool xml_node::empty() const
	{
		return !_root;
	}

	PUGI__FN const char_t* xml_node::name() const
	{
		return (_root && _root->name) ? _root->name : PUGIXML_TEXT("");
	}

	PUGI__FN xml_node_type xml_node::type() const
	{
		return _root ? static_cast<xml_node_type>((_root->header & impl::xml_memory_page_type_mask) + 1) : node_null;
	}

	PUGI__FN const char_t* xml_node::value() const
	{
		return (_root && _root->value) ? _root->value : PUGIXML_TEXT("");
	}

	PUGI__FN xml_node xml_node::child(const char_t* name_) const
	{
		if (!_root)
			return xml_node();

		for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
			if (i->name && impl::strequal(name_, i->name))
				return xml_node(i);

		return xml_node();
	}

	PUGI__FN xml_attribute xml_node::attribute(const char_t* name_) const
	{
		if (!_root)
			return xml_attribute();

		for (xml_attribute_struct* i = _root->first_attribute; i; i = i->next_attribute)
			if (i->name && impl::strequal(name_, i->name))
				return xml_attribute(i);

		return xml_attribute();
	}

	PUGI__FN xml_node xml_node::next_sibling(const char_t* name_) const
	{
		if (!_root)
			return xml_node();

		for (xml_node_struct* i = _root->next_sibling; i; i = i->next_sibling)
			if (i->name && impl::strequal(name_, i->name))
				return xml_node(i);

		return xml_node();
	}

	PUGI__FN xml_node xml_node::next_sibling() const
	{
		if (!_root)
			return xml_node();

		if (_root->next_sibling)
			return xml_node(_root->next_sibling);
		else
			return xml_node();
	}

	PUGI__FN xml_node xml_node::previous_sibling(const char_t* name_) const
	{
		if (!_root)
			return xml_node();

		for (xml_node_struct* i = _root->prev_sibling_c; i->next_sibling; i = i->prev_sibling_c)
			if (i->name && impl::strequal(name_, i->name))
				return xml_node(i);

		return xml_node();
	}

	PUGI__FN xml_node xml_node::previous_sibling() const
	{
		if (!_root)
			return xml_node();

		if (_root->prev_sibling_c->next_sibling)
			return xml_node(_root->prev_sibling_c);
		else
			return xml_node();
	}

	PUGI__FN xml_node xml_node::parent() const
	{
		return _root ? xml_node(_root->parent) : xml_node();
	}

	PUGI__FN xml_node xml_node::root() const
	{
		if (!_root)
			return xml_node();

		impl::xml_memory_page* page =
			reinterpret_cast<impl::xml_memory_page*>(_root->header & impl::xml_memory_page_pointer_mask);

		return xml_node(static_cast<impl::xml_document_struct*>(page->allocator));
	}

	PUGI__FN xml_text xml_node::text() const
	{
		return xml_text(_root);
	}

	PUGI__FN const char_t* xml_node::child_value() const
	{
		if (!_root)
			return PUGIXML_TEXT("");

		for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
			if (i->value && impl::is_text_node(i))
				return i->value;

		return PUGIXML_TEXT("");
	}

	PUGI__FN const char_t* xml_node::child_value(const char_t* name_) const
	{
		return child(name_).child_value();
	}

	PUGI__FN xml_attribute xml_node::first_attribute() const
	{
		return _root ? xml_attribute(_root->first_attribute) : xml_attribute();
	}

	PUGI__FN xml_attribute xml_node::last_attribute() const
	{
		return _root && _root->first_attribute
			? xml_attribute(_root->first_attribute->prev_attribute_c)
			: xml_attribute();
	}

	PUGI__FN xml_node xml_node::first_child() const
	{
		return _root ? xml_node(_root->first_child) : xml_node();
	}

	PUGI__FN xml_node xml_node::last_child() const
	{
		return _root && _root->first_child ? xml_node(_root->first_child->prev_sibling_c) : xml_node();
	}

	PUGI__FN bool xml_node::set_name(const char_t* rhs)
	{
		switch (type())
		{
		case node_pi:
		case node_declaration:
		case node_element:
			return impl::strcpy_insitu(_root->name, _root->header, impl::xml_memory_page_name_allocated_mask, rhs);

		default: return false;
		}
	}

	PUGI__FN bool xml_node::set_value(const char_t* rhs)
	{
		switch (type())
		{
		case node_pi:
		case node_cdata:
		case node_pcdata:
		case node_comment:
		case node_doctype:
			return impl::strcpy_insitu(_root->value, _root->header, impl::xml_memory_page_value_allocated_mask, rhs);

		default: return false;
		}
	}

	PUGI__FN xml_attribute xml_node::append_attribute(const char_t* name_)
	{
		if (type() != node_element && type() != node_declaration)
			return xml_attribute();

		xml_attribute a(impl::append_attribute_ll(_root, impl::get_allocator(_root)));
		a.set_name(name_);

		return a;
	}

	PUGI__FN xml_attribute xml_node::prepend_attribute(const char_t* name_)
	{
		if (type() != node_element && type() != node_declaration)
			return xml_attribute();

		xml_attribute a(impl::allocate_attribute(impl::get_allocator(_root)));
		if (!a)
			return xml_attribute();

		a.set_name(name_);

		xml_attribute_struct* head = _root->first_attribute;

		if (head)
		{
			a._attr->prev_attribute_c = head->prev_attribute_c;
			head->prev_attribute_c = a._attr;
		}
		else
			a._attr->prev_attribute_c = a._attr;

		a._attr->next_attribute = head;
		_root->first_attribute = a._attr;

		return a;
	}

	PUGI__FN xml_attribute xml_node::insert_attribute_before(const char_t* name_, const xml_attribute& attr)
	{
		if ((type() != node_element && type() != node_declaration) || attr.empty())
			return xml_attribute();

		// check that attribute belongs to *this
		xml_attribute_struct* cur = attr._attr;

		while (cur->prev_attribute_c->next_attribute)
			cur = cur->prev_attribute_c;

		if (cur != _root->first_attribute)
			return xml_attribute();

		xml_attribute a(impl::allocate_attribute(impl::get_allocator(_root)));
		if (!a)
			return xml_attribute();

		a.set_name(name_);

		if (attr._attr->prev_attribute_c->next_attribute)
			attr._attr->prev_attribute_c->next_attribute = a._attr;
		else
			_root->first_attribute = a._attr;

		a._attr->prev_attribute_c = attr._attr->prev_attribute_c;
		a._attr->next_attribute = attr._attr;
		attr._attr->prev_attribute_c = a._attr;

		return a;
	}

	PUGI__FN xml_attribute xml_node::insert_attribute_after(const char_t* name_, const xml_attribute& attr)
	{
		if ((type() != node_element && type() != node_declaration) || attr.empty())
			return xml_attribute();

		// check that attribute belongs to *this
		xml_attribute_struct* cur = attr._attr;

		while (cur->prev_attribute_c->next_attribute)
			cur = cur->prev_attribute_c;

		if (cur != _root->first_attribute)
			return xml_attribute();

		xml_attribute a(impl::allocate_attribute(impl::get_allocator(_root)));
		if (!a)
			return xml_attribute();

		a.set_name(name_);

		if (attr._attr->next_attribute)
			attr._attr->next_attribute->prev_attribute_c = a._attr;
		else
			_root->first_attribute->prev_attribute_c = a._attr;

		a._attr->next_attribute = attr._attr->next_attribute;
		a._attr->prev_attribute_c = attr._attr;
		attr._attr->next_attribute = a._attr;

		return a;
	}

	PUGI__FN xml_attribute xml_node::append_copy(const xml_attribute& proto)
	{
		if (!proto)
			return xml_attribute();

		xml_attribute result = append_attribute(proto.name());
		result.set_value(proto.value());

		return result;
	}

	PUGI__FN xml_attribute xml_node::prepend_copy(const xml_attribute& proto)
	{
		if (!proto)
			return xml_attribute();

		xml_attribute result = prepend_attribute(proto.name());
		result.set_value(proto.value());

		return result;
	}

	PUGI__FN xml_attribute xml_node::insert_copy_after(const xml_attribute& proto, const xml_attribute& attr)
	{
		if (!proto)
			return xml_attribute();

		xml_attribute result = insert_attribute_after(proto.name(), attr);
		result.set_value(proto.value());

		return result;
	}

	PUGI__FN xml_attribute xml_node::insert_copy_before(const xml_attribute& proto, const xml_attribute& attr)
	{
		if (!proto)
			return xml_attribute();

		xml_attribute result = insert_attribute_before(proto.name(), attr);
		result.set_value(proto.value());

		return result;
	}

	PUGI__FN xml_node xml_node::append_child(xml_node_type type_)
	{
		if (!impl::allow_insert_child(this->type(), type_))
			return xml_node();

		xml_node n(impl::append_node(_root, impl::get_allocator(_root), type_));

		if (type_ == node_declaration)
			n.set_name(PUGIXML_TEXT("xml"));

		return n;
	}

	PUGI__FN xml_node xml_node::prepend_child(xml_node_type type_)
	{
		if (!impl::allow_insert_child(this->type(), type_))
			return xml_node();

		xml_node n(impl::allocate_node(impl::get_allocator(_root), type_));
		if (!n)
			return xml_node();

		n._root->parent = _root;

		xml_node_struct* head = _root->first_child;

		if (head)
		{
			n._root->prev_sibling_c = head->prev_sibling_c;
			head->prev_sibling_c = n._root;
		}
		else
			n._root->prev_sibling_c = n._root;

		n._root->next_sibling = head;
		_root->first_child = n._root;

		if (type_ == node_declaration)
			n.set_name(PUGIXML_TEXT("xml"));

		return n;
	}

	PUGI__FN xml_node xml_node::insert_child_before(xml_node_type type_, const xml_node& node)
	{
		if (!impl::allow_insert_child(this->type(), type_))
			return xml_node();
		if (!node._root || node._root->parent != _root)
			return xml_node();

		xml_node n(impl::allocate_node(impl::get_allocator(_root), type_));
		if (!n)
			return xml_node();

		n._root->parent = _root;

		if (node._root->prev_sibling_c->next_sibling)
			node._root->prev_sibling_c->next_sibling = n._root;
		else
			_root->first_child = n._root;

		n._root->prev_sibling_c = node._root->prev_sibling_c;
		n._root->next_sibling = node._root;
		node._root->prev_sibling_c = n._root;

		if (type_ == node_declaration)
			n.set_name(PUGIXML_TEXT("xml"));

		return n;
	}

	PUGI__FN xml_node xml_node::insert_child_after(xml_node_type type_, const xml_node& node)
	{
		if (!impl::allow_insert_child(this->type(), type_))
			return xml_node();
		if (!node._root || node._root->parent != _root)
			return xml_node();

		xml_node n(impl::allocate_node(impl::get_allocator(_root), type_));
		if (!n)
			return xml_node();

		n._root->parent = _root;

		if (node._root->next_sibling)
			node._root->next_sibling->prev_sibling_c = n._root;
		else
			_root->first_child->prev_sibling_c = n._root;

		n._root->next_sibling = node._root->next_sibling;
		n._root->prev_sibling_c = node._root;
		node._root->next_sibling = n._root;

		if (type_ == node_declaration)
			n.set_name(PUGIXML_TEXT("xml"));

		return n;
	}

	PUGI__FN xml_node xml_node::append_child(const char_t* name_)
	{
		xml_node result = append_child(node_element);

		result.set_name(name_);

		return result;
	}

	PUGI__FN xml_node xml_node::prepend_child(const char_t* name_)
	{
		xml_node result = prepend_child(node_element);

		result.set_name(name_);

		return result;
	}

	PUGI__FN xml_node xml_node::insert_child_after(const char_t* name_, const xml_node& node)
	{
		xml_node result = insert_child_after(node_element, node);

		result.set_name(name_);

		return result;
	}

	PUGI__FN xml_node xml_node::insert_child_before(const char_t* name_, const xml_node& node)
	{
		xml_node result = insert_child_before(node_element, node);

		result.set_name(name_);

		return result;
	}

	PUGI__FN xml_node xml_node::append_copy(const xml_node& proto)
	{
		xml_node result = append_child(proto.type());

		if (result)
			impl::recursive_copy_skip(result, proto, result);

		return result;
	}

	PUGI__FN xml_node xml_node::prepend_copy(const xml_node& proto)
	{
		xml_node result = prepend_child(proto.type());

		if (result)
			impl::recursive_copy_skip(result, proto, result);

		return result;
	}

	PUGI__FN xml_node xml_node::insert_copy_after(const xml_node& proto, const xml_node& node)
	{
		xml_node result = insert_child_after(proto.type(), node);

		if (result)
			impl::recursive_copy_skip(result, proto, result);

		return result;
	}

	PUGI__FN xml_node xml_node::insert_copy_before(const xml_node& proto, const xml_node& node)
	{
		xml_node result = insert_child_before(proto.type(), node);

		if (result)
			impl::recursive_copy_skip(result, proto, result);

		return result;
	}

	PUGI__FN bool xml_node::remove_attribute(const char_t* name_)
	{
		return remove_attribute(attribute(name_));
	}

	PUGI__FN bool xml_node::remove_attribute(const xml_attribute& a)
	{
		if (!_root || !a._attr)
			return false;

		// check that attribute belongs to *this
		xml_attribute_struct* attr = a._attr;

		while (attr->prev_attribute_c->next_attribute)
			attr = attr->prev_attribute_c;

		if (attr != _root->first_attribute)
			return false;

		if (a._attr->next_attribute)
			a._attr->next_attribute->prev_attribute_c = a._attr->prev_attribute_c;
		else if (_root->first_attribute)
			_root->first_attribute->prev_attribute_c = a._attr->prev_attribute_c;

		if (a._attr->prev_attribute_c->next_attribute)
			a._attr->prev_attribute_c->next_attribute = a._attr->next_attribute;
		else
			_root->first_attribute = a._attr->next_attribute;

		impl::destroy_attribute(a._attr, impl::get_allocator(_root));

		return true;
	}

	PUGI__FN bool xml_node::remove_child(const char_t* name_)
	{
		return remove_child(child(name_));
	}

	PUGI__FN bool xml_node::remove_child(const xml_node& n)
	{
		if (!_root || !n._root || n._root->parent != _root)
			return false;

		if (n._root->next_sibling)
			n._root->next_sibling->prev_sibling_c = n._root->prev_sibling_c;
		else if (_root->first_child)
			_root->first_child->prev_sibling_c = n._root->prev_sibling_c;

		if (n._root->prev_sibling_c->next_sibling)
			n._root->prev_sibling_c->next_sibling = n._root->next_sibling;
		else
			_root->first_child = n._root->next_sibling;

		impl::destroy_node(n._root, impl::get_allocator(_root));

		return true;
	}

	PUGI__FN xml_node
	xml_node::find_child_by_attribute(const char_t* name_, const char_t* attr_name, const char_t* attr_value) const
	{
		if (!_root)
			return xml_node();

		for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
			if (i->name && impl::strequal(name_, i->name))
			{
				for (xml_attribute_struct* a = i->first_attribute; a; a = a->next_attribute)
					if (impl::strequal(attr_name, a->name) && impl::strequal(attr_value, a->value))
						return xml_node(i);
			}

		return xml_node();
	}

	PUGI__FN xml_node xml_node::find_child_by_attribute(const char_t* attr_name, const char_t* attr_value) const
	{
		if (!_root)
			return xml_node();

		for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
			for (xml_attribute_struct* a = i->first_attribute; a; a = a->next_attribute)
				if (impl::strequal(attr_name, a->name) && impl::strequal(attr_value, a->value))
					return xml_node(i);

		return xml_node();
	}

#ifndef PUGIXML_NO_STL
	PUGI__FN string_t xml_node::path(char_t delimiter) const
	{
		xml_node cursor = *this; // Make a copy.

		string_t result = cursor.name();

		while (cursor.parent())
		{
			cursor = cursor.parent();

			string_t temp = cursor.name();
			temp += delimiter;
			temp += result;
			result.swap(temp);
		}

		return result;
	}
#endif

	PUGI__FN xml_node xml_node::first_element_by_path(const char_t* path_, char_t delimiter) const
	{
		xml_node found = *this; // Current search context.

		if (!_root || !path_ || !path_[0])
			return found;

		if (path_[0] == delimiter)
		{
			// Absolute path; e.g. '/foo/bar'
			found = found.root();
			++path_;
		}

		const char_t* path_segment = path_;

		while (*path_segment == delimiter)
			++path_segment;

		const char_t* path_segment_end = path_segment;

		while (*path_segment_end && *path_segment_end != delimiter)
			++path_segment_end;

		if (path_segment == path_segment_end)
			return found;

		const char_t* next_segment = path_segment_end;

		while (*next_segment == delimiter)
			++next_segment;

		if (*path_segment == '.' && path_segment + 1 == path_segment_end)
			return found.first_element_by_path(next_segment, delimiter);
		else if (*path_segment == '.' && *(path_segment + 1) == '.' && path_segment + 2 == path_segment_end)
			return found.parent().first_element_by_path(next_segment, delimiter);
		else
		{
			for (xml_node_struct* j = found._root->first_child; j; j = j->next_sibling)
			{
				if (j->name &&
					impl::strequalrange(j->name, path_segment, static_cast<size_t>(path_segment_end - path_segment)))
				{
					xml_node subsearch = xml_node(j).first_element_by_path(next_segment, delimiter);

					if (subsearch)
						return subsearch;
				}
			}

			return xml_node();
		}
	}

	PUGI__FN bool xml_node::traverse(xml_tree_walker& walker)
	{
		walker._depth = -1;

		xml_node arg_begin = *this;
		if (!walker.begin(arg_begin))
			return false;

		xml_node cur = first_child();

		if (cur)
		{
			++walker._depth;

			do
			{
				xml_node arg_for_each = cur;
				if (!walker.for_each(arg_for_each))
					return false;

				if (cur.first_child())
				{
					++walker._depth;
					cur = cur.first_child();
				}
				else if (cur.next_sibling())
					cur = cur.next_sibling();
				else
				{
					// Borland C++ workaround
					while (!cur.next_sibling() && cur != *this && !cur.parent().empty())
					{
						--walker._depth;
						cur = cur.parent();
					}

					if (cur != *this)
						cur = cur.next_sibling();
				}
			} while (cur && cur != *this);
		}

		assert(walker._depth == -1);

		xml_node arg_end = *this;
		return walker.end(arg_end);
	}

	PUGI__FN size_t xml_node::hash_value() const
	{
		return static_cast<size_t>(reinterpret_cast<uintptr_t>(_root) / sizeof(xml_node_struct));
	}

	PUGI__FN xml_node_struct* xml_node::internal_object() const
	{
		return _root;
	}

	PUGI__FN void xml_node::print(
		xml_writer& writer,
		const char_t* indent,
		unsigned int flags,
		xml_encoding encoding,
		unsigned int depth) const
	{
		if (!_root)
			return;

		impl::xml_buffered_writer buffered_writer(writer, encoding);

		impl::node_output(buffered_writer, *this, indent, flags, depth);
	}

#ifndef PUGIXML_NO_STL
	PUGI__FN void xml_node::print(
		std::basic_ostream<char, std::char_traits<char>>& stream,
		const char_t* indent,
		unsigned int flags,
		xml_encoding encoding,
		unsigned int depth) const
	{
		xml_writer_stream writer(stream);

		print(writer, indent, flags, encoding, depth);
	}

	PUGI__FN void xml_node::print(
		std::basic_ostream<wchar_t, std::char_traits<wchar_t>>& stream,
		const char_t* indent,
		unsigned int flags,
		unsigned int depth) const
	{
		xml_writer_stream writer(stream);

		print(writer, indent, flags, encoding_wchar, depth);
	}
#endif

	PUGI__FN ptrdiff_t xml_node::offset_debug() const
	{
		xml_node_struct* r = root()._root;

		if (!r)
			return -1;

		const char_t* buffer = static_cast<impl::xml_document_struct*>(r)->buffer;

		if (!buffer)
			return -1;

		switch (type())
		{
		case node_document: return 0;

		case node_element:
		case node_declaration:
		case node_pi: return (_root->header & impl::xml_memory_page_name_allocated_mask) ? -1 : _root->name - buffer;

		case node_pcdata:
		case node_cdata:
		case node_comment:
		case node_doctype:
			return (_root->header & impl::xml_memory_page_value_allocated_mask) ? -1 : _root->value - buffer;

		default: return -1;
		}
	}

#ifdef __BORLANDC__
	PUGI__FN bool operator&&(const xml_node& lhs, bool rhs)
	{
		return (bool)lhs && rhs;
	}

	PUGI__FN bool operator||(const xml_node& lhs, bool rhs)
	{
		return (bool)lhs || rhs;
	}
#endif

	PUGI__FN xml_text::xml_text(xml_node_struct* root) :
		_root(root)
	{
	}

	PUGI__FN xml_node_struct* xml_text::_data() const
	{
		if (!_root || impl::is_text_node(_root))
			return _root;

		for (xml_node_struct* node = _root->first_child; node; node = node->next_sibling)
			if (impl::is_text_node(node))
				return node;

		return 0;
	}

	PUGI__FN xml_node_struct* xml_text::_data_new()
	{
		xml_node_struct* d = _data();
		if (d)
			return d;

		return xml_node(_root).append_child(node_pcdata).internal_object();
	}

	PUGI__FN xml_text::xml_text() :
		_root(0)
	{
	}

	PUGI__FN static void unspecified_bool_xml_text(xml_text***)
	{
	}

	PUGI__FN xml_text::operator xml_text::unspecified_bool_type() const
	{
		return _data() ? unspecified_bool_xml_text : 0;
	}

	PUGI__FN bool xml_text::operator!() const
	{
		return !_data();
	}

	PUGI__FN bool xml_text::empty() const
	{
		return _data() == 0;
	}

	PUGI__FN const char_t* xml_text::get() const
	{
		xml_node_struct* d = _data();

		return (d && d->value) ? d->value : PUGIXML_TEXT("");
	}

	PUGI__FN const char_t* xml_text::as_string(const char_t* def) const
	{
		xml_node_struct* d = _data();

		return (d && d->value) ? d->value : def;
	}

	PUGI__FN int xml_text::as_int(int def) const
	{
		xml_node_struct* d = _data();

		return impl::get_value_int(d ? d->value : 0, def);
	}

	PUGI__FN unsigned int xml_text::as_uint(unsigned int def) const
	{
		xml_node_struct* d = _data();

		return impl::get_value_uint(d ? d->value : 0, def);
	}

	PUGI__FN double xml_text::as_double(double def) const
	{
		xml_node_struct* d = _data();

		return impl::get_value_double(d ? d->value : 0, def);
	}

	PUGI__FN float xml_text::as_float(float def) const
	{
		xml_node_struct* d = _data();

		return impl::get_value_float(d ? d->value : 0, def);
	}

	PUGI__FN bool xml_text::as_bool(bool def) const
	{
		xml_node_struct* d = _data();

		return impl::get_value_bool(d ? d->value : 0, def);
	}

	PUGI__FN bool xml_text::set(const char_t* rhs)
	{
		xml_node_struct* dn = _data_new();

		return dn ? impl::strcpy_insitu(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
	}

	PUGI__FN bool xml_text::set(int rhs)
	{
		xml_node_struct* dn = _data_new();

		return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs)
				  : false;
	}

	PUGI__FN bool xml_text::set(unsigned int rhs)
	{
		xml_node_struct* dn = _data_new();

		return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs)
				  : false;
	}

	PUGI__FN bool xml_text::set(double rhs)
	{
		xml_node_struct* dn = _data_new();

		return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs)
				  : false;
	}

	PUGI__FN bool xml_text::set(bool rhs)
	{
		xml_node_struct* dn = _data_new();

		return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs)
				  : false;
	}

	PUGI__FN xml_text& xml_text::operator=(const char_t* rhs)
	{
		set(rhs);
		return *this;
	}

	PUGI__FN xml_text& xml_text::operator=(int rhs)
	{
		set(rhs);
		return *this;
	}

	PUGI__FN xml_text& xml_text::operator=(unsigned int rhs)
	{
		set(rhs);
		return *this;
	}

	PUGI__FN xml_text& xml_text::operator=(double rhs)
	{
		set(rhs);
		return *this;
	}

	PUGI__FN xml_text& xml_text::operator=(bool rhs)
	{
		set(rhs);
		return *this;
	}

	PUGI__FN xml_node xml_text::data() const
	{
		return xml_node(_data());
	}

#ifdef __BORLANDC__
	PUGI__FN bool operator&&(const xml_text& lhs, bool rhs)
	{
		return (bool)lhs && rhs;
	}

	PUGI__FN bool operator||(const xml_text& lhs, bool rhs)
	{
		return (bool)lhs || rhs;
	}
#endif

	PUGI__FN xml_node_iterator::xml_node_iterator()
	{
	}

	PUGI__FN xml_node_iterator::xml_node_iterator(const xml_node& node) :
		_wrap(node),
		_parent(node.parent())
	{
	}

	PUGI__FN xml_node_iterator::xml_node_iterator(xml_node_struct* ref, xml_node_struct* parent) :
		_wrap(ref),
		_parent(parent)
	{
	}

	PUGI__FN bool xml_node_iterator::operator==(const xml_node_iterator& rhs) const
	{
		return _wrap._root == rhs._wrap._root && _parent._root == rhs._parent._root;
	}

	PUGI__FN bool xml_node_iterator::operator!=(const xml_node_iterator& rhs) const
	{
		return _wrap._root != rhs._wrap._root || _parent._root != rhs._parent._root;
	}

	PUGI__FN xml_node& xml_node_iterator::operator*() const
	{
		assert(_wrap._root);
		return _wrap;
	}

	PUGI__FN xml_node* xml_node_iterator::operator->() const
	{
		assert(_wrap._root);
		return const_cast<xml_node*>(&_wrap); // BCC32 workaround
	}

	PUGI__FN const xml_node_iterator& xml_node_iterator::operator++()
	{
		assert(_wrap._root);
		_wrap._root = _wrap._root->next_sibling;
		return *this;
	}

	PUGI__FN xml_node_iterator xml_node_iterator::operator++(int)
	{
		xml_node_iterator temp = *this;
		++*this;
		return temp;
	}

	PUGI__FN const xml_node_iterator& xml_node_iterator::operator--()
	{
		_wrap = _wrap._root ? _wrap.previous_sibling() : _parent.last_child();
		return *this;
	}

	PUGI__FN xml_node_iterator xml_node_iterator::operator--(int)
	{
		xml_node_iterator temp = *this;
		--*this;
		return temp;
	}

	PUGI__FN xml_attribute_iterator::xml_attribute_iterator()
	{
	}

	PUGI__FN xml_attribute_iterator::xml_attribute_iterator(const xml_attribute& attr, const xml_node& parent) :
		_wrap(attr),
		_parent(parent)
	{
	}

	PUGI__FN xml_attribute_iterator::xml_attribute_iterator(xml_attribute_struct* ref, xml_node_struct* parent) :
		_wrap(ref),
		_parent(parent)
	{
	}

	PUGI__FN bool xml_attribute_iterator::operator==(const xml_attribute_iterator& rhs) const
	{
		return _wrap._attr == rhs._wrap._attr && _parent._root == rhs._parent._root;
	}

	PUGI__FN bool xml_attribute_iterator::operator!=(const xml_attribute_iterator& rhs) const
	{
		return _wrap._attr != rhs._wrap._attr || _parent._root != rhs._parent._root;
	}

	PUGI__FN xml_attribute& xml_attribute_iterator::operator*() const
	{
		assert(_wrap._attr);
		return _wrap;
	}

	PUGI__FN xml_attribute* xml_attribute_iterator::operator->() const
	{
		assert(_wrap._attr);
		return const_cast<xml_attribute*>(&_wrap); // BCC32 workaround
	}

	PUGI__FN const xml_attribute_iterator& xml_attribute_iterator::operator++()
	{
		assert(_wrap._attr);
		_wrap._attr = _wrap._attr->next_attribute;
		return *this;
	}

	PUGI__FN xml_attribute_iterator xml_attribute_iterator::operator++(int)
	{
		xml_attribute_iterator temp = *this;
		++*this;
		return temp;
	}

	PUGI__FN const xml_attribute_iterator& xml_attribute_iterator::operator--()
	{
		_wrap = _wrap._attr ? _wrap.previous_attribute() : _parent.last_attribute();
		return *this;
	}

	PUGI__FN xml_attribute_iterator xml_attribute_iterator::operator--(int)
	{
		xml_attribute_iterator temp = *this;
		--*this;
		return temp;
	}

	PUGI__FN xml_named_node_iterator::xml_named_node_iterator() :
		_name(0)
	{
	}

	PUGI__FN xml_named_node_iterator::xml_named_node_iterator(const xml_node& node, const char_t* name) :
		_node(node),
		_name(name)
	{
	}

	PUGI__FN bool xml_named_node_iterator::operator==(const xml_named_node_iterator& rhs) const
	{
		return _node == rhs._node;
	}

	PUGI__FN bool xml_named_node_iterator::operator!=(const xml_named_node_iterator& rhs) const
	{
		return _node != rhs._node;
	}

	PUGI__FN xml_node& xml_named_node_iterator::operator*() const
	{
		assert(_node._root);
		return _node;
	}

	PUGI__FN xml_node* xml_named_node_iterator::operator->() const
	{
		assert(_node._root);
		return const_cast<xml_node*>(&_node); // BCC32 workaround
	}

	PUGI__FN const xml_named_node_iterator& xml_named_node_iterator::operator++()
	{
		assert(_node._root);
		_node = _node.next_sibling(_name);
		return *this;
	}

	PUGI__FN xml_named_node_iterator xml_named_node_iterator::operator++(int)
	{
		xml_named_node_iterator temp = *this;
		++*this;
		return temp;
	}

	PUGI__FN xml_parse_result::xml_parse_result() :
		status(status_internal_error),
		offset(0),
		encoding(encoding_auto)
	{
	}

	PUGI__FN xml_parse_result::operator bool() const
	{
		return status == status_ok;
	}

	PUGI__FN const char* xml_parse_result::description() const
	{
		switch (status)
		{
		case status_ok: return "No error";

		case status_file_not_found: return "File was not found";
		case status_io_error: return "Error reading from file/stream";
		case status_out_of_memory: return "Could not allocate memory";
		case status_internal_error: return "Internal error occurred";

		case status_unrecognized_tag: return "Could not determine tag type";

		case status_bad_pi: return "Error parsing document declaration/processing instruction";
		case status_bad_comment: return "Error parsing comment";
		case status_bad_cdata: return "Error parsing CDATA section";
		case status_bad_doctype: return "Error parsing document type declaration";
		case status_bad_pcdata: return "Error parsing PCDATA section";
		case status_bad_start_element: return "Error parsing start element tag";
		case status_bad_attribute: return "Error parsing element attribute";
		case status_bad_end_element: return "Error parsing end element tag";
		case status_end_element_mismatch: return "Start-end tags mismatch";

		default: return "Unknown error";
		}
	}

	PUGI__FN xml_document::xml_document() :
		_buffer(0)
	{
		create();
	}

	PUGI__FN xml_document::~xml_document()
	{
		destroy();
	}

	PUGI__FN void xml_document::reset()
	{
		destroy();
		create();
	}

	PUGI__FN void xml_document::reset(const xml_document& proto)
	{
		reset();

		for (xml_node cur = proto.first_child(); cur; cur = cur.next_sibling())
			append_copy(cur);
	}

	PUGI__FN void xml_document::create()
	{
		// initialize sentinel page
		PUGI__STATIC_ASSERT(
			offsetof(impl::xml_memory_page, data) + sizeof(impl::xml_document_struct) +
				impl::xml_memory_page_alignment <=
			sizeof(_memory));

		// align upwards to page boundary
		void* page_memory = reinterpret_cast<void*>(
			(reinterpret_cast<uintptr_t>(_memory) + (impl::xml_memory_page_alignment - 1)) &
			~(impl::xml_memory_page_alignment - 1));

		// prepare page structure
		impl::xml_memory_page* page = impl::xml_memory_page::construct(page_memory);

		page->busy_size = impl::xml_memory_page_size;

		// allocate new root
		_root = new (page->data) impl::xml_document_struct(page);
		_root->prev_sibling_c = _root;

		// setup sentinel page
		page->allocator = static_cast<impl::xml_document_struct*>(_root);
	}

	PUGI__FN void xml_document::destroy()
	{
		// destroy static storage
		if (_buffer)
		{
			impl::xml_memory::deallocate(_buffer);
			_buffer = 0;
		}

		// destroy dynamic storage, leave sentinel page (it's in static memory)
		if (_root)
		{
			impl::xml_memory_page* root_page =
				reinterpret_cast<impl::xml_memory_page*>(_root->header & impl::xml_memory_page_pointer_mask);
			assert(root_page && !root_page->prev && !root_page->memory);

			// destroy all pages
			for (impl::xml_memory_page* page = root_page->next; page;)
			{
				impl::xml_memory_page* next = page->next;

				impl::xml_allocator::deallocate_page(page);

				page = next;
			}

			// cleanup root page
			root_page->allocator = 0;
			root_page->next = 0;
			root_page->busy_size = root_page->freed_size = 0;

			_root = 0;
		}
	}

#ifndef PUGIXML_NO_STL
	PUGI__FN xml_parse_result xml_document::load(
		std::basic_istream<char, std::char_traits<char>>& stream,
		unsigned int options,
		xml_encoding encoding)
	{
		reset();

		return impl::load_stream_impl(*this, stream, options, encoding);
	}

	PUGI__FN xml_parse_result
	xml_document::load(std::basic_istream<wchar_t, std::char_traits<wchar_t>>& stream, unsigned int options)
	{
		reset();

		return impl::load_stream_impl(*this, stream, options, encoding_wchar);
	}
#endif

	PUGI__FN xml_parse_result xml_document::load(const char_t* contents, unsigned int options)
	{
		// Force native encoding (skip autodetection)
#ifdef PUGIXML_WCHAR_MODE
		xml_encoding encoding = encoding_wchar;
#else
		xml_encoding encoding = encoding_utf8;
#endif

		return load_buffer(contents, impl::strlength(contents) * sizeof(char_t), options, encoding);
	}

	PUGI__FN xml_parse_result xml_document::load_file(const char* path_, unsigned int options, xml_encoding encoding)
	{
		reset();

		FILE* file = fopen(path_, "rb");

		return impl::load_file_impl(*this, file, options, encoding);
	}

	PUGI__FN xml_parse_result xml_document::load_file(const wchar_t* path_, unsigned int options, xml_encoding encoding)
	{
		reset();

		FILE* file = impl::open_file_wide(path_, L"rb");

		return impl::load_file_impl(*this, file, options, encoding);
	}

	PUGI__FN xml_parse_result xml_document::load_buffer_impl(
		void* contents,
		size_t size,
		unsigned int options,
		xml_encoding encoding,
		bool is_mutable,
		bool own)
	{
		reset();

		// check input buffer
		assert(contents || size == 0);

		// get actual encoding
		xml_encoding buffer_encoding = impl::get_buffer_encoding(encoding, contents, size);

		// get private buffer
		char_t* buffer = nullptr;
		size_t length = 0;

		if (!impl::convert_buffer(buffer, length, buffer_encoding, contents, size, is_mutable))
			return impl::make_parse_result(status_out_of_memory);

		// delete original buffer if we performed a conversion
		if (own && buffer != contents && contents)
			impl::xml_memory::deallocate(contents);

		// parse
		xml_parse_result res = impl::xml_parser::parse(buffer, length, _root, options);

		// remember encoding
		res.encoding = buffer_encoding;

		// grab onto buffer if it's our buffer, user is responsible for deallocating contens himself
		if (own || buffer != contents)
			_buffer = buffer;

		return res;
	}

	PUGI__FN xml_parse_result
	xml_document::load_buffer(const void* contents, size_t size, unsigned int options, xml_encoding encoding)
	{
		return load_buffer_impl(const_cast<void*>(contents), size, options, encoding, false, false);
	}

	PUGI__FN xml_parse_result
	xml_document::load_buffer_inplace(void* contents, size_t size, unsigned int options, xml_encoding encoding)
	{
		return load_buffer_impl(contents, size, options, encoding, true, false);
	}

	PUGI__FN xml_parse_result
	xml_document::load_buffer_inplace_own(void* contents, size_t size, unsigned int options, xml_encoding encoding)
	{
		return load_buffer_impl(contents, size, options, encoding, true, true);
	}

	PUGI__FN void xml_document::save(
		xml_writer& writer,
		const char_t* indent,
		unsigned int flags,
		xml_encoding encoding) const
	{
		impl::xml_buffered_writer buffered_writer(writer, encoding);

		if ((flags & format_write_bom) && encoding != encoding_latin1)
		{
			// BOM always represents the codepoint U+FEFF, so just write it in native encoding
#ifdef PUGIXML_WCHAR_MODE
			unsigned int bom = 0xfeff;
			buffered_writer.write(static_cast<wchar_t>(bom));
#else
			buffered_writer.write('\xef', '\xbb', '\xbf');
#endif
		}

		if (!(flags & format_no_declaration) && !impl::has_declaration(*this))
		{
			buffered_writer.write(PUGIXML_TEXT("<?xml version=\"1.0\""));
			if (encoding == encoding_latin1)
				buffered_writer.write(PUGIXML_TEXT(" encoding=\"ISO-8859-1\""));
			buffered_writer.write('?', '>');
			if (!(flags & format_raw))
			{
				buffered_writer.write('\n');
			}
		}

		impl::node_output(buffered_writer, *this, indent, flags, 0);
	}

#ifndef PUGIXML_NO_STL
	PUGI__FN void xml_document::save(
		std::basic_ostream<char, std::char_traits<char>>& stream,
		const char_t* indent,
		unsigned int flags,
		xml_encoding encoding) const
	{
		xml_writer_stream writer(stream);

		save(writer, indent, flags, encoding);
	}

	PUGI__FN void xml_document::save(
		std::basic_ostream<wchar_t, std::char_traits<wchar_t>>& stream,
		const char_t* indent,
		unsigned int flags) const
	{
		xml_writer_stream writer(stream);

		save(writer, indent, flags, encoding_wchar);
	}
#endif

	PUGI__FN bool xml_document::save_file(
		const char* path_,
		const char_t* indent,
		unsigned int flags,
		xml_encoding encoding) const
	{
		FILE* file = fopen(path_, (flags & format_save_file_text) ? "w" : "wb");
		return impl::save_file_impl(*this, file, indent, flags, encoding);
	}

	PUGI__FN bool xml_document::save_file(
		const wchar_t* path_,
		const char_t* indent,
		unsigned int flags,
		xml_encoding encoding) const
	{
		FILE* file = impl::open_file_wide(path_, (flags & format_save_file_text) ? L"w" : L"wb");
		return impl::save_file_impl(*this, file, indent, flags, encoding);
	}

	PUGI__FN xml_node xml_document::document_element() const
	{
		for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
			if ((i->header & impl::xml_memory_page_type_mask) + 1 == node_element)
				return xml_node(i);

		return xml_node();
	}

#ifndef PUGIXML_NO_STL
	PUGI__FN std::string PUGIXML_FUNCTION as_utf8(const wchar_t* str)
	{
		assert(str);

		return impl::as_utf8_impl(str, wcslen(str));
	}

	PUGI__FN std::string PUGIXML_FUNCTION as_utf8(const std::basic_string<wchar_t>& str)
	{
		return impl::as_utf8_impl(str.c_str(), str.size());
	}

	PUGI__FN std::basic_string<wchar_t> PUGIXML_FUNCTION as_wide(const char* str)
	{
		assert(str);

		return impl::as_wide_impl(str, strlen(str));
	}

	PUGI__FN std::basic_string<wchar_t> PUGIXML_FUNCTION as_wide(const std::string& str)
	{
		return impl::as_wide_impl(str.c_str(), str.size());
	}
#endif

	PUGI__FN void PUGIXML_FUNCTION
	set_memory_management_functions(allocation_function allocate, deallocation_function deallocate)
	{
		impl::xml_memory::allocate = allocate;
		impl::xml_memory::deallocate = deallocate;
	}

	PUGI__FN allocation_function PUGIXML_FUNCTION get_memory_allocation_function()
	{
		return impl::xml_memory::allocate;
	}

	PUGI__FN deallocation_function PUGIXML_FUNCTION get_memory_deallocation_function()
	{
		return impl::xml_memory::deallocate;
	}
}

#if !defined(PUGIXML_NO_STL) && (defined(_MSC_VER) || defined(__ICC))
namespace std
{
	// Workarounds for (non-standard) iterator category detection for older versions (MSVC7/IC8 and earlier)
	PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_node_iterator&)
	{
		return std::bidirectional_iterator_tag();
	}

	PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_attribute_iterator&)
	{
		return std::bidirectional_iterator_tag();
	}

	PUGI__FN std::forward_iterator_tag _Iter_cat(const pugi::xml_named_node_iterator&)
	{
		return std::forward_iterator_tag();
	}
}
#endif

#if !defined(PUGIXML_NO_STL) && defined(__SUNPRO_CC)
namespace std
{
	// Workarounds for (non-standard) iterator category detection
	PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_node_iterator&)
	{
		return std::bidirectional_iterator_tag();
	}

	PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_attribute_iterator&)
	{
		return std::bidirectional_iterator_tag();
	}

	PUGI__FN std::forward_iterator_tag __iterator_category(const pugi::xml_named_node_iterator&)
	{
		return std::forward_iterator_tag();
	}
}
#endif

#ifndef PUGIXML_NO_XPATH

// STL replacements
PUGI__NS_BEGIN
struct equal_to
{
	template<typename T>
	bool operator()(const T& lhs, const T& rhs) const
	{
		return lhs == rhs;
	}
};

struct not_equal_to
{
	template<typename T>
	bool operator()(const T& lhs, const T& rhs) const
	{
		return lhs != rhs;
	}
};

struct less
{
	template<typename T>
	bool operator()(const T& lhs, const T& rhs) const
	{
		return lhs < rhs;
	}
};

struct less_equal
{
	template<typename T>
	bool operator()(const T& lhs, const T& rhs) const
	{
		return lhs <= rhs;
	}
};

template<typename T>
void swap(T& lhs, T& rhs)
{
	T temp = lhs;
	lhs = rhs;
	rhs = temp;
}

template<typename I, typename Pred>
I min_element(I begin, I end, const Pred& pred)
{
	I result = begin;

	for (I it = begin + 1; it != end; ++it)
		if (pred(*it, *result))
			result = it;

	return result;
}

template<typename I>
void reverse(I begin, I end)
{
	while (begin + 1 < end)
		swap(*begin++, *--end);
}

template<typename I>
I unique(I begin, I end)
{
	// fast skip head
	while (begin + 1 < end && *begin != *(begin + 1))
		begin++;

	if (begin == end)
		return begin;

	// last written element
	I write = begin++;

	// merge unique elements
	while (begin != end)
	{
		if (*begin != *write)
			*++write = *begin++;
		else
			begin++;
	}

	// past-the-end (write points to live element)
	return write + 1;
}

template<typename I>
void copy_backwards(I begin, I end, I target)
{
	while (begin != end)
		*--target = *--end;
}

template<typename I, typename Pred, typename T>
void insertion_sort(I begin, I end, const Pred& pred, T*)
{
	assert(begin != end);

	for (I it = begin + 1; it != end; ++it)
	{
		T val = *it;

		if (pred(val, *begin))
		{
			// move to front
			copy_backwards(begin, it, it + 1);
			*begin = val;
		}
		else
		{
			I hole = it;

			// move hole backwards
			while (pred(val, *(hole - 1)))
			{
				*hole = *(hole - 1);
				hole--;
			}

			// fill hole with element
			*hole = val;
		}
	}
}

// std variant for elements with ==
template<typename I, typename Pred>
void partition(I begin, I middle, I end, const Pred& pred, I* out_eqbeg, I* out_eqend)
{
	I eqbeg = middle, eqend = middle + 1;

	// expand equal range
	while (eqbeg != begin && *(eqbeg - 1) == *eqbeg)
		--eqbeg;
	while (eqend != end && *eqend == *eqbeg)
		++eqend;

	// process outer elements
	I ltend = eqbeg, gtbeg = eqend;

	for (;;)
	{
		// find the element from the right side that belongs to the left one
		for (; gtbeg != end; ++gtbeg)
			if (!pred(*eqbeg, *gtbeg))
			{
				if (*gtbeg == *eqbeg)
					swap(*gtbeg, *eqend++);
				else
					break;
			}

		// find the element from the left side that belongs to the right one
		for (; ltend != begin; --ltend)
			if (!pred(*(ltend - 1), *eqbeg))
			{
				if (*eqbeg == *(ltend - 1))
					swap(*(ltend - 1), *--eqbeg);
				else
					break;
			}

		// scanned all elements
		if (gtbeg == end && ltend == begin)
		{
			*out_eqbeg = eqbeg;
			*out_eqend = eqend;
			return;
		}

		// make room for elements by moving equal area
		if (gtbeg == end)
		{
			if (--ltend != --eqbeg)
				swap(*ltend, *eqbeg);
			swap(*eqbeg, *--eqend);
		}
		else if (ltend == begin)
		{
			if (eqend != gtbeg)
				swap(*eqbeg, *eqend);
			++eqend;
			swap(*gtbeg++, *eqbeg++);
		}
		else
			swap(*gtbeg++, *--ltend);
	}
}

template<typename I, typename Pred>
void median3(I first, I middle, I last, const Pred& pred)
{
	if (pred(*middle, *first))
		swap(*middle, *first);
	if (pred(*last, *middle))
		swap(*last, *middle);
	if (pred(*middle, *first))
		swap(*middle, *first);
}

template<typename I, typename Pred>
void median(I first, I middle, I last, const Pred& pred)
{
	if (last - first <= 40)
	{
		// median of three for small chunks
		median3(first, middle, last, pred);
	}
	else
	{
		// median of nine
		size_t step = (last - first + 1) / 8;

		median3(first, first + step, first + 2 * step, pred);
		median3(middle - step, middle, middle + step, pred);
		median3(last - 2 * step, last - step, last, pred);
		median3(first + step, middle, last - step, pred);
	}
}

template<typename I, typename Pred>
void sort(I begin, I end, const Pred& pred)
{
	// sort large chunks
	while (end - begin > 32)
	{
		// find median element
		I middle = begin + (end - begin) / 2;
		median(begin, middle, end - 1, pred);

		// partition in three chunks (< = >)
		I eqbeg, eqend;
		partition(begin, middle, end, pred, &eqbeg, &eqend);

		// loop on larger half
		if (eqbeg - begin > end - eqend)
		{
			sort(eqend, end, pred);
			end = eqbeg;
		}
		else
		{
			sort(begin, eqbeg, pred);
			begin = eqend;
		}
	}

	// insertion sort small chunk
	if (begin != end)
		insertion_sort(begin, end, pred, &*begin);
}
PUGI__NS_END

// Allocator used for AST and evaluation stacks
PUGI__NS_BEGIN
struct xpath_memory_block
{
	xpath_memory_block* next;

	char data[
	#ifdef PUGIXML_MEMORY_XPATH_PAGE_SIZE
		PUGIXML_MEMORY_XPATH_PAGE_SIZE
	#else
		4096
	#endif
	];
};

class xpath_allocator
{
	xpath_memory_block* _root;
	size_t _root_size;

public:
	#ifdef PUGIXML_NO_EXCEPTIONS
	jmp_buf* error_handler;
	#endif

	xpath_allocator(xpath_memory_block* root, size_t root_size = 0) :
		_root(root),
		_root_size(root_size)
	{
	#ifdef PUGIXML_NO_EXCEPTIONS
		error_handler = 0;
	#endif
	}

	void* allocate_nothrow(size_t size)
	{
		const size_t block_capacity = sizeof(_root->data);

		// align size so that we're able to store pointers in subsequent blocks
		size = (size + sizeof(void*) - 1) & ~(sizeof(void*) - 1);

		if (_root_size + size <= block_capacity)
		{
			void* buf = _root->data + _root_size;
			_root_size += size;
			return buf;
		}
		else
		{
			size_t block_data_size = (size > block_capacity) ? size : block_capacity;
			size_t block_size = block_data_size + offsetof(xpath_memory_block, data);

			xpath_memory_block* block = static_cast<xpath_memory_block*>(xml_memory::allocate(block_size));
			if (!block)
				return 0;

			block->next = _root;

			_root = block;
			_root_size = size;

			return block->data;
		}
	}

	void* allocate(size_t size)
	{
		void* result = allocate_nothrow(size);

		if (!result)
		{
	#ifdef PUGIXML_NO_EXCEPTIONS
			assert(error_handler);
			longjmp(*error_handler, 1);
	#else
			throw std::bad_alloc();
	#endif
		}

		return result;
	}

	void* reallocate(void* ptr, size_t old_size, size_t new_size)
	{
		// align size so that we're able to store pointers in subsequent blocks
		old_size = (old_size + sizeof(void*) - 1) & ~(sizeof(void*) - 1);
		new_size = (new_size + sizeof(void*) - 1) & ~(sizeof(void*) - 1);

		// we can only reallocate the last object
		assert(ptr == 0 || static_cast<char*>(ptr) + old_size == _root->data + _root_size);

		// adjust root size so that we have not allocated the object at all
		bool only_object = (_root_size == old_size);

		if (ptr)
			_root_size -= old_size;

		// allocate a new version (this will obviously reuse the memory if possible)
		void* result = allocate(new_size);
		assert(result);

		// we have a new block
		if (result != ptr && ptr)
		{
			// copy old data
			assert(new_size > old_size);
			memcpy(result, ptr, old_size);

			// free the previous page if it had no other objects
			if (only_object)
			{
				assert(_root->data == result);
				assert(_root->next);

				xpath_memory_block* next = _root->next->next;

				if (next)
				{
					// deallocate the whole page, unless it was the first one
					xml_memory::deallocate(_root->next);
					_root->next = next;
				}
			}
		}

		return result;
	}

	void revert(const xpath_allocator& state)
	{
		// free all new pages
		xpath_memory_block* cur = _root;

		while (cur != state._root)
		{
			xpath_memory_block* next = cur->next;

			xml_memory::deallocate(cur);

			cur = next;
		}

		// restore state
		_root = state._root;
		_root_size = state._root_size;
	}

	void release()
	{
		xpath_memory_block* cur = _root;
		assert(cur);

		while (cur->next)
		{
			xpath_memory_block* next = cur->next;

			xml_memory::deallocate(cur);

			cur = next;
		}
	}
};

struct xpath_allocator_capture
{
	xpath_allocator_capture(xpath_allocator* alloc) :
		_target(alloc),
		_state(*alloc)
	{
	}

	~xpath_allocator_capture()
	{
		_target->revert(_state);
	}

	xpath_allocator* _target;
	xpath_allocator _state;
};

struct xpath_stack
{
	xpath_allocator* result;
	xpath_allocator* temp;
};

struct xpath_stack_data
{
	xpath_memory_block blocks[2];
	xpath_allocator result;
	xpath_allocator temp;
	xpath_stack stack;

	#ifdef PUGIXML_NO_EXCEPTIONS
	jmp_buf error_handler;
	#endif

	xpath_stack_data() :
		result(blocks + 0),
		temp(blocks + 1)
	{
		blocks[0].next = blocks[1].next = 0;

		stack.result = &result;
		stack.temp = &temp;

	#ifdef PUGIXML_NO_EXCEPTIONS
		result.error_handler = temp.error_handler = &error_handler;
	#endif
	}

	~xpath_stack_data()
	{
		result.release();
		temp.release();
	}
};
PUGI__NS_END

// String class
PUGI__NS_BEGIN
class xpath_string
{
	const char_t* _buffer;
	bool _uses_heap;

	static char_t* duplicate_string(const char_t* string, size_t length, xpath_allocator* alloc)
	{
		char_t* result = static_cast<char_t*>(alloc->allocate((length + 1) * sizeof(char_t)));
		assert(result);

		memcpy(result, string, length * sizeof(char_t));
		result[length] = 0;

		return result;
	}

	static char_t* duplicate_string(const char_t* string, xpath_allocator* alloc)
	{
		return duplicate_string(string, strlength(string), alloc);
	}

public:
	xpath_string() :
		_buffer(PUGIXML_TEXT("")),
		_uses_heap(false)
	{
	}

	explicit xpath_string(const char_t* str, xpath_allocator* alloc)
	{
		bool empty_ = (*str == 0);

		_buffer = empty_ ? PUGIXML_TEXT("") : duplicate_string(str, alloc);
		_uses_heap = !empty_;
	}

	explicit xpath_string(const char_t* str, bool use_heap) :
		_buffer(str),
		_uses_heap(use_heap)
	{
	}

	xpath_string(const char_t* begin, const char_t* end, xpath_allocator* alloc)
	{
		assert(begin <= end);

		bool empty_ = (begin == end);

		_buffer = empty_ ? PUGIXML_TEXT("") : duplicate_string(begin, static_cast<size_t>(end - begin), alloc);
		_uses_heap = !empty_;
	}

	void append(const xpath_string& o, xpath_allocator* alloc)
	{
		// skip empty sources
		if (!*o._buffer)
			return;

		// fast append for constant empty target and constant source
		if (!*_buffer && !_uses_heap && !o._uses_heap)
		{
			_buffer = o._buffer;
		}
		else
		{
			// need to make heap copy
			size_t target_length = strlength(_buffer);
			size_t source_length = strlength(o._buffer);
			size_t result_length = target_length + source_length;

			// allocate new buffer
			char_t* result = static_cast<char_t*>(alloc->reallocate(
				_uses_heap ? const_cast<char_t*>(_buffer) : 0,
				(target_length + 1) * sizeof(char_t),
				(result_length + 1) * sizeof(char_t)));
			assert(result);

			// append first string to the new buffer in case there was no reallocation
			if (!_uses_heap)
				memcpy(result, _buffer, target_length * sizeof(char_t));

			// append second string to the new buffer
			memcpy(result + target_length, o._buffer, source_length * sizeof(char_t));
			result[result_length] = 0;

			// finalize
			_buffer = result;
			_uses_heap = true;
		}
	}

	const char_t* c_str() const
	{
		return _buffer;
	}

	size_t length() const
	{
		return strlength(_buffer);
	}

	char_t* data(xpath_allocator* alloc)
	{
		// make private heap copy
		if (!_uses_heap)
		{
			_buffer = duplicate_string(_buffer, alloc);
			_uses_heap = true;
		}

		return const_cast<char_t*>(_buffer);
	}

	bool empty() const
	{
		return *_buffer == 0;
	}

	bool operator==(const xpath_string& o) const
	{
		return strequal(_buffer, o._buffer);
	}

	bool operator!=(const xpath_string& o) const
	{
		return !strequal(_buffer, o._buffer);
	}

	bool uses_heap() const
	{
		return _uses_heap;
	}
};

PUGI__FN xpath_string xpath_string_const(const char_t* str)
{
	return xpath_string(str, false);
}
PUGI__NS_END

PUGI__NS_BEGIN
PUGI__FN bool starts_with(const char_t* string, const char_t* pattern)
{
	while (*pattern && *string == *pattern)
	{
		string++;
		pattern++;
	}

	return *pattern == 0;
}

PUGI__FN const char_t* find_char(const char_t* s, char_t c)
{
	#ifdef PUGIXML_WCHAR_MODE
	return wcschr(s, c);
	#else
	return strchr(s, c);
	#endif
}

PUGI__FN const char_t* find_substring(const char_t* s, const char_t* p)
{
	#ifdef PUGIXML_WCHAR_MODE
	// MSVC6 wcsstr bug workaround (if s is empty it always returns 0)
	return (*p == 0) ? s : wcsstr(s, p);
	#else
	return strstr(s, p);
	#endif
}

// Converts symbol to lower case, if it is an ASCII one
PUGI__FN char_t tolower_ascii(char_t ch)
{
	return static_cast<unsigned int>(ch - 'A') < 26 ? static_cast<char_t>(ch | ' ') : ch;
}

PUGI__FN xpath_string string_value(const xpath_node& na, xpath_allocator* alloc)
{
	if (na.attribute())
		return xpath_string_const(na.attribute().value());
	else
	{
		const xml_node& n = na.node();

		switch (n.type())
		{
		case node_pcdata:
		case node_cdata:
		case node_comment:
		case node_pi: return xpath_string_const(n.value());

		case node_document:
		case node_element:
		{
			xpath_string result;

			xml_node cur = n.first_child();

			while (cur && cur != n)
			{
				if (cur.type() == node_pcdata || cur.type() == node_cdata)
					result.append(xpath_string_const(cur.value()), alloc);

				if (cur.first_child())
					cur = cur.first_child();
				else if (cur.next_sibling())
					cur = cur.next_sibling();
				else
				{
					while (!cur.next_sibling() && cur != n)
						cur = cur.parent();

					if (cur != n)
						cur = cur.next_sibling();
				}
			}

			return result;
		}

		default: return xpath_string();
		}
	}
}

PUGI__FN unsigned int node_height(xml_node n)
{
	unsigned int result = 0;

	while (n)
	{
		++result;
		n = n.parent();
	}

	return result;
}

PUGI__FN bool node_is_before(xml_node ln, unsigned int lh, xml_node rn, unsigned int rh)
{
	// normalize heights
	for (unsigned int i = rh; i < lh; i++)
		ln = ln.parent();
	for (unsigned int j = lh; j < rh; j++)
		rn = rn.parent();

	// one node is the ancestor of the other
	if (ln == rn)
		return lh < rh;

	// find common ancestor
	while (ln.parent() != rn.parent())
	{
		ln = ln.parent();
		rn = rn.parent();
	}

	// there is no common ancestor (the shared parent is null), nodes are from different documents
	if (!ln.parent())
		return ln < rn;

	// determine sibling order
	for (; ln; ln = ln.next_sibling())
		if (ln == rn)
			return true;

	return false;
}

PUGI__FN bool node_is_ancestor(xml_node parent, xml_node node)
{
	while (node && node != parent)
		node = node.parent();

	return parent && node == parent;
}

PUGI__FN const void* document_order(const xpath_node& xnode)
{
	xml_node_struct* node = xnode.node().internal_object();

	if (node)
	{
		if (node->name && (node->header & xml_memory_page_name_allocated_mask) == 0)
			return node->name;
		if (node->value && (node->header & xml_memory_page_value_allocated_mask) == 0)
			return node->value;
		return 0;
	}

	xml_attribute_struct* attr = xnode.attribute().internal_object();

	if (attr)
	{
		if ((attr->header & xml_memory_page_name_allocated_mask) == 0)
			return attr->name;
		if ((attr->header & xml_memory_page_value_allocated_mask) == 0)
			return attr->value;
		return 0;
	}

	return 0;
}

struct document_order_comparator
{
	bool operator()(const xpath_node& lhs, const xpath_node& rhs) const
	{
		// optimized document order based check
		const void* lo = document_order(lhs);
		const void* ro = document_order(rhs);

		if (lo && ro)
			return lo < ro;

		// slow comparison
		xml_node ln = lhs.node(), rn = rhs.node();

		// compare attributes
		if (lhs.attribute() && rhs.attribute())
		{
			// shared parent
			if (lhs.parent() == rhs.parent())
			{
				// determine sibling order
				for (xml_attribute a = lhs.attribute(); a; a = a.next_attribute())
					if (a == rhs.attribute())
						return true;

				return false;
			}

			// compare attribute parents
			ln = lhs.parent();
			rn = rhs.parent();
		}
		else if (lhs.attribute())
		{
			// attributes go after the parent element
			if (lhs.parent() == rhs.node())
				return false;

			ln = lhs.parent();
		}
		else if (rhs.attribute())
		{
			// attributes go after the parent element
			if (rhs.parent() == lhs.node())
				return true;

			rn = rhs.parent();
		}

		if (ln == rn)
			return false;

		unsigned int lh = node_height(ln);
		unsigned int rh = node_height(rn);

		return node_is_before(ln, lh, rn, rh);
	}
};

struct duplicate_comparator
{
	bool operator()(const xpath_node& lhs, const xpath_node& rhs) const
	{
		if (lhs.attribute())
			return rhs.attribute() ? lhs.attribute() < rhs.attribute() : true;
		else
			return rhs.attribute() ? false : lhs.node() < rhs.node();
	}
};

PUGI__FN double gen_nan()
{
	#if defined(__STDC_IEC_559__) || ((FLT_RADIX - 0 == 2) && (FLT_MAX_EXP - 0 == 128) && (FLT_MANT_DIG - 0 == 24))
	union
	{
		float f;
		uint32_t i;
	} u[sizeof(float) == sizeof(uint32_t) ? 1 : -1];
	u[0].i = 0x7fc00000;
	return u[0].f;
	#else
	// fallback
	const volatile double zero = 0.0;
	return zero / zero;
	#endif
}

PUGI__FN bool is_nan(double value)
{
	#if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__)
	return !!_isnan(value);
	#elif defined(fpclassify) && defined(FP_NAN)
	return fpclassify(value) == FP_NAN;
	#else
	// fallback
	const volatile double v = value;
	return v != v;
	#endif
}

PUGI__FN const char_t* convert_number_to_string_special(double value)
{
	#if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__)
	if (_finite(value))
		return (value == 0) ? PUGIXML_TEXT("0") : 0;
	if (_isnan(value))
		return PUGIXML_TEXT("NaN");
	return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
	#elif defined(fpclassify) && defined(FP_NAN) && defined(FP_INFINITE) && defined(FP_ZERO)
	switch (fpclassify(value))
	{
	case FP_NAN: return PUGIXML_TEXT("NaN");

	case FP_INFINITE: return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");

	case FP_ZERO: return PUGIXML_TEXT("0");

	default: return 0;
	}
	#else
	// fallback
	const volatile double v = value;

	if (v == 0)
		return PUGIXML_TEXT("0");
	if (v != v)
		return PUGIXML_TEXT("NaN");
	if (v * 2 == v)
		return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
	return 0;
	#endif
}

PUGI__FN bool convert_number_to_boolean(double value)
{
	return (value != 0 && !is_nan(value));
}

PUGI__FN void truncate_zeros(char* begin, char* end)
{
	while (begin != end && end[-1] == '0')
		end--;

	*end = 0;
}

// gets mantissa digits in the form of 0.xxxxx with 0. implied and the exponent
	#if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 && !defined(_WIN32_WCE)
PUGI__FN void convert_number_to_mantissa_exponent(
	double value,
	char* buffer,
	size_t buffer_size,
	char** out_mantissa,
	int* out_exponent)
{
	// get base values
	int sign, exponent;
	_ecvt_s(buffer, buffer_size, value, DBL_DIG + 1, &exponent, &sign);

	// truncate redundant zeros
	truncate_zeros(buffer, buffer + strlen(buffer));

	// fill results
	*out_mantissa = buffer;
	*out_exponent = exponent;
}
	#else
PUGI__FN void convert_number_to_mantissa_exponent(
	double value,
	char* buffer,
	size_t buffer_size,
	char** out_mantissa,
	int* out_exponent)
{
	// get a scientific notation value with IEEE DBL_DIG decimals
	sprintf(buffer, "%.*e", DBL_DIG, value);
	assert(strlen(buffer) < buffer_size);
	(void)!buffer_size;

	// get the exponent (possibly negative)
	char* exponent_string = strchr(buffer, 'e');
	assert(exponent_string);

	int exponent = atoi(exponent_string + 1);

	// extract mantissa string: skip sign
	char* mantissa = buffer[0] == '-' ? buffer + 1 : buffer;
	assert(mantissa[0] != '0' && mantissa[1] == '.');

	// divide mantissa by 10 to eliminate integer part
	mantissa[1] = mantissa[0];
	mantissa++;
	exponent++;

	// remove extra mantissa digits and zero-terminate mantissa
	truncate_zeros(mantissa, exponent_string);

	// fill results
	*out_mantissa = mantissa;
	*out_exponent = exponent;
}
	#endif

PUGI__FN xpath_string convert_number_to_string(double value, xpath_allocator* alloc)
{
	// try special number conversion
	const char_t* special = convert_number_to_string_special(value);
	if (special)
		return xpath_string_const(special);

	// get mantissa + exponent form
	char mantissa_buffer[64];

	char* mantissa;
	int exponent;
	convert_number_to_mantissa_exponent(value, mantissa_buffer, sizeof(mantissa_buffer), &mantissa, &exponent);

	// make the number!
	char_t result[512];
	char_t* s = result;

	// sign
	if (value < 0)
		*s++ = '-';

	// integer part
	if (exponent <= 0)
	{
		*s++ = '0';
	}
	else
	{
		while (exponent > 0)
		{
			assert(*mantissa == 0 || static_cast<unsigned int>(*mantissa - '0') <= 9);
			*s++ = *mantissa ? *mantissa++ : '0';
			exponent--;
		}
	}

	// fractional part
	if (*mantissa)
	{
		// decimal point
		*s++ = '.';

		// extra zeroes from negative exponent
		while (exponent < 0)
		{
			*s++ = '0';
			exponent++;
		}

		// extra mantissa digits
		while (*mantissa)
		{
			assert(static_cast<unsigned int>(*mantissa - '0') <= 9);
			*s++ = *mantissa++;
		}
	}

	// zero-terminate
	assert(s < result + sizeof(result) / sizeof(result[0]));
	*s = 0;

	return xpath_string(result, alloc);
}

PUGI__FN bool check_string_to_number_format(const char_t* string)
{
	// parse leading whitespace
	while (PUGI__IS_CHARTYPE(*string, ct_space))
		++string;

	// parse sign
	if (*string == '-')
		++string;

	if (!*string)
		return false;

	// if there is no integer part, there should be a decimal part with at least one digit
	if (!PUGI__IS_CHARTYPEX(string[0], ctx_digit) && (string[0] != '.' || !PUGI__IS_CHARTYPEX(string[1], ctx_digit)))
		return false;

	// parse integer part
	while (PUGI__IS_CHARTYPEX(*string, ctx_digit))
		++string;

	// parse decimal part
	if (*string == '.')
	{
		++string;

		while (PUGI__IS_CHARTYPEX(*string, ctx_digit))
			++string;
	}

	// parse trailing whitespace
	while (PUGI__IS_CHARTYPE(*string, ct_space))
		++string;

	return *string == 0;
}

PUGI__FN double convert_string_to_number(const char_t* string)
{
	// check string format
	if (!check_string_to_number_format(string))
		return gen_nan();

		// parse string
	#ifdef PUGIXML_WCHAR_MODE
	return wcstod(string, 0);
	#else
	return atof(string);
	#endif
}

PUGI__FN bool convert_string_to_number(const char_t* begin, const char_t* end, double* out_result)
{
	char_t buffer[32];

	size_t length = static_cast<size_t>(end - begin);
	char_t* scratch = buffer;

	if (length >= sizeof(buffer) / sizeof(buffer[0]))
	{
		// need to make dummy on-heap copy
		scratch = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
		if (!scratch)
			return false;
	}

	// copy string to zero-terminated buffer and perform conversion
	memcpy(scratch, begin, length * sizeof(char_t));
	scratch[length] = 0;

	*out_result = convert_string_to_number(scratch);

	// free dummy buffer
	if (scratch != buffer)
		xml_memory::deallocate(scratch);

	return true;
}

PUGI__FN double round_nearest(double value)
{
	return floor(value + 0.5);
}

PUGI__FN double round_nearest_nzero(double value)
{
	// same as round_nearest, but returns -0 for [-0.5, -0]
	// ceil is used to differentiate between +0 and -0 (we return -0 for [-0.5, -0] and +0 for +0)
	return (value >= -0.5 && value <= 0) ? ceil(value) : floor(value + 0.5);
}

PUGI__FN const char_t* qualified_name(const xpath_node& node)
{
	return node.attribute() ? node.attribute().name() : node.node().name();
}

PUGI__FN const char_t* local_name(const xpath_node& node)
{
	const char_t* name = qualified_name(node);
	const char_t* p = find_char(name, ':');

	return p ? p + 1 : name;
}

struct namespace_uri_predicate
{
	const char_t* prefix;
	size_t prefix_length;

	namespace_uri_predicate(const char_t* name)
	{
		const char_t* pos = find_char(name, ':');

		prefix = pos ? name : 0;
		prefix_length = pos ? static_cast<size_t>(pos - name) : 0;
	}

	bool operator()(const xml_attribute& a) const
	{
		const char_t* name = a.name();

		if (!starts_with(name, PUGIXML_TEXT("xmlns")))
			return false;

		return prefix ? name[5] == ':' && strequalrange(name + 6, prefix, prefix_length) : name[5] == 0;
	}
};

PUGI__FN const char_t* namespace_uri(const xml_node& node)
{
	namespace_uri_predicate pred = node.name();

	xml_node p = node;

	while (p)
	{
		xml_attribute a = p.find_attribute(pred);

		if (a)
			return a.value();

		p = p.parent();
	}

	return PUGIXML_TEXT("");
}

PUGI__FN const char_t* namespace_uri(const xml_attribute& attr, const xml_node& parent)
{
	namespace_uri_predicate pred = attr.name();

	// Default namespace does not apply to attributes
	if (!pred.prefix)
		return PUGIXML_TEXT("");

	xml_node p = parent;

	while (p)
	{
		xml_attribute a = p.find_attribute(pred);

		if (a)
			return a.value();

		p = p.parent();
	}

	return PUGIXML_TEXT("");
}

PUGI__FN const char_t* namespace_uri(const xpath_node& node)
{
	return node.attribute() ? namespace_uri(node.attribute(), node.parent()) : namespace_uri(node.node());
}

PUGI__FN void normalize_space(char_t* buffer)
{
	char_t* write = buffer;

	for (char_t* it = buffer; *it;)
	{
		char_t ch = *it++;

		if (PUGI__IS_CHARTYPE(ch, ct_space))
		{
			// replace whitespace sequence with single space
			while (PUGI__IS_CHARTYPE(*it, ct_space))
				it++;

			// avoid leading spaces
			if (write != buffer)
				*write++ = ' ';
		}
		else
			*write++ = ch;
	}

	// remove trailing space
	if (write != buffer && PUGI__IS_CHARTYPE(write[-1], ct_space))
		write--;

	// zero-terminate
	*write = 0;
}

PUGI__FN void translate(char_t* buffer, const char_t* from, const char_t* to)
{
	size_t to_length = strlength(to);

	char_t* write = buffer;

	while (*buffer)
	{
		PUGI__DMC_VOLATILE char_t ch = *buffer++;

		const char_t* pos = find_char(from, ch);

		if (!pos)
			*write++ = ch; // do not process
		else if (static_cast<size_t>(pos - from) < to_length)
			*write++ = to[pos - from]; // replace
	}

	// zero-terminate
	*write = 0;
}

struct xpath_variable_boolean : xpath_variable
{
	xpath_variable_boolean() :
		value(false)
	{
	}

	bool value;
	char_t name[1];
};

struct xpath_variable_number : xpath_variable
{
	xpath_variable_number() :
		value(0)
	{
	}

	double value;
	char_t name[1];
};

struct xpath_variable_string : xpath_variable
{
	xpath_variable_string() :
		value(0)
	{
	}

	~xpath_variable_string()
	{
		if (value)
			xml_memory::deallocate(value);
	}

	char_t* value;
	char_t name[1];
};

struct xpath_variable_node_set : xpath_variable
{
	xpath_node_set value;
	char_t name[1];
};

static const xpath_node_set dummy_node_set;

PUGI__FN unsigned int hash_string(const char_t* str)
{
	// Jenkins one-at-a-time hash (http://en.wikipedia.org/wiki/Jenkins_hash_function#one-at-a-time)
	unsigned int result = 0;

	while (*str)
	{
		result += static_cast<unsigned int>(*str++);
		result += result << 10;
		result ^= result >> 6;
	}

	result += result << 3;
	result ^= result >> 11;
	result += result << 15;

	return result;
}

template<typename T>
PUGI__FN T* new_xpath_variable(const char_t* name)
{
	size_t length = strlength(name);
	if (length == 0)
		return 0; // empty variable names are invalid

	// $$ we can't use offsetof(T, name) because T is non-POD, so we just allocate additional length characters
	void* memory = xml_memory::allocate(sizeof(T) + length * sizeof(char_t));
	if (!memory)
		return 0;

	T* result = new (memory) T();

	memcpy(result->name, name, (length + 1) * sizeof(char_t));

	return result;
}

PUGI__FN xpath_variable* new_xpath_variable(xpath_value_type type, const char_t* name)
{
	switch (type)
	{
	case xpath_type_node_set: return new_xpath_variable<xpath_variable_node_set>(name);

	case xpath_type_number: return new_xpath_variable<xpath_variable_number>(name);

	case xpath_type_string: return new_xpath_variable<xpath_variable_string>(name);

	case xpath_type_boolean: return new_xpath_variable<xpath_variable_boolean>(name);

	default: return 0;
	}
}

template<typename T>
PUGI__FN void delete_xpath_variable(T* var)
{
	var->~T();
	xml_memory::deallocate(var);
}

PUGI__FN void delete_xpath_variable(xpath_value_type type, xpath_variable* var)
{
	switch (type)
	{
	case xpath_type_node_set: delete_xpath_variable(static_cast<xpath_variable_node_set*>(var)); break;

	case xpath_type_number: delete_xpath_variable(static_cast<xpath_variable_number*>(var)); break;

	case xpath_type_string: delete_xpath_variable(static_cast<xpath_variable_string*>(var)); break;

	case xpath_type_boolean: delete_xpath_variable(static_cast<xpath_variable_boolean*>(var)); break;

	default: assert(!"Invalid variable type");
	}
}

PUGI__FN xpath_variable* get_variable(xpath_variable_set* set, const char_t* begin, const char_t* end)
{
	char_t buffer[32];

	size_t length = static_cast<size_t>(end - begin);
	char_t* scratch = buffer;

	if (length >= sizeof(buffer) / sizeof(buffer[0]))
	{
		// need to make dummy on-heap copy
		scratch = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
		if (!scratch)
			return 0;
	}

	// copy string to zero-terminated buffer and perform lookup
	memcpy(scratch, begin, length * sizeof(char_t));
	scratch[length] = 0;

	xpath_variable* result = set->get(scratch);

	// free dummy buffer
	if (scratch != buffer)
		xml_memory::deallocate(scratch);

	return result;
}
PUGI__NS_END

// Internal node set class
PUGI__NS_BEGIN
PUGI__FN xpath_node_set::type_t xpath_sort(xpath_node* begin, xpath_node* end, xpath_node_set::type_t type, bool rev)
{
	xpath_node_set::type_t order = rev ? xpath_node_set::type_sorted_reverse : xpath_node_set::type_sorted;

	if (type == xpath_node_set::type_unsorted)
	{
		sort(begin, end, document_order_comparator());

		type = xpath_node_set::type_sorted;
	}

	if (type != order)
		reverse(begin, end);

	return order;
}

PUGI__FN xpath_node xpath_first(const xpath_node* begin, const xpath_node* end, xpath_node_set::type_t type)
{
	if (begin == end)
		return xpath_node();

	switch (type)
	{
	case xpath_node_set::type_sorted: return *begin;

	case xpath_node_set::type_sorted_reverse: return *(end - 1);

	case xpath_node_set::type_unsorted: return *min_element(begin, end, document_order_comparator());

	default: assert(!"Invalid node set type"); return xpath_node();
	}
}

class xpath_node_set_raw
{
	xpath_node_set::type_t _type;

	xpath_node* _begin;
	xpath_node* _end;
	xpath_node* _eos;

public:
	xpath_node_set_raw() :
		_type(xpath_node_set::type_unsorted),
		_begin(0),
		_end(0),
		_eos(0)
	{
	}

	xpath_node* begin() const
	{
		return _begin;
	}

	xpath_node* end() const
	{
		return _end;
	}

	bool empty() const
	{
		return _begin == _end;
	}

	size_t size() const
	{
		return static_cast<size_t>(_end - _begin);
	}

	xpath_node first() const
	{
		return xpath_first(_begin, _end, _type);
	}

	void push_back(const xpath_node& node, xpath_allocator* alloc)
	{
		if (_end == _eos)
		{
			size_t capacity = static_cast<size_t>(_eos - _begin);

			// get new capacity (1.5x rule)
			size_t new_capacity = capacity + capacity / 2 + 1;

			// reallocate the old array or allocate a new one
			xpath_node* data = static_cast<xpath_node*>(
				alloc->reallocate(_begin, capacity * sizeof(xpath_node), new_capacity * sizeof(xpath_node)));
			assert(data);

			// finalize
			_begin = data;
			_end = data + capacity;
			_eos = data + new_capacity;
		}

		*_end++ = node;
	}

	void append(const xpath_node* begin_, const xpath_node* end_, xpath_allocator* alloc)
	{
		size_t size_ = static_cast<size_t>(_end - _begin);
		size_t capacity = static_cast<size_t>(_eos - _begin);
		size_t count = static_cast<size_t>(end_ - begin_);

		if (size_ + count > capacity)
		{
			// reallocate the old array or allocate a new one
			xpath_node* data = static_cast<xpath_node*>(
				alloc->reallocate(_begin, capacity * sizeof(xpath_node), (size_ + count) * sizeof(xpath_node)));
			assert(data);

			// finalize
			_begin = data;
			_end = data + size_;
			_eos = data + size_ + count;
		}

		memcpy(_end, begin_, count * sizeof(xpath_node));
		_end += count;
	}

	void sort_do()
	{
		_type = xpath_sort(_begin, _end, _type, false);
	}

	void truncate(xpath_node* pos)
	{
		assert(_begin <= pos && pos <= _end);

		_end = pos;
	}

	void remove_duplicates()
	{
		if (_type == xpath_node_set::type_unsorted)
			sort(_begin, _end, duplicate_comparator());

		_end = unique(_begin, _end);
	}

	xpath_node_set::type_t type() const
	{
		return _type;
	}

	void set_type(xpath_node_set::type_t value)
	{
		_type = value;
	}
};
PUGI__NS_END

PUGI__NS_BEGIN
struct xpath_context
{
	xpath_node n;
	size_t position, size;

	xpath_context(const xpath_node& n_, size_t position_, size_t size_) :
		n(n_),
		position(position_),
		size(size_)
	{
	}
};

enum lexeme_t
{
	lex_none = 0,
	lex_equal,
	lex_not_equal,
	lex_less,
	lex_greater,
	lex_less_or_equal,
	lex_greater_or_equal,
	lex_plus,
	lex_minus,
	lex_multiply,
	lex_union,
	lex_var_ref,
	lex_open_brace,
	lex_close_brace,
	lex_quoted_string,
	lex_number,
	lex_slash,
	lex_double_slash,
	lex_open_square_brace,
	lex_close_square_brace,
	lex_string,
	lex_comma,
	lex_axis_attribute,
	lex_dot,
	lex_double_dot,
	lex_double_colon,
	lex_eof
};

struct xpath_lexer_string
{
	const char_t* begin;
	const char_t* end;

	xpath_lexer_string() :
		begin(0),
		end(0)
	{
	}

	bool operator==(const char_t* other) const
	{
		size_t length = static_cast<size_t>(end - begin);

		return strequalrange(other, begin, length);
	}
};

class xpath_lexer
{
	const char_t* _cur;
	const char_t* _cur_lexeme_pos;
	xpath_lexer_string _cur_lexeme_contents;

	lexeme_t _cur_lexeme;

public:
	explicit xpath_lexer(const char_t* query) :
		_cur(query)
	{
		next();
	}

	const char_t* state() const
	{
		return _cur;
	}

	void next()
	{
		const char_t* cur = _cur;

		while (PUGI__IS_CHARTYPE(*cur, ct_space))
			++cur;

		// save lexeme position for error reporting
		_cur_lexeme_pos = cur;

		switch (*cur)
		{
		case 0: _cur_lexeme = lex_eof; break;

		case '>':
			if (*(cur + 1) == '=')
			{
				cur += 2;
				_cur_lexeme = lex_greater_or_equal;
			}
			else
			{
				cur += 1;
				_cur_lexeme = lex_greater;
			}
			break;

		case '<':
			if (*(cur + 1) == '=')
			{
				cur += 2;
				_cur_lexeme = lex_less_or_equal;
			}
			else
			{
				cur += 1;
				_cur_lexeme = lex_less;
			}
			break;

		case '!':
			if (*(cur + 1) == '=')
			{
				cur += 2;
				_cur_lexeme = lex_not_equal;
			}
			else
			{
				_cur_lexeme = lex_none;
			}
			break;

		case '=':
			cur += 1;
			_cur_lexeme = lex_equal;

			break;

		case '+':
			cur += 1;
			_cur_lexeme = lex_plus;

			break;

		case '-':
			cur += 1;
			_cur_lexeme = lex_minus;

			break;

		case '*':
			cur += 1;
			_cur_lexeme = lex_multiply;

			break;

		case '|':
			cur += 1;
			_cur_lexeme = lex_union;

			break;

		case '$':
			cur += 1;

			if (PUGI__IS_CHARTYPEX(*cur, ctx_start_symbol))
			{
				_cur_lexeme_contents.begin = cur;

				while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol))
					cur++;

				if (cur[0] == ':' && PUGI__IS_CHARTYPEX(cur[1], ctx_symbol)) // qname
				{
					cur++; // :

					while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol))
						cur++;
				}

				_cur_lexeme_contents.end = cur;

				_cur_lexeme = lex_var_ref;
			}
			else
			{
				_cur_lexeme = lex_none;
			}

			break;

		case '(':
			cur += 1;
			_cur_lexeme = lex_open_brace;

			break;

		case ')':
			cur += 1;
			_cur_lexeme = lex_close_brace;

			break;

		case '[':
			cur += 1;
			_cur_lexeme = lex_open_square_brace;

			break;

		case ']':
			cur += 1;
			_cur_lexeme = lex_close_square_brace;

			break;

		case ',':
			cur += 1;
			_cur_lexeme = lex_comma;

			break;

		case '/':
			if (*(cur + 1) == '/')
			{
				cur += 2;
				_cur_lexeme = lex_double_slash;
			}
			else
			{
				cur += 1;
				_cur_lexeme = lex_slash;
			}
			break;

		case '.':
			if (*(cur + 1) == '.')
			{
				cur += 2;
				_cur_lexeme = lex_double_dot;
			}
			else if (PUGI__IS_CHARTYPEX(*(cur + 1), ctx_digit))
			{
				_cur_lexeme_contents.begin = cur; // .

				++cur;

				while (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
					cur++;

				_cur_lexeme_contents.end = cur;

				_cur_lexeme = lex_number;
			}
			else
			{
				cur += 1;
				_cur_lexeme = lex_dot;
			}
			break;

		case '@':
			cur += 1;
			_cur_lexeme = lex_axis_attribute;

			break;

		case '"':
		case '\'':
		{
			char_t terminator = *cur;

			++cur;

			_cur_lexeme_contents.begin = cur;
			while (*cur && *cur != terminator)
				cur++;
			_cur_lexeme_contents.end = cur;

			if (!*cur)
				_cur_lexeme = lex_none;
			else
			{
				cur += 1;
				_cur_lexeme = lex_quoted_string;
			}

			break;
		}

		case ':':
			if (*(cur + 1) == ':')
			{
				cur += 2;
				_cur_lexeme = lex_double_colon;
			}
			else
			{
				_cur_lexeme = lex_none;
			}
			break;

		default:
			if (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
			{
				_cur_lexeme_contents.begin = cur;

				while (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
					cur++;

				if (*cur == '.')
				{
					cur++;

					while (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
						cur++;
				}

				_cur_lexeme_contents.end = cur;

				_cur_lexeme = lex_number;
			}
			else if (PUGI__IS_CHARTYPEX(*cur, ctx_start_symbol))
			{
				_cur_lexeme_contents.begin = cur;

				while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol))
					cur++;

				if (cur[0] == ':')
				{
					if (cur[1] == '*') // namespace test ncname:*
					{
						cur += 2; // :*
					}
					else if (PUGI__IS_CHARTYPEX(cur[1], ctx_symbol)) // namespace test qname
					{
						cur++; // :

						while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol))
							cur++;
					}
				}

				_cur_lexeme_contents.end = cur;

				_cur_lexeme = lex_string;
			}
			else
			{
				_cur_lexeme = lex_none;
			}
		}

		_cur = cur;
	}

	lexeme_t current() const
	{
		return _cur_lexeme;
	}

	const char_t* current_pos() const
	{
		return _cur_lexeme_pos;
	}

	const xpath_lexer_string& contents() const
	{
		assert(
			_cur_lexeme == lex_var_ref || _cur_lexeme == lex_number || _cur_lexeme == lex_string ||
			_cur_lexeme == lex_quoted_string);

		return _cur_lexeme_contents;
	}
};

enum ast_type_t
{
	ast_op_or, // left or right
	ast_op_and, // left and right
	ast_op_equal, // left = right
	ast_op_not_equal, // left != right
	ast_op_less, // left < right
	ast_op_greater, // left > right
	ast_op_less_or_equal, // left <= right
	ast_op_greater_or_equal, // left >= right
	ast_op_add, // left + right
	ast_op_subtract, // left - right
	ast_op_multiply, // left * right
	ast_op_divide, // left / right
	ast_op_mod, // left % right
	ast_op_negate, // left - right
	ast_op_union, // left | right
	ast_predicate, // apply predicate to set; next points to next predicate
	ast_filter, // select * from left where right
	ast_filter_posinv, // select * from left where right; proximity position invariant
	ast_string_constant, // string constant
	ast_number_constant, // number constant
	ast_variable, // variable
	ast_func_last, // last()
	ast_func_position, // position()
	ast_func_count, // count(left)
	ast_func_id, // id(left)
	ast_func_local_name_0, // local-name()
	ast_func_local_name_1, // local-name(left)
	ast_func_namespace_uri_0, // namespace-uri()
	ast_func_namespace_uri_1, // namespace-uri(left)
	ast_func_name_0, // name()
	ast_func_name_1, // name(left)
	ast_func_string_0, // string()
	ast_func_string_1, // string(left)
	ast_func_concat, // concat(left, right, siblings)
	ast_func_starts_with, // starts_with(left, right)
	ast_func_contains, // contains(left, right)
	ast_func_substring_before, // substring-before(left, right)
	ast_func_substring_after, // substring-after(left, right)
	ast_func_substring_2, // substring(left, right)
	ast_func_substring_3, // substring(left, right, third)
	ast_func_string_length_0, // string-length()
	ast_func_string_length_1, // string-length(left)
	ast_func_normalize_space_0, // normalize-space()
	ast_func_normalize_space_1, // normalize-space(left)
	ast_func_translate, // translate(left, right, third)
	ast_func_boolean, // boolean(left)
	ast_func_not, // not(left)
	ast_func_true, // true()
	ast_func_false, // false()
	ast_func_lang, // lang(left)
	ast_func_number_0, // number()
	ast_func_number_1, // number(left)
	ast_func_sum, // sum(left)
	ast_func_floor, // floor(left)
	ast_func_ceiling, // ceiling(left)
	ast_func_round, // round(left)
	ast_step, // process set left with step
	ast_step_root // select root node
};

enum axis_t
{
	axis_ancestor,
	axis_ancestor_or_self,
	axis_attribute,
	axis_child,
	axis_descendant,
	axis_descendant_or_self,
	axis_following,
	axis_following_sibling,
	axis_namespace,
	axis_parent,
	axis_preceding,
	axis_preceding_sibling,
	axis_self
};

enum nodetest_t
{
	nodetest_none,
	nodetest_name,
	nodetest_type_node,
	nodetest_type_comment,
	nodetest_type_pi,
	nodetest_type_text,
	nodetest_pi,
	nodetest_all,
	nodetest_all_in_namespace
};

template<axis_t N>
struct axis_to_type
{
	static const axis_t axis;
};

template<axis_t N>
const axis_t axis_to_type<N>::axis = N;

class xpath_ast_node
{
private:
	// node type
	char _type;
	char _rettype;

	// for ast_step / ast_predicate
	char _axis;
	char _test;

	// tree node structure
	xpath_ast_node* _left;
	xpath_ast_node* _right;
	xpath_ast_node* _next;

	union
	{
		// value for ast_string_constant
		const char_t* string;
		// value for ast_number_constant
		double number;
		// variable for ast_variable
		xpath_variable* variable;
		// node test for ast_step (node name/namespace/node type/pi target)
		const char_t* nodetest;
	} _data;

	xpath_ast_node(const xpath_ast_node&) = delete;
	xpath_ast_node& operator=(const xpath_ast_node&) = delete;

	template<class Comp>
	static bool compare_eq(
		xpath_ast_node* lhs,
		xpath_ast_node* rhs,
		const xpath_context& c,
		const xpath_stack& stack,
		const Comp& comp)
	{
		xpath_value_type lt = lhs->rettype(), rt = rhs->rettype();

		if (lt != xpath_type_node_set && rt != xpath_type_node_set)
		{
			if (lt == xpath_type_boolean || rt == xpath_type_boolean)
				return comp(lhs->eval_boolean(c, stack), rhs->eval_boolean(c, stack));
			else if (lt == xpath_type_number || rt == xpath_type_number)
				return comp(lhs->eval_number(c, stack), rhs->eval_number(c, stack));
			else if (lt == xpath_type_string || rt == xpath_type_string)
			{
				xpath_allocator_capture cr(stack.result);

				xpath_string ls = lhs->eval_string(c, stack);
				xpath_string rs = rhs->eval_string(c, stack);

				return comp(ls, rs);
			}
		}
		else if (lt == xpath_type_node_set && rt == xpath_type_node_set)
		{
			xpath_allocator_capture cr(stack.result);

			xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
			xpath_node_set_raw rs = rhs->eval_node_set(c, stack);

			for (const xpath_node* li = ls.begin(); li != ls.end(); ++li)
				for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
				{
					xpath_allocator_capture cri(stack.result);

					if (comp(string_value(*li, stack.result), string_value(*ri, stack.result)))
						return true;
				}

			return false;
		}
		else
		{
			if (lt == xpath_type_node_set)
			{
				swap(lhs, rhs);
				swap(lt, rt);
			}

			if (lt == xpath_type_boolean)
				return comp(lhs->eval_boolean(c, stack), rhs->eval_boolean(c, stack));
			else if (lt == xpath_type_number)
			{
				xpath_allocator_capture cr(stack.result);

				double l = lhs->eval_number(c, stack);
				xpath_node_set_raw rs = rhs->eval_node_set(c, stack);

				for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
				{
					xpath_allocator_capture cri(stack.result);

					if (comp(l, convert_string_to_number(string_value(*ri, stack.result).c_str())))
						return true;
				}

				return false;
			}
			else if (lt == xpath_type_string)
			{
				xpath_allocator_capture cr(stack.result);

				xpath_string l = lhs->eval_string(c, stack);
				xpath_node_set_raw rs = rhs->eval_node_set(c, stack);

				for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
				{
					xpath_allocator_capture cri(stack.result);

					if (comp(l, string_value(*ri, stack.result)))
						return true;
				}

				return false;
			}
		}

		assert(!"Wrong types");
		return false;
	}

	template<class Comp>
	static bool compare_rel(
		xpath_ast_node* lhs,
		xpath_ast_node* rhs,
		const xpath_context& c,
		const xpath_stack& stack,
		const Comp& comp)
	{
		xpath_value_type lt = lhs->rettype(), rt = rhs->rettype();

		if (lt != xpath_type_node_set && rt != xpath_type_node_set)
			return comp(lhs->eval_number(c, stack), rhs->eval_number(c, stack));
		else if (lt == xpath_type_node_set && rt == xpath_type_node_set)
		{
			xpath_allocator_capture cr(stack.result);

			xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
			xpath_node_set_raw rs = rhs->eval_node_set(c, stack);

			for (const xpath_node* li = ls.begin(); li != ls.end(); ++li)
			{
				xpath_allocator_capture cri(stack.result);

				double l = convert_string_to_number(string_value(*li, stack.result).c_str());

				for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
				{
					xpath_allocator_capture crii(stack.result);

					if (comp(l, convert_string_to_number(string_value(*ri, stack.result).c_str())))
						return true;
				}
			}

			return false;
		}
		else if (lt != xpath_type_node_set && rt == xpath_type_node_set)
		{
			xpath_allocator_capture cr(stack.result);

			double l = lhs->eval_number(c, stack);
			xpath_node_set_raw rs = rhs->eval_node_set(c, stack);

			for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
			{
				xpath_allocator_capture cri(stack.result);

				if (comp(l, convert_string_to_number(string_value(*ri, stack.result).c_str())))
					return true;
			}

			return false;
		}
		else if (lt == xpath_type_node_set && rt != xpath_type_node_set)
		{
			xpath_allocator_capture cr(stack.result);

			xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
			double r = rhs->eval_number(c, stack);

			for (const xpath_node* li = ls.begin(); li != ls.end(); ++li)
			{
				xpath_allocator_capture cri(stack.result);

				if (comp(convert_string_to_number(string_value(*li, stack.result).c_str()), r))
					return true;
			}

			return false;
		}
		else
		{
			assert(!"Wrong types");
			return false;
		}
	}

	void apply_predicate(xpath_node_set_raw& ns, size_t first, xpath_ast_node* expr, const xpath_stack& stack)
	{
		assert(ns.size() >= first);

		size_t i = 1;
		size_t size = ns.size() - first;

		xpath_node* last = ns.begin() + first;

		// remove_if... or well, sort of
		for (xpath_node* it = last; it != ns.end(); ++it, ++i)
		{
			xpath_context c(*it, i, size);

			if (expr->rettype() == xpath_type_number)
			{
				if (expr->eval_number(c, stack) == i)
					*last++ = *it;
			}
			else if (expr->eval_boolean(c, stack))
				*last++ = *it;
		}

		ns.truncate(last);
	}

	void apply_predicates(xpath_node_set_raw& ns, size_t first, const xpath_stack& stack)
	{
		if (ns.size() == first)
			return;

		for (xpath_ast_node* pred = _right; pred; pred = pred->_next)
		{
			apply_predicate(ns, first, pred->_left, stack);
		}
	}

	void step_push(xpath_node_set_raw& ns, const xml_attribute& a, const xml_node& parent, xpath_allocator* alloc)
	{
		if (!a)
			return;

		const char_t* name = a.name();

		// There are no attribute nodes corresponding to attributes that declare namespaces
		// That is, "xmlns:..." or "xmlns"
		if (starts_with(name, PUGIXML_TEXT("xmlns")) && (name[5] == 0 || name[5] == ':'))
			return;

		switch (_test)
		{
		case nodetest_name:
			if (strequal(name, _data.nodetest))
				ns.push_back(xpath_node(a, parent), alloc);
			break;

		case nodetest_type_node:
		case nodetest_all: ns.push_back(xpath_node(a, parent), alloc); break;

		case nodetest_all_in_namespace:
			if (starts_with(name, _data.nodetest))
				ns.push_back(xpath_node(a, parent), alloc);
			break;

		default:;
		}
	}

	void step_push(xpath_node_set_raw& ns, const xml_node& n, xpath_allocator* alloc)
	{
		if (!n)
			return;

		switch (_test)
		{
		case nodetest_name:
			if (n.type() == node_element && strequal(n.name(), _data.nodetest))
				ns.push_back(n, alloc);
			break;

		case nodetest_type_node: ns.push_back(n, alloc); break;

		case nodetest_type_comment:
			if (n.type() == node_comment)
				ns.push_back(n, alloc);
			break;

		case nodetest_type_text:
			if (n.type() == node_pcdata || n.type() == node_cdata)
				ns.push_back(n, alloc);
			break;

		case nodetest_type_pi:
			if (n.type() == node_pi)
				ns.push_back(n, alloc);
			break;

		case nodetest_pi:
			if (n.type() == node_pi && strequal(n.name(), _data.nodetest))
				ns.push_back(n, alloc);
			break;

		case nodetest_all:
			if (n.type() == node_element)
				ns.push_back(n, alloc);
			break;

		case nodetest_all_in_namespace:
			if (n.type() == node_element && starts_with(n.name(), _data.nodetest))
				ns.push_back(n, alloc);
			break;

		default: assert(!"Unknown axis");
		}
	}

	template<class T>
	void step_fill(xpath_node_set_raw& ns, const xml_node& n, xpath_allocator* alloc, T)
	{
		const axis_t axis = T::axis;

		switch (axis)
		{
		case axis_attribute:
		{
			for (xml_attribute a = n.first_attribute(); a; a = a.next_attribute())
				step_push(ns, a, n, alloc);

			break;
		}

		case axis_child:
		{
			for (xml_node c = n.first_child(); c; c = c.next_sibling())
				step_push(ns, c, alloc);

			break;
		}

		case axis_descendant:
		case axis_descendant_or_self:
		{
			if (axis == axis_descendant_or_self)
				step_push(ns, n, alloc);

			xml_node cur = n.first_child();

			while (cur && cur != n)
			{
				step_push(ns, cur, alloc);

				if (cur.first_child())
					cur = cur.first_child();
				else if (cur.next_sibling())
					cur = cur.next_sibling();
				else
				{
					while (!cur.next_sibling() && cur != n)
						cur = cur.parent();

					if (cur != n)
						cur = cur.next_sibling();
				}
			}

			break;
		}

		case axis_following_sibling:
		{
			for (xml_node c = n.next_sibling(); c; c = c.next_sibling())
				step_push(ns, c, alloc);

			break;
		}

		case axis_preceding_sibling:
		{
			for (xml_node c = n.previous_sibling(); c; c = c.previous_sibling())
				step_push(ns, c, alloc);

			break;
		}

		case axis_following:
		{
			xml_node cur = n;

			// exit from this node so that we don't include descendants
			while (cur && !cur.next_sibling())
				cur = cur.parent();
			cur = cur.next_sibling();

			for (;;)
			{
				step_push(ns, cur, alloc);

				if (cur.first_child())
					cur = cur.first_child();
				else if (cur.next_sibling())
					cur = cur.next_sibling();
				else
				{
					while (cur && !cur.next_sibling())
						cur = cur.parent();
					cur = cur.next_sibling();

					if (!cur)
						break;
				}
			}

			break;
		}

		case axis_preceding:
		{
			xml_node cur = n;

			while (cur && !cur.previous_sibling())
				cur = cur.parent();
			cur = cur.previous_sibling();

			for (;;)
			{
				if (cur.last_child())
					cur = cur.last_child();
				else
				{
					// leaf node, can't be ancestor
					step_push(ns, cur, alloc);

					if (cur.previous_sibling())
						cur = cur.previous_sibling();
					else
					{
						do
						{
							cur = cur.parent();
							if (!cur)
								break;

							if (!node_is_ancestor(cur, n))
								step_push(ns, cur, alloc);
						} while (!cur.previous_sibling());

						cur = cur.previous_sibling();

						if (!cur)
							break;
					}
				}
			}

			break;
		}

		case axis_ancestor:
		case axis_ancestor_or_self:
		{
			if (axis == axis_ancestor_or_self)
				step_push(ns, n, alloc);

			xml_node cur = n.parent();

			while (cur)
			{
				step_push(ns, cur, alloc);

				cur = cur.parent();
			}

			break;
		}

		case axis_self:
		{
			step_push(ns, n, alloc);

			break;
		}

		case axis_parent:
		{
			if (n.parent())
				step_push(ns, n.parent(), alloc);

			break;
		}

		default: assert(!"Unimplemented axis");
		}
	}

	template<class T>
	void step_fill(xpath_node_set_raw& ns, const xml_attribute& a, const xml_node& p, xpath_allocator* alloc, T v)
	{
		const axis_t axis = T::axis;

		switch (axis)
		{
		case axis_ancestor:
		case axis_ancestor_or_self:
		{
			if (axis == axis_ancestor_or_self &&
				_test == nodetest_type_node) // reject attributes based on principal node type test
				step_push(ns, a, p, alloc);

			xml_node cur = p;

			while (cur)
			{
				step_push(ns, cur, alloc);

				cur = cur.parent();
			}

			break;
		}

		case axis_descendant_or_self:
		case axis_self:
		{
			if (_test == nodetest_type_node) // reject attributes based on principal node type test
				step_push(ns, a, p, alloc);

			break;
		}

		case axis_following:
		{
			xml_node cur = p;

			for (;;)
			{
				if (cur.first_child())
					cur = cur.first_child();
				else if (cur.next_sibling())
					cur = cur.next_sibling();
				else
				{
					while (cur && !cur.next_sibling())
						cur = cur.parent();
					cur = cur.next_sibling();

					if (!cur)
						break;
				}

				step_push(ns, cur, alloc);
			}

			break;
		}

		case axis_parent:
		{
			step_push(ns, p, alloc);

			break;
		}

		case axis_preceding:
		{
			// preceding:: axis does not include attribute nodes and attribute ancestors (they are the same as parent's ancestors), so we can reuse node preceding
			step_fill(ns, p, alloc, v);
			break;
		}

		default: assert(!"Unimplemented axis");
		}
	}

	template<class T>
	xpath_node_set_raw step_do(const xpath_context& c, const xpath_stack& stack, T v)
	{
		const axis_t axis = T::axis;
		bool attributes =
			(axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_descendant_or_self ||
			 axis == axis_following || axis == axis_parent || axis == axis_preceding || axis == axis_self);

		xpath_node_set_raw ns;
		ns.set_type(
			(axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_preceding ||
			 axis == axis_preceding_sibling)
				? xpath_node_set::type_sorted_reverse
				: xpath_node_set::type_sorted);

		if (_left)
		{
			xpath_node_set_raw s = _left->eval_node_set(c, stack);

			// self axis preserves the original order
			if (axis == axis_self)
				ns.set_type(s.type());

			for (const xpath_node* it = s.begin(); it != s.end(); ++it)
			{
				size_t size = ns.size();

				// in general, all axes generate elements in a particular order, but there is no order guarantee if axis is applied to two nodes
				if (axis != axis_self && size != 0)
					ns.set_type(xpath_node_set::type_unsorted);

				if (it->node())
					step_fill(ns, it->node(), stack.result, v);
				else if (attributes)
					step_fill(ns, it->attribute(), it->parent(), stack.result, v);

				apply_predicates(ns, size, stack);
			}
		}
		else
		{
			if (c.n.node())
				step_fill(ns, c.n.node(), stack.result, v);
			else if (attributes)
				step_fill(ns, c.n.attribute(), c.n.parent(), stack.result, v);

			apply_predicates(ns, 0, stack);
		}

		// child, attribute and self axes always generate unique set of nodes
		// for other axis, if the set stayed sorted, it stayed unique because the traversal algorithms do not visit the same node twice
		if (axis != axis_child && axis != axis_attribute && axis != axis_self &&
			ns.type() == xpath_node_set::type_unsorted)
			ns.remove_duplicates();

		return ns;
	}

public:
	xpath_ast_node(ast_type_t type, xpath_value_type rettype_, const char_t* value) :
		_type(static_cast<char>(type)),
		_rettype(static_cast<char>(rettype_)),
		_axis(0),
		_test(0),
		_left(0),
		_right(0),
		_next(0)
	{
		assert(type == ast_string_constant);
		_data.string = value;
	}

	xpath_ast_node(ast_type_t type, xpath_value_type rettype_, double value) :
		_type(static_cast<char>(type)),
		_rettype(static_cast<char>(rettype_)),
		_axis(0),
		_test(0),
		_left(0),
		_right(0),
		_next(0)
	{
		assert(type == ast_number_constant);
		_data.number = value;
	}

	xpath_ast_node(ast_type_t type, xpath_value_type rettype_, xpath_variable* value) :
		_type(static_cast<char>(type)),
		_rettype(static_cast<char>(rettype_)),
		_axis(0),
		_test(0),
		_left(0),
		_right(0),
		_next(0)
	{
		assert(type == ast_variable);
		_data.variable = value;
	}

	xpath_ast_node(
		ast_type_t type,
		xpath_value_type rettype_,
		xpath_ast_node* left = nullptr,
		xpath_ast_node* right = nullptr) :
		_type(static_cast<char>(type)),
		_rettype(static_cast<char>(rettype_)),
		_axis(0),
		_test(0),
		_left(left),
		_right(right),
		_next(0)
	{
	}

	xpath_ast_node(ast_type_t type, xpath_ast_node* left, axis_t axis, nodetest_t test, const char_t* contents) :
		_type(static_cast<char>(type)),
		_rettype(xpath_type_node_set),
		_axis(static_cast<char>(axis)),
		_test(static_cast<char>(test)),
		_left(left),
		_right(0),
		_next(0)
	{
		_data.nodetest = contents;
	}

	void set_next(xpath_ast_node* value)
	{
		_next = value;
	}

	void set_right(xpath_ast_node* value)
	{
		_right = value;
	}

	bool eval_boolean(const xpath_context& c, const xpath_stack& stack)
	{
		switch (_type)
		{
		case ast_op_or: return _left->eval_boolean(c, stack) || _right->eval_boolean(c, stack);

		case ast_op_and: return _left->eval_boolean(c, stack) && _right->eval_boolean(c, stack);

		case ast_op_equal: return compare_eq(_left, _right, c, stack, equal_to());

		case ast_op_not_equal: return compare_eq(_left, _right, c, stack, not_equal_to());

		case ast_op_less: return compare_rel(_left, _right, c, stack, less());

		case ast_op_greater: return compare_rel(_right, _left, c, stack, less());

		case ast_op_less_or_equal: return compare_rel(_left, _right, c, stack, less_equal());

		case ast_op_greater_or_equal: return compare_rel(_right, _left, c, stack, less_equal());

		case ast_func_starts_with:
		{
			xpath_allocator_capture cr(stack.result);

			xpath_string lr = _left->eval_string(c, stack);
			xpath_string rr = _right->eval_string(c, stack);

			return starts_with(lr.c_str(), rr.c_str());
		}

		case ast_func_contains:
		{
			xpath_allocator_capture cr(stack.result);

			xpath_string lr = _left->eval_string(c, stack);
			xpath_string rr = _right->eval_string(c, stack);

			return find_substring(lr.c_str(), rr.c_str()) != 0;
		}

		case ast_func_boolean: return _left->eval_boolean(c, stack);

		case ast_func_not: return !_left->eval_boolean(c, stack);

		case ast_func_true: return true;

		case ast_func_false: return false;

		case ast_func_lang:
		{
			if (c.n.attribute())
				return false;

			xpath_allocator_capture cr(stack.result);

			xpath_string lang = _left->eval_string(c, stack);

			for (xml_node n = c.n.node(); n; n = n.parent())
			{
				xml_attribute a = n.attribute(PUGIXML_TEXT("xml:lang"));

				if (a)
				{
					const char_t* value = a.value();

					// strnicmp / strncasecmp is not portable
					for (const char_t* lit = lang.c_str(); *lit; ++lit)
					{
						if (tolower_ascii(*lit) != tolower_ascii(*value))
							return false;
						++value;
					}

					return *value == 0 || *value == '-';
				}
			}

			return false;
		}

		case ast_variable:
		{
			assert(_rettype == _data.variable->type());

			if (_rettype == xpath_type_boolean)
				return _data.variable->get_boolean();

			// fallthrough to type conversion
		}

		default:
		{
			switch (_rettype)
			{
			case xpath_type_number: return convert_number_to_boolean(eval_number(c, stack));

			case xpath_type_string:
			{
				xpath_allocator_capture cr(stack.result);

				return !eval_string(c, stack).empty();
			}

			case xpath_type_node_set:
			{
				xpath_allocator_capture cr(stack.result);

				return !eval_node_set(c, stack).empty();
			}

			default: assert(!"Wrong expression for return type boolean"); return false;
			}
		}
		}
	}

	double eval_number(const xpath_context& c, const xpath_stack& stack)
	{
		switch (_type)
		{
		case ast_op_add: return _left->eval_number(c, stack) + _right->eval_number(c, stack);

		case ast_op_subtract: return _left->eval_number(c, stack) - _right->eval_number(c, stack);

		case ast_op_multiply: return _left->eval_number(c, stack) * _right->eval_number(c, stack);

		case ast_op_divide: return _left->eval_number(c, stack) / _right->eval_number(c, stack);

		case ast_op_mod: return fmod(_left->eval_number(c, stack), _right->eval_number(c, stack));

		case ast_op_negate: return -_left->eval_number(c, stack);

		case ast_number_constant: return _data.number;

		case ast_func_last: return static_cast<double>(c.size);

		case ast_func_position: return static_cast<double>(c.position);

		case ast_func_count:
		{
			xpath_allocator_capture cr(stack.result);

			return static_cast<double>(_left->eval_node_set(c, stack).size());
		}

		case ast_func_string_length_0:
		{
			xpath_allocator_capture cr(stack.result);

			return static_cast<double>(string_value(c.n, stack.result).length());
		}

		case ast_func_string_length_1:
		{
			xpath_allocator_capture cr(stack.result);

			return static_cast<double>(_left->eval_string(c, stack).length());
		}

		case ast_func_number_0:
		{
			xpath_allocator_capture cr(stack.result);

			return convert_string_to_number(string_value(c.n, stack.result).c_str());
		}

		case ast_func_number_1: return _left->eval_number(c, stack);

		case ast_func_sum:
		{
			xpath_allocator_capture cr(stack.result);

			double r = 0;

			xpath_node_set_raw ns = _left->eval_node_set(c, stack);

			for (const xpath_node* it = ns.begin(); it != ns.end(); ++it)
			{
				xpath_allocator_capture cri(stack.result);

				r += convert_string_to_number(string_value(*it, stack.result).c_str());
			}

			return r;
		}

		case ast_func_floor:
		{
			double r = _left->eval_number(c, stack);

			return r == r ? floor(r) : r;
		}

		case ast_func_ceiling:
		{
			double r = _left->eval_number(c, stack);

			return r == r ? ceil(r) : r;
		}

		case ast_func_round: return round_nearest_nzero(_left->eval_number(c, stack));

		case ast_variable:
		{
			assert(_rettype == _data.variable->type());

			if (_rettype == xpath_type_number)
				return _data.variable->get_number();

			// fallthrough to type conversion
		}

		default:
		{
			switch (_rettype)
			{
			case xpath_type_boolean: return eval_boolean(c, stack) ? 1 : 0;

			case xpath_type_string:
			{
				xpath_allocator_capture cr(stack.result);

				return convert_string_to_number(eval_string(c, stack).c_str());
			}

			case xpath_type_node_set:
			{
				xpath_allocator_capture cr(stack.result);

				return convert_string_to_number(eval_string(c, stack).c_str());
			}

			default: assert(!"Wrong expression for return type number"); return 0;
			}
		}
		}
	}

	xpath_string eval_string_concat(const xpath_context& c, const xpath_stack& stack)
	{
		assert(_type == ast_func_concat);

		xpath_allocator_capture ct(stack.temp);

		// count the string number
		size_t count = 1;
		for (xpath_ast_node* nc = _right; nc; nc = nc->_next)
			count++;

		// gather all strings
		xpath_string static_buffer[4];
		xpath_string* buffer = static_buffer;

		// allocate on-heap for large concats
		if (count > sizeof(static_buffer) / sizeof(static_buffer[0]))
		{
			buffer = static_cast<xpath_string*>(stack.temp->allocate(count * sizeof(xpath_string)));
			assert(buffer);
		}

		// evaluate all strings to temporary stack
		xpath_stack swapped_stack = {stack.temp, stack.result};

		buffer[0] = _left->eval_string(c, swapped_stack);

		size_t pos = 1;
		for (xpath_ast_node* n = _right; n; n = n->_next, ++pos)
			buffer[pos] = n->eval_string(c, swapped_stack);
		assert(pos == count);

		// get total length
		size_t length = 0;
		for (size_t i = 0; i < count; ++i)
			length += buffer[i].length();

		// create final string
		char_t* result = static_cast<char_t*>(stack.result->allocate((length + 1) * sizeof(char_t)));
		assert(result);

		char_t* ri = result;

		for (size_t j = 0; j < count; ++j)
			for (const char_t* bi = buffer[j].c_str(); *bi; ++bi)
				*ri++ = *bi;

		*ri = 0;

		return xpath_string(result, true);
	}

	xpath_string eval_string(const xpath_context& c, const xpath_stack& stack)
	{
		switch (_type)
		{
		case ast_string_constant: return xpath_string_const(_data.string);

		case ast_func_local_name_0:
		{
			xpath_node na = c.n;

			return xpath_string_const(local_name(na));
		}

		case ast_func_local_name_1:
		{
			xpath_allocator_capture cr(stack.result);

			xpath_node_set_raw ns = _left->eval_node_set(c, stack);
			xpath_node na = ns.first();

			return xpath_string_const(local_name(na));
		}

		case ast_func_name_0:
		{
			xpath_node na = c.n;

			return xpath_string_const(qualified_name(na));
		}

		case ast_func_name_1:
		{
			xpath_allocator_capture cr(stack.result);

			xpath_node_set_raw ns = _left->eval_node_set(c, stack);
			xpath_node na = ns.first();

			return xpath_string_const(qualified_name(na));
		}

		case ast_func_namespace_uri_0:
		{
			xpath_node na = c.n;

			return xpath_string_const(namespace_uri(na));
		}

		case ast_func_namespace_uri_1:
		{
			xpath_allocator_capture cr(stack.result);

			xpath_node_set_raw ns = _left->eval_node_set(c, stack);
			xpath_node na = ns.first();

			return xpath_string_const(namespace_uri(na));
		}

		case ast_func_string_0: return string_value(c.n, stack.result);

		case ast_func_string_1: return _left->eval_string(c, stack);

		case ast_func_concat: return eval_string_concat(c, stack);

		case ast_func_substring_before:
		{
			xpath_allocator_capture cr(stack.temp);

			xpath_stack swapped_stack = {stack.temp, stack.result};

			xpath_string s = _left->eval_string(c, swapped_stack);
			xpath_string p = _right->eval_string(c, swapped_stack);

			const char_t* pos = find_substring(s.c_str(), p.c_str());

			return pos ? xpath_string(s.c_str(), pos, stack.result) : xpath_string();
		}

		case ast_func_substring_after:
		{
			xpath_allocator_capture cr(stack.temp);

			xpath_stack swapped_stack = {stack.temp, stack.result};

			xpath_string s = _left->eval_string(c, swapped_stack);
			xpath_string p = _right->eval_string(c, swapped_stack);

			const char_t* pos = find_substring(s.c_str(), p.c_str());
			if (!pos)
				return xpath_string();

			const char_t* result = pos + p.length();

			return s.uses_heap() ? xpath_string(result, stack.result) : xpath_string_const(result);
		}

		case ast_func_substring_2:
		{
			xpath_allocator_capture cr(stack.temp);

			xpath_stack swapped_stack = {stack.temp, stack.result};

			xpath_string s = _left->eval_string(c, swapped_stack);
			size_t s_length = s.length();

			double first = round_nearest(_right->eval_number(c, stack));

			if (is_nan(first))
				return xpath_string(); // NaN
			else if (first >= s_length + 1)
				return xpath_string();

			size_t pos = first < 1 ? 1 : static_cast<size_t>(first);
			assert(1 <= pos && pos <= s_length + 1);

			const char_t* rbegin = s.c_str() + (pos - 1);

			return s.uses_heap() ? xpath_string(rbegin, stack.result) : xpath_string_const(rbegin);
		}

		case ast_func_substring_3:
		{
			xpath_allocator_capture cr(stack.temp);

			xpath_stack swapped_stack = {stack.temp, stack.result};

			xpath_string s = _left->eval_string(c, swapped_stack);
			size_t s_length = s.length();

			double first = round_nearest(_right->eval_number(c, stack));
			double last = first + round_nearest(_right->_next->eval_number(c, stack));

			if (is_nan(first) || is_nan(last))
				return xpath_string();
			else if (first >= s_length + 1)
				return xpath_string();
			else if (first >= last)
				return xpath_string();
			else if (last < 1)
				return xpath_string();

			size_t pos = first < 1 ? 1 : static_cast<size_t>(first);
			size_t end = last >= s_length + 1 ? s_length + 1 : static_cast<size_t>(last);

			assert(1 <= pos && pos <= end && end <= s_length + 1);
			const char_t* rbegin = s.c_str() + (pos - 1);
			const char_t* rend = s.c_str() + (end - 1);

			return (end == s_length + 1 && !s.uses_heap())
				? xpath_string_const(rbegin)
				: xpath_string(rbegin, rend, stack.result);
		}

		case ast_func_normalize_space_0:
		{
			xpath_string s = string_value(c.n, stack.result);

			normalize_space(s.data(stack.result));

			return s;
		}

		case ast_func_normalize_space_1:
		{
			xpath_string s = _left->eval_string(c, stack);

			normalize_space(s.data(stack.result));

			return s;
		}

		case ast_func_translate:
		{
			xpath_allocator_capture cr(stack.temp);

			xpath_stack swapped_stack = {stack.temp, stack.result};

			xpath_string s = _left->eval_string(c, stack);
			xpath_string from = _right->eval_string(c, swapped_stack);
			xpath_string to = _right->_next->eval_string(c, swapped_stack);

			translate(s.data(stack.result), from.c_str(), to.c_str());

			return s;
		}

		case ast_variable:
		{
			assert(_rettype == _data.variable->type());

			if (_rettype == xpath_type_string)
				return xpath_string_const(_data.variable->get_string());

			// fallthrough to type conversion
		}

		default:
		{
			switch (_rettype)
			{
			case xpath_type_boolean:
				return xpath_string_const(eval_boolean(c, stack) ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));

			case xpath_type_number: return convert_number_to_string(eval_number(c, stack), stack.result);

			case xpath_type_node_set:
			{
				xpath_allocator_capture cr(stack.temp);

				xpath_stack swapped_stack = {stack.temp, stack.result};

				xpath_node_set_raw ns = eval_node_set(c, swapped_stack);
				return ns.empty() ? xpath_string() : string_value(ns.first(), stack.result);
			}

			default: assert(!"Wrong expression for return type string"); return xpath_string();
			}
		}
		}
	}

	xpath_node_set_raw eval_node_set(const xpath_context& c, const xpath_stack& stack)
	{
		switch (_type)
		{
		case ast_op_union:
		{
			xpath_allocator_capture cr(stack.temp);

			xpath_stack swapped_stack = {stack.temp, stack.result};

			xpath_node_set_raw ls = _left->eval_node_set(c, swapped_stack);
			xpath_node_set_raw rs = _right->eval_node_set(c, stack);

			// we can optimize merging two sorted sets, but this is a very rare operation, so don't bother
			rs.set_type(xpath_node_set::type_unsorted);

			rs.append(ls.begin(), ls.end(), stack.result);
			rs.remove_duplicates();

			return rs;
		}

		case ast_filter:
		case ast_filter_posinv:
		{
			xpath_node_set_raw set = _left->eval_node_set(c, stack);

			// either expression is a number or it contains position() call; sort by document order
			if (_type == ast_filter)
				set.sort_do();

			apply_predicate(set, 0, _right, stack);

			return set;
		}

		case ast_func_id: return xpath_node_set_raw();

		case ast_step:
		{
			switch (_axis)
			{
			case axis_ancestor: return step_do(c, stack, axis_to_type<axis_ancestor>());

			case axis_ancestor_or_self: return step_do(c, stack, axis_to_type<axis_ancestor_or_self>());

			case axis_attribute: return step_do(c, stack, axis_to_type<axis_attribute>());

			case axis_child: return step_do(c, stack, axis_to_type<axis_child>());

			case axis_descendant: return step_do(c, stack, axis_to_type<axis_descendant>());

			case axis_descendant_or_self: return step_do(c, stack, axis_to_type<axis_descendant_or_self>());

			case axis_following: return step_do(c, stack, axis_to_type<axis_following>());

			case axis_following_sibling: return step_do(c, stack, axis_to_type<axis_following_sibling>());

			case axis_namespace:
				// namespaced axis is not supported
				return xpath_node_set_raw();

			case axis_parent: return step_do(c, stack, axis_to_type<axis_parent>());

			case axis_preceding: return step_do(c, stack, axis_to_type<axis_preceding>());

			case axis_preceding_sibling: return step_do(c, stack, axis_to_type<axis_preceding_sibling>());

			case axis_self: return step_do(c, stack, axis_to_type<axis_self>());

			default: assert(!"Unknown axis"); return xpath_node_set_raw();
			}
		}

		case ast_step_root:
		{
			assert(!_right); // root step can't have any predicates

			xpath_node_set_raw ns;

			ns.set_type(xpath_node_set::type_sorted);

			if (c.n.node())
				ns.push_back(c.n.node().root(), stack.result);
			else if (c.n.attribute())
				ns.push_back(c.n.parent().root(), stack.result);

			return ns;
		}

		case ast_variable:
		{
			assert(_rettype == _data.variable->type());

			if (_rettype == xpath_type_node_set)
			{
				const xpath_node_set& s = _data.variable->get_node_set();

				xpath_node_set_raw ns;

				ns.set_type(s.type());
				ns.append(s.begin(), s.end(), stack.result);

				return ns;
			}

			// fallthrough to type conversion
		}

		default: assert(!"Wrong expression for return type node set"); return xpath_node_set_raw();
		}
	}

	bool is_posinv()
	{
		switch (_type)
		{
		case ast_func_position: return false;

		case ast_string_constant:
		case ast_number_constant:
		case ast_variable: return true;

		case ast_step:
		case ast_step_root: return true;

		case ast_predicate:
		case ast_filter:
		case ast_filter_posinv: return true;

		default:
			if (_left && !_left->is_posinv())
				return false;

			for (xpath_ast_node* n = _right; n; n = n->_next)
				if (!n->is_posinv())
					return false;

			return true;
		}
	}

	xpath_value_type rettype() const
	{
		return static_cast<xpath_value_type>(_rettype);
	}
};

struct xpath_parser
{
	xpath_allocator* _alloc;
	xpath_lexer _lexer;

	const char_t* _query;
	xpath_variable_set* _variables;

	xpath_parse_result* _result;

	#ifdef PUGIXML_NO_EXCEPTIONS
	jmp_buf _error_handler;
	#endif

	void throw_error(const char* message)
	{
		_result->error = message;
		_result->offset = _lexer.current_pos() - _query;

	#ifdef PUGIXML_NO_EXCEPTIONS
		longjmp(_error_handler, 1);
	#else
		throw xpath_exception(*_result);
	#endif
	}

	void throw_error_oom()
	{
	#ifdef PUGIXML_NO_EXCEPTIONS
		throw_error("Out of memory");
	#else
		throw std::bad_alloc();
	#endif
	}

	void* alloc_node()
	{
		void* result = _alloc->allocate_nothrow(sizeof(xpath_ast_node));

		if (!result)
			throw_error_oom();

		return result;
	}

	const char_t* alloc_string(const xpath_lexer_string& value)
	{
		if (value.begin)
		{
			size_t length = static_cast<size_t>(value.end - value.begin);

			char_t* c = static_cast<char_t*>(_alloc->allocate_nothrow((length + 1) * sizeof(char_t)));
			if (!c)
				throw_error_oom();

			memcpy(c, value.begin, length * sizeof(char_t));
			c[length] = 0;

			return c;
		}
		else
			return 0;
	}

	xpath_ast_node* parse_function_helper(ast_type_t type0, ast_type_t type1, size_t argc, xpath_ast_node* args[2])
	{
		assert(argc <= 1);

		if (argc == 1 && args[0]->rettype() != xpath_type_node_set)
			throw_error("Function has to be applied to node set");

		return new (alloc_node()) xpath_ast_node(argc == 0 ? type0 : type1, xpath_type_string, args[0]);
	}

	xpath_ast_node* parse_function(const xpath_lexer_string& name, size_t argc, xpath_ast_node* args[2])
	{
		switch (name.begin[0])
		{
		case 'b':
			if (name == PUGIXML_TEXT("boolean") && argc == 1)
				return new (alloc_node()) xpath_ast_node(ast_func_boolean, xpath_type_boolean, args[0]);

			break;

		case 'c':
			if (name == PUGIXML_TEXT("count") && argc == 1)
			{
				if (args[0]->rettype() != xpath_type_node_set)
					throw_error("Function has to be applied to node set");
				return new (alloc_node()) xpath_ast_node(ast_func_count, xpath_type_number, args[0]);
			}
			else if (name == PUGIXML_TEXT("contains") && argc == 2)
				return new (alloc_node()) xpath_ast_node(ast_func_contains, xpath_type_string, args[0], args[1]);
			else if (name == PUGIXML_TEXT("concat") && argc >= 2)
				return new (alloc_node()) xpath_ast_node(ast_func_concat, xpath_type_string, args[0], args[1]);
			else if (name == PUGIXML_TEXT("ceiling") && argc == 1)
				return new (alloc_node()) xpath_ast_node(ast_func_ceiling, xpath_type_number, args[0]);

			break;

		case 'f':
			if (name == PUGIXML_TEXT("false") && argc == 0)
				return new (alloc_node()) xpath_ast_node(ast_func_false, xpath_type_boolean);
			else if (name == PUGIXML_TEXT("floor") && argc == 1)
				return new (alloc_node()) xpath_ast_node(ast_func_floor, xpath_type_number, args[0]);

			break;

		case 'i':
			if (name == PUGIXML_TEXT("id") && argc == 1)
				return new (alloc_node()) xpath_ast_node(ast_func_id, xpath_type_node_set, args[0]);

			break;

		case 'l':
			if (name == PUGIXML_TEXT("last") && argc == 0)
				return new (alloc_node()) xpath_ast_node(ast_func_last, xpath_type_number);
			else if (name == PUGIXML_TEXT("lang") && argc == 1)
				return new (alloc_node()) xpath_ast_node(ast_func_lang, xpath_type_boolean, args[0]);
			else if (name == PUGIXML_TEXT("local-name") && argc <= 1)
				return parse_function_helper(ast_func_local_name_0, ast_func_local_name_1, argc, args);

			break;

		case 'n':
			if (name == PUGIXML_TEXT("name") && argc <= 1)
				return parse_function_helper(ast_func_name_0, ast_func_name_1, argc, args);
			else if (name == PUGIXML_TEXT("namespace-uri") && argc <= 1)
				return parse_function_helper(ast_func_namespace_uri_0, ast_func_namespace_uri_1, argc, args);
			else if (name == PUGIXML_TEXT("normalize-space") && argc <= 1)
				return new (alloc_node()) xpath_ast_node(
					argc == 0 ? ast_func_normalize_space_0 : ast_func_normalize_space_1,
					xpath_type_string,
					args[0],
					args[1]);
			else if (name == PUGIXML_TEXT("not") && argc == 1)
				return new (alloc_node()) xpath_ast_node(ast_func_not, xpath_type_boolean, args[0]);
			else if (name == PUGIXML_TEXT("number") && argc <= 1)
				return new (alloc_node())
					xpath_ast_node(argc == 0 ? ast_func_number_0 : ast_func_number_1, xpath_type_number, args[0]);

			break;

		case 'p':
			if (name == PUGIXML_TEXT("position") && argc == 0)
				return new (alloc_node()) xpath_ast_node(ast_func_position, xpath_type_number);

			break;

		case 'r':
			if (name == PUGIXML_TEXT("round") && argc == 1)
				return new (alloc_node()) xpath_ast_node(ast_func_round, xpath_type_number, args[0]);

			break;

		case 's':
			if (name == PUGIXML_TEXT("string") && argc <= 1)
				return new (alloc_node())
					xpath_ast_node(argc == 0 ? ast_func_string_0 : ast_func_string_1, xpath_type_string, args[0]);
			else if (name == PUGIXML_TEXT("string-length") && argc <= 1)
				return new (alloc_node()) xpath_ast_node(
					argc == 0 ? ast_func_string_length_0 : ast_func_string_length_1,
					xpath_type_string,
					args[0]);
			else if (name == PUGIXML_TEXT("starts-with") && argc == 2)
				return new (alloc_node()) xpath_ast_node(ast_func_starts_with, xpath_type_boolean, args[0], args[1]);
			else if (name == PUGIXML_TEXT("substring-before") && argc == 2)
				return new (alloc_node())
					xpath_ast_node(ast_func_substring_before, xpath_type_string, args[0], args[1]);
			else if (name == PUGIXML_TEXT("substring-after") && argc == 2)
				return new (alloc_node()) xpath_ast_node(ast_func_substring_after, xpath_type_string, args[0], args[1]);
			else if (name == PUGIXML_TEXT("substring") && (argc == 2 || argc == 3))
				return new (alloc_node()) xpath_ast_node(
					argc == 2 ? ast_func_substring_2 : ast_func_substring_3,
					xpath_type_string,
					args[0],
					args[1]);
			else if (name == PUGIXML_TEXT("sum") && argc == 1)
			{
				if (args[0]->rettype() != xpath_type_node_set)
					throw_error("Function has to be applied to node set");
				return new (alloc_node()) xpath_ast_node(ast_func_sum, xpath_type_number, args[0]);
			}

			break;

		case 't':
			if (name == PUGIXML_TEXT("translate") && argc == 3)
				return new (alloc_node()) xpath_ast_node(ast_func_translate, xpath_type_string, args[0], args[1]);
			else if (name == PUGIXML_TEXT("true") && argc == 0)
				return new (alloc_node()) xpath_ast_node(ast_func_true, xpath_type_boolean);

			break;

		default: break;
		}

		throw_error("Unrecognized function or wrong parameter count");

		return 0;
	}

	axis_t parse_axis_name(const xpath_lexer_string& name, bool& specified)
	{
		specified = true;

		switch (name.begin[0])
		{
		case 'a':
			if (name == PUGIXML_TEXT("ancestor"))
				return axis_ancestor;
			else if (name == PUGIXML_TEXT("ancestor-or-self"))
				return axis_ancestor_or_self;
			else if (name == PUGIXML_TEXT("attribute"))
				return axis_attribute;

			break;

		case 'c':
			if (name == PUGIXML_TEXT("child"))
				return axis_child;

			break;

		case 'd':
			if (name == PUGIXML_TEXT("descendant"))
				return axis_descendant;
			else if (name == PUGIXML_TEXT("descendant-or-self"))
				return axis_descendant_or_self;

			break;

		case 'f':
			if (name == PUGIXML_TEXT("following"))
				return axis_following;
			else if (name == PUGIXML_TEXT("following-sibling"))
				return axis_following_sibling;

			break;

		case 'n':
			if (name == PUGIXML_TEXT("namespace"))
				return axis_namespace;

			break;

		case 'p':
			if (name == PUGIXML_TEXT("parent"))
				return axis_parent;
			else if (name == PUGIXML_TEXT("preceding"))
				return axis_preceding;
			else if (name == PUGIXML_TEXT("preceding-sibling"))
				return axis_preceding_sibling;

			break;

		case 's':
			if (name == PUGIXML_TEXT("self"))
				return axis_self;

			break;

		default: break;
		}

		specified = false;
		return axis_child;
	}

	nodetest_t parse_node_test_type(const xpath_lexer_string& name)
	{
		switch (name.begin[0])
		{
		case 'c':
			if (name == PUGIXML_TEXT("comment"))
				return nodetest_type_comment;

			break;

		case 'n':
			if (name == PUGIXML_TEXT("node"))
				return nodetest_type_node;

			break;

		case 'p':
			if (name == PUGIXML_TEXT("processing-instruction"))
				return nodetest_type_pi;

			break;

		case 't':
			if (name == PUGIXML_TEXT("text"))
				return nodetest_type_text;

			break;

		default: break;
		}

		return nodetest_none;
	}

	// PrimaryExpr ::= VariableReference | '(' Expr ')' | Literal | Number | FunctionCall
	xpath_ast_node* parse_primary_expression()
	{
		switch (_lexer.current())
		{
		case lex_var_ref:
		{
			xpath_lexer_string name = _lexer.contents();

			if (!_variables)
				throw_error("Unknown variable: variable set is not provided");

			xpath_variable* var = get_variable(_variables, name.begin, name.end);

			if (!var)
				throw_error("Unknown variable: variable set does not contain the given name");

			_lexer.next();

			return new (alloc_node()) xpath_ast_node(ast_variable, var->type(), var);
		}

		case lex_open_brace:
		{
			_lexer.next();

			xpath_ast_node* n = parse_expression();

			if (_lexer.current() != lex_close_brace)
				throw_error("Unmatched braces");

			_lexer.next();

			return n;
		}

		case lex_quoted_string:
		{
			const char_t* value = alloc_string(_lexer.contents());

			xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_string_constant, xpath_type_string, value);
			_lexer.next();

			return n;
		}

		case lex_number:
		{
			double value = 0;

			if (!convert_string_to_number(_lexer.contents().begin, _lexer.contents().end, &value))
				throw_error_oom();

			xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_number_constant, xpath_type_number, value);
			_lexer.next();

			return n;
		}

		case lex_string:
		{
			xpath_ast_node* args[2] = {0};
			size_t argc = 0;

			xpath_lexer_string function = _lexer.contents();
			_lexer.next();

			xpath_ast_node* last_arg = nullptr;

			if (_lexer.current() != lex_open_brace)
				throw_error("Unrecognized function call");
			_lexer.next();

			if (_lexer.current() != lex_close_brace)
				args[argc++] = parse_expression();

			while (_lexer.current() != lex_close_brace)
			{
				if (_lexer.current() != lex_comma)
					throw_error("No comma between function arguments");
				_lexer.next();

				xpath_ast_node* n = parse_expression();

				if (argc < 2)
					args[argc] = n;
				else
					last_arg->set_next(n);

				argc++;
				last_arg = n;
			}

			_lexer.next();

			return parse_function(function, argc, args);
		}

		default: throw_error("Unrecognizable primary expression"); return 0;
		}
	}

	// FilterExpr ::= PrimaryExpr | FilterExpr Predicate
	// Predicate ::= '[' PredicateExpr ']'
	// PredicateExpr ::= Expr
	xpath_ast_node* parse_filter_expression()
	{
		xpath_ast_node* n = parse_primary_expression();

		while (_lexer.current() == lex_open_square_brace)
		{
			_lexer.next();

			xpath_ast_node* expr = parse_expression();

			if (n->rettype() != xpath_type_node_set)
				throw_error("Predicate has to be applied to node set");

			bool posinv = expr->rettype() != xpath_type_number && expr->is_posinv();

			n = new (alloc_node())
				xpath_ast_node(posinv ? ast_filter_posinv : ast_filter, xpath_type_node_set, n, expr);

			if (_lexer.current() != lex_close_square_brace)
				throw_error("Unmatched square brace");

			_lexer.next();
		}

		return n;
	}

	// Step ::= AxisSpecifier NodeTest Predicate* | AbbreviatedStep
	// AxisSpecifier ::= AxisName '::' | '@'?
	// NodeTest ::= NameTest | NodeType '(' ')' | 'processing-instruction' '(' Literal ')'
	// NameTest ::= '*' | NCName ':' '*' | QName
	// AbbreviatedStep ::= '.' | '..'
	xpath_ast_node* parse_step(xpath_ast_node* set)
	{
		if (set && set->rettype() != xpath_type_node_set)
			throw_error("Step has to be applied to node set");

		bool axis_specified = false;
		axis_t axis = axis_child; // implied child axis

		if (_lexer.current() == lex_axis_attribute)
		{
			axis = axis_attribute;
			axis_specified = true;

			_lexer.next();
		}
		else if (_lexer.current() == lex_dot)
		{
			_lexer.next();

			return new (alloc_node()) xpath_ast_node(ast_step, set, axis_self, nodetest_type_node, 0);
		}
		else if (_lexer.current() == lex_double_dot)
		{
			_lexer.next();

			return new (alloc_node()) xpath_ast_node(ast_step, set, axis_parent, nodetest_type_node, 0);
		}

		nodetest_t nt_type = nodetest_none;
		xpath_lexer_string nt_name;

		if (_lexer.current() == lex_string)
		{
			// node name test
			nt_name = _lexer.contents();
			_lexer.next();

			// was it an axis name?
			if (_lexer.current() == lex_double_colon)
			{
				// parse axis name
				if (axis_specified)
					throw_error("Two axis specifiers in one step");

				axis = parse_axis_name(nt_name, axis_specified);

				if (!axis_specified)
					throw_error("Unknown axis");

				// read actual node test
				_lexer.next();

				if (_lexer.current() == lex_multiply)
				{
					nt_type = nodetest_all;
					nt_name = xpath_lexer_string();
					_lexer.next();
				}
				else if (_lexer.current() == lex_string)
				{
					nt_name = _lexer.contents();
					_lexer.next();
				}
				else
					throw_error("Unrecognized node test");
			}

			if (nt_type == nodetest_none)
			{
				// node type test or processing-instruction
				if (_lexer.current() == lex_open_brace)
				{
					_lexer.next();

					if (_lexer.current() == lex_close_brace)
					{
						_lexer.next();

						nt_type = parse_node_test_type(nt_name);

						if (nt_type == nodetest_none)
							throw_error("Unrecognized node type");

						nt_name = xpath_lexer_string();
					}
					else if (nt_name == PUGIXML_TEXT("processing-instruction"))
					{
						if (_lexer.current() != lex_quoted_string)
							throw_error("Only literals are allowed as arguments to processing-instruction()");

						nt_type = nodetest_pi;
						nt_name = _lexer.contents();
						_lexer.next();

						if (_lexer.current() != lex_close_brace)
							throw_error("Unmatched brace near processing-instruction()");
						_lexer.next();
					}
					else
						throw_error("Unmatched brace near node type test");
				}
				// QName or NCName:*
				else
				{
					if (nt_name.end - nt_name.begin > 2 && nt_name.end[-2] == ':' && nt_name.end[-1] == '*') // NCName:*
					{
						nt_name.end--; // erase *

						nt_type = nodetest_all_in_namespace;
					}
					else
						nt_type = nodetest_name;
				}
			}
		}
		else if (_lexer.current() == lex_multiply)
		{
			nt_type = nodetest_all;
			_lexer.next();
		}
		else
			throw_error("Unrecognized node test");

		xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step, set, axis, nt_type, alloc_string(nt_name));

		xpath_ast_node* last = nullptr;

		while (_lexer.current() == lex_open_square_brace)
		{
			_lexer.next();

			xpath_ast_node* expr = parse_expression();

			xpath_ast_node* pred = new (alloc_node()) xpath_ast_node(ast_predicate, xpath_type_node_set, expr);

			if (_lexer.current() != lex_close_square_brace)
				throw_error("Unmatched square brace");
			_lexer.next();

			if (last)
				last->set_next(pred);
			else
				n->set_right(pred);

			last = pred;
		}

		return n;
	}

	// RelativeLocationPath ::= Step | RelativeLocationPath '/' Step | RelativeLocationPath '//' Step
	xpath_ast_node* parse_relative_location_path(xpath_ast_node* set)
	{
		xpath_ast_node* n = parse_step(set);

		while (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
		{
			lexeme_t l = _lexer.current();
			_lexer.next();

			if (l == lex_double_slash)
				n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);

			n = parse_step(n);
		}

		return n;
	}

	// LocationPath ::= RelativeLocationPath | AbsoluteLocationPath
	// AbsoluteLocationPath ::= '/' RelativeLocationPath? | '//' RelativeLocationPath
	xpath_ast_node* parse_location_path()
	{
		if (_lexer.current() == lex_slash)
		{
			_lexer.next();

			xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step_root, xpath_type_node_set);

			// relative location path can start from axis_attribute, dot, double_dot, multiply and string lexemes; any other lexeme means standalone root path
			lexeme_t l = _lexer.current();

			if (l == lex_string || l == lex_axis_attribute || l == lex_dot || l == lex_double_dot || l == lex_multiply)
				return parse_relative_location_path(n);
			else
				return n;
		}
		else if (_lexer.current() == lex_double_slash)
		{
			_lexer.next();

			xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step_root, xpath_type_node_set);
			n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);

			return parse_relative_location_path(n);
		}

		// else clause moved outside of if because of bogus warning 'control may reach end of non-void function being inlined' in gcc 4.0.1
		return parse_relative_location_path(0);
	}

	// PathExpr ::= LocationPath
	//				| FilterExpr
	//				| FilterExpr '/' RelativeLocationPath
	//				| FilterExpr '//' RelativeLocationPath
	xpath_ast_node* parse_path_expression()
	{
		// Clarification.
		// PathExpr begins with either LocationPath or FilterExpr.
		// FilterExpr begins with PrimaryExpr
		// PrimaryExpr begins with '$' in case of it being a variable reference,
		// '(' in case of it being an expression, string literal, number constant or
		// function call.

		if (_lexer.current() == lex_var_ref || _lexer.current() == lex_open_brace ||
			_lexer.current() == lex_quoted_string || _lexer.current() == lex_number || _lexer.current() == lex_string)
		{
			if (_lexer.current() == lex_string)
			{
				// This is either a function call, or not - if not, we shall proceed with location path
				const char_t* state = _lexer.state();

				while (PUGI__IS_CHARTYPE(*state, ct_space))
					++state;

				if (*state != '(')
					return parse_location_path();

				// This looks like a function call; however this still can be a node-test. Check it.
				if (parse_node_test_type(_lexer.contents()) != nodetest_none)
					return parse_location_path();
			}

			xpath_ast_node* n = parse_filter_expression();

			if (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
			{
				lexeme_t l = _lexer.current();
				_lexer.next();

				if (l == lex_double_slash)
				{
					if (n->rettype() != xpath_type_node_set)
						throw_error("Step has to be applied to node set");

					n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
				}

				// select from location path
				return parse_relative_location_path(n);
			}

			return n;
		}
		else
			return parse_location_path();
	}

	// UnionExpr ::= PathExpr | UnionExpr '|' PathExpr
	xpath_ast_node* parse_union_expression()
	{
		xpath_ast_node* n = parse_path_expression();

		while (_lexer.current() == lex_union)
		{
			_lexer.next();

			xpath_ast_node* expr = parse_union_expression();

			if (n->rettype() != xpath_type_node_set || expr->rettype() != xpath_type_node_set)
				throw_error("Union operator has to be applied to node sets");

			n = new (alloc_node()) xpath_ast_node(ast_op_union, xpath_type_node_set, n, expr);
		}

		return n;
	}

	// UnaryExpr ::= UnionExpr | '-' UnaryExpr
	xpath_ast_node* parse_unary_expression()
	{
		if (_lexer.current() == lex_minus)
		{
			_lexer.next();

			xpath_ast_node* expr = parse_unary_expression();

			return new (alloc_node()) xpath_ast_node(ast_op_negate, xpath_type_number, expr);
		}
		else
			return parse_union_expression();
	}

	// MultiplicativeExpr ::= UnaryExpr
	//						  | MultiplicativeExpr '*' UnaryExpr
	//						  | MultiplicativeExpr 'div' UnaryExpr
	//						  | MultiplicativeExpr 'mod' UnaryExpr
	xpath_ast_node* parse_multiplicative_expression()
	{
		xpath_ast_node* n = parse_unary_expression();

		while (_lexer.current() == lex_multiply ||
			   (_lexer.current() == lex_string &&
				(_lexer.contents() == PUGIXML_TEXT("mod") || _lexer.contents() == PUGIXML_TEXT("div"))))
		{
			ast_type_t op = _lexer.current() == lex_multiply ? ast_op_multiply
				: _lexer.contents().begin[0] == 'd'
				? ast_op_divide
				: ast_op_mod;
			_lexer.next();

			xpath_ast_node* expr = parse_unary_expression();

			n = new (alloc_node()) xpath_ast_node(op, xpath_type_number, n, expr);
		}

		return n;
	}

	// AdditiveExpr ::= MultiplicativeExpr
	//					| AdditiveExpr '+' MultiplicativeExpr
	//					| AdditiveExpr '-' MultiplicativeExpr
	xpath_ast_node* parse_additive_expression()
	{
		xpath_ast_node* n = parse_multiplicative_expression();

		while (_lexer.current() == lex_plus || _lexer.current() == lex_minus)
		{
			lexeme_t l = _lexer.current();

			_lexer.next();

			xpath_ast_node* expr = parse_multiplicative_expression();

			n = new (alloc_node())
				xpath_ast_node(l == lex_plus ? ast_op_add : ast_op_subtract, xpath_type_number, n, expr);
		}

		return n;
	}

	// RelationalExpr ::= AdditiveExpr
	//					  | RelationalExpr '<' AdditiveExpr
	//					  | RelationalExpr '>' AdditiveExpr
	//					  | RelationalExpr '<=' AdditiveExpr
	//					  | RelationalExpr '>=' AdditiveExpr
	xpath_ast_node* parse_relational_expression()
	{
		xpath_ast_node* n = parse_additive_expression();

		while (_lexer.current() == lex_less || _lexer.current() == lex_less_or_equal ||
			   _lexer.current() == lex_greater || _lexer.current() == lex_greater_or_equal)
		{
			lexeme_t l = _lexer.current();
			_lexer.next();

			xpath_ast_node* expr = parse_additive_expression();

			n = new (alloc_node()) xpath_ast_node(
				l == lex_less				 ? ast_op_less
					: l == lex_greater		 ? ast_op_greater
					: l == lex_less_or_equal ? ast_op_less_or_equal
											 : ast_op_greater_or_equal,
				xpath_type_boolean,
				n,
				expr);
		}

		return n;
	}

	// EqualityExpr ::= RelationalExpr
	//					| EqualityExpr '=' RelationalExpr
	//					| EqualityExpr '!=' RelationalExpr
	xpath_ast_node* parse_equality_expression()
	{
		xpath_ast_node* n = parse_relational_expression();

		while (_lexer.current() == lex_equal || _lexer.current() == lex_not_equal)
		{
			lexeme_t l = _lexer.current();

			_lexer.next();

			xpath_ast_node* expr = parse_relational_expression();

			n = new (alloc_node())
				xpath_ast_node(l == lex_equal ? ast_op_equal : ast_op_not_equal, xpath_type_boolean, n, expr);
		}

		return n;
	}

	// AndExpr ::= EqualityExpr | AndExpr 'and' EqualityExpr
	xpath_ast_node* parse_and_expression()
	{
		xpath_ast_node* n = parse_equality_expression();

		while (_lexer.current() == lex_string && _lexer.contents() == PUGIXML_TEXT("and"))
		{
			_lexer.next();

			xpath_ast_node* expr = parse_equality_expression();

			n = new (alloc_node()) xpath_ast_node(ast_op_and, xpath_type_boolean, n, expr);
		}

		return n;
	}

	// OrExpr ::= AndExpr | OrExpr 'or' AndExpr
	xpath_ast_node* parse_or_expression()
	{
		xpath_ast_node* n = parse_and_expression();

		while (_lexer.current() == lex_string && _lexer.contents() == PUGIXML_TEXT("or"))
		{
			_lexer.next();

			xpath_ast_node* expr = parse_and_expression();

			n = new (alloc_node()) xpath_ast_node(ast_op_or, xpath_type_boolean, n, expr);
		}

		return n;
	}

	// Expr ::= OrExpr
	xpath_ast_node* parse_expression()
	{
		return parse_or_expression();
	}

	xpath_parser(
		const char_t* query,
		xpath_variable_set* variables,
		xpath_allocator* alloc,
		xpath_parse_result* result) :
		_alloc(alloc),
		_lexer(query),
		_query(query),
		_variables(variables),
		_result(result)
	{
	}

	xpath_ast_node* parse()
	{
		xpath_ast_node* result = parse_expression();

		if (_lexer.current() != lex_eof)
		{
			// there are still unparsed tokens left, error
			throw_error("Incorrect query");
		}

		return result;
	}

	static xpath_ast_node* parse(
		const char_t* query,
		xpath_variable_set* variables,
		xpath_allocator* alloc,
		xpath_parse_result* result)
	{
		xpath_parser parser(query, variables, alloc, result);

	#ifdef PUGIXML_NO_EXCEPTIONS
		int error = setjmp(parser._error_handler);

		return (error == 0) ? parser.parse() : 0;
	#else
		return parser.parse();
	#endif
	}
};

struct xpath_query_impl
{
	static xpath_query_impl* create()
	{
		void* memory = xml_memory::allocate(sizeof(xpath_query_impl));

		return new (memory) xpath_query_impl();
	}

	static void destroy(void* ptr)
	{
		if (!ptr)
			return;

		// free all allocated pages
		static_cast<xpath_query_impl*>(ptr)->alloc.release();

		// free allocator memory (with the first page)
		xml_memory::deallocate(ptr);
	}

	xpath_query_impl() :
		root(0),
		alloc(&block)
	{
		block.next = 0;
	}

	xpath_ast_node* root;
	xpath_allocator alloc;
	xpath_memory_block block;
};

PUGI__FN xpath_string evaluate_string_impl(xpath_query_impl* impl, const xpath_node& n, xpath_stack_data& sd)
{
	if (!impl)
		return xpath_string();

	#ifdef PUGIXML_NO_EXCEPTIONS
	if (setjmp(sd.error_handler))
		return xpath_string();
	#endif

	xpath_context c(n, 1, 1);

	return impl->root->eval_string(c, sd.stack);
}
PUGI__NS_END

namespace pugi
{
	#ifndef PUGIXML_NO_EXCEPTIONS
	PUGI__FN xpath_exception::xpath_exception(const xpath_parse_result& result_) :
		_result(result_)
	{
		assert(_result.error);
	}

	PUGI__FN const char* xpath_exception::what() const noexcept
	{
		return _result.error;
	}

	PUGI__FN const xpath_parse_result& xpath_exception::result() const
	{
		return _result;
	}
	#endif

	PUGI__FN xpath_node::xpath_node()
	{
	}

	PUGI__FN xpath_node::xpath_node(const xml_node& node_) :
		_node(node_)
	{
	}

	PUGI__FN xpath_node::xpath_node(const xml_attribute& attribute_, const xml_node& parent_) :
		_node(attribute_ ? parent_ : xml_node()),
		_attribute(attribute_)
	{
	}

	PUGI__FN xml_node xpath_node::node() const
	{
		return _attribute ? xml_node() : _node;
	}

	PUGI__FN xml_attribute xpath_node::attribute() const
	{
		return _attribute;
	}

	PUGI__FN xml_node xpath_node::parent() const
	{
		return _attribute ? _node : _node.parent();
	}

	PUGI__FN static void unspecified_bool_xpath_node(xpath_node***)
	{
	}

	PUGI__FN xpath_node::operator xpath_node::unspecified_bool_type() const
	{
		return (_node || _attribute) ? unspecified_bool_xpath_node : 0;
	}

	PUGI__FN bool xpath_node::operator!() const
	{
		return !(_node || _attribute);
	}

	PUGI__FN bool xpath_node::operator==(const xpath_node& n) const
	{
		return _node == n._node && _attribute == n._attribute;
	}

	PUGI__FN bool xpath_node::operator!=(const xpath_node& n) const
	{
		return _node != n._node || _attribute != n._attribute;
	}

	#ifdef __BORLANDC__
	PUGI__FN bool operator&&(const xpath_node& lhs, bool rhs)
	{
		return (bool)lhs && rhs;
	}

	PUGI__FN bool operator||(const xpath_node& lhs, bool rhs)
	{
		return (bool)lhs || rhs;
	}
	#endif

	PUGI__FN void xpath_node_set::_assign(const_iterator begin_, const_iterator end_)
	{
		assert(begin_ <= end_);

		size_t size_ = static_cast<size_t>(end_ - begin_);

		if (size_ <= 1)
		{
			// deallocate old buffer
			if (_begin != &_storage)
				impl::xml_memory::deallocate(_begin);

			// use internal buffer
			if (begin_ != end_)
				_storage = *begin_;

			_begin = &_storage;
			_end = &_storage + size_;
		}
		else
		{
			// make heap copy
			xpath_node* storage = static_cast<xpath_node*>(impl::xml_memory::allocate(size_ * sizeof(xpath_node)));

			if (!storage)
			{
	#ifdef PUGIXML_NO_EXCEPTIONS
				return;
	#else
				throw std::bad_alloc();
	#endif
			}

			memcpy(storage, begin_, size_ * sizeof(xpath_node));

			// deallocate old buffer
			if (_begin != &_storage)
				impl::xml_memory::deallocate(_begin);

			// finalize
			_begin = storage;
			_end = storage + size_;
		}
	}

	PUGI__FN xpath_node_set::xpath_node_set() :
		_type(type_unsorted),
		_begin(&_storage),
		_end(&_storage)
	{
	}

	PUGI__FN xpath_node_set::xpath_node_set(const_iterator begin_, const_iterator end_, type_t type_) :
		_type(type_),
		_begin(&_storage),
		_end(&_storage)
	{
		_assign(begin_, end_);
	}

	PUGI__FN xpath_node_set::~xpath_node_set()
	{
		if (_begin != &_storage)
			impl::xml_memory::deallocate(_begin);
	}

	PUGI__FN xpath_node_set::xpath_node_set(const xpath_node_set& ns) :
		_type(ns._type),
		_begin(&_storage),
		_end(&_storage)
	{
		_assign(ns._begin, ns._end);
	}

	PUGI__FN xpath_node_set& xpath_node_set::operator=(const xpath_node_set& ns)
	{
		if (this == &ns)
			return *this;

		_type = ns._type;
		_assign(ns._begin, ns._end);

		return *this;
	}

	PUGI__FN xpath_node_set::type_t xpath_node_set::type() const
	{
		return _type;
	}

	PUGI__FN size_t xpath_node_set::size() const
	{
		return _end - _begin;
	}

	PUGI__FN bool xpath_node_set::empty() const
	{
		return _begin == _end;
	}

	PUGI__FN const xpath_node& xpath_node_set::operator[](size_t index) const
	{
		assert(index < size());
		return _begin[index];
	}

	PUGI__FN xpath_node_set::const_iterator xpath_node_set::begin() const
	{
		return _begin;
	}

	PUGI__FN xpath_node_set::const_iterator xpath_node_set::end() const
	{
		return _end;
	}

	PUGI__FN void xpath_node_set::sort(bool reverse)
	{
		_type = impl::xpath_sort(_begin, _end, _type, reverse);
	}

	PUGI__FN xpath_node xpath_node_set::first() const
	{
		return impl::xpath_first(_begin, _end, _type);
	}

	PUGI__FN xpath_parse_result::xpath_parse_result() :
		error("Internal error"),
		offset(0)
	{
	}

	PUGI__FN xpath_parse_result::operator bool() const
	{
		return error == 0;
	}

	PUGI__FN const char* xpath_parse_result::description() const
	{
		return error ? error : "No error";
	}

	PUGI__FN xpath_variable::xpath_variable()
	{
	}

	PUGI__FN const char_t* xpath_variable::name() const
	{
		switch (_type)
		{
		case xpath_type_node_set: return static_cast<const impl::xpath_variable_node_set*>(this)->name;

		case xpath_type_number: return static_cast<const impl::xpath_variable_number*>(this)->name;

		case xpath_type_string: return static_cast<const impl::xpath_variable_string*>(this)->name;

		case xpath_type_boolean: return static_cast<const impl::xpath_variable_boolean*>(this)->name;

		default: assert(!"Invalid variable type"); return 0;
		}
	}

	PUGI__FN xpath_value_type xpath_variable::type() const
	{
		return _type;
	}

	PUGI__FN bool xpath_variable::get_boolean() const
	{
		return (_type == xpath_type_boolean) ? static_cast<const impl::xpath_variable_boolean*>(this)->value : false;
	}

	PUGI__FN double xpath_variable::get_number() const
	{
		return (_type == xpath_type_number)
			? static_cast<const impl::xpath_variable_number*>(this)->value
			: impl::gen_nan();
	}

	PUGI__FN const char_t* xpath_variable::get_string() const
	{
		const char_t* value =
			(_type == xpath_type_string) ? static_cast<const impl::xpath_variable_string*>(this)->value : 0;
		return value ? value : PUGIXML_TEXT("");
	}

	PUGI__FN const xpath_node_set& xpath_variable::get_node_set() const
	{
		return (_type == xpath_type_node_set)
			? static_cast<const impl::xpath_variable_node_set*>(this)->value
			: impl::dummy_node_set;
	}

	PUGI__FN bool xpath_variable::set(bool value)
	{
		if (_type != xpath_type_boolean)
			return false;

		static_cast<impl::xpath_variable_boolean*>(this)->value = value;
		return true;
	}

	PUGI__FN bool xpath_variable::set(double value)
	{
		if (_type != xpath_type_number)
			return false;

		static_cast<impl::xpath_variable_number*>(this)->value = value;
		return true;
	}

	PUGI__FN bool xpath_variable::set(const char_t* value)
	{
		if (_type != xpath_type_string)
			return false;

		impl::xpath_variable_string* var = static_cast<impl::xpath_variable_string*>(this);

		// duplicate string
		size_t size = (impl::strlength(value) + 1) * sizeof(char_t);

		char_t* copy = static_cast<char_t*>(impl::xml_memory::allocate(size));
		if (!copy)
			return false;

		memcpy(copy, value, size);

		// replace old string
		if (var->value)
			impl::xml_memory::deallocate(var->value);
		var->value = copy;

		return true;
	}

	PUGI__FN bool xpath_variable::set(const xpath_node_set& value)
	{
		if (_type != xpath_type_node_set)
			return false;

		static_cast<impl::xpath_variable_node_set*>(this)->value = value;
		return true;
	}

	PUGI__FN xpath_variable_set::xpath_variable_set()
	{
		for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
			_data[i] = 0;
	}

	PUGI__FN xpath_variable_set::~xpath_variable_set()
	{
		for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
		{
			xpath_variable* var = _data[i];

			while (var)
			{
				xpath_variable* next = var->_next;

				impl::delete_xpath_variable(var->_type, var);

				var = next;
			}
		}
	}

	PUGI__FN xpath_variable* xpath_variable_set::find(const char_t* name) const
	{
		const size_t hash_size = sizeof(_data) / sizeof(_data[0]);
		size_t hash = impl::hash_string(name) % hash_size;

		// look for existing variable
		for (xpath_variable* var = _data[hash]; var; var = var->_next)
			if (impl::strequal(var->name(), name))
				return var;

		return 0;
	}

	PUGI__FN xpath_variable* xpath_variable_set::add(const char_t* name, xpath_value_type type)
	{
		const size_t hash_size = sizeof(_data) / sizeof(_data[0]);
		size_t hash = impl::hash_string(name) % hash_size;

		// look for existing variable
		for (xpath_variable* var = _data[hash]; var; var = var->_next)
			if (impl::strequal(var->name(), name))
				return var->type() == type ? var : 0;

		// add new variable
		xpath_variable* result = impl::new_xpath_variable(type, name);

		if (result)
		{
			result->_type = type;
			result->_next = _data[hash];

			_data[hash] = result;
		}

		return result;
	}

	PUGI__FN bool xpath_variable_set::set(const char_t* name, bool value)
	{
		xpath_variable* var = add(name, xpath_type_boolean);
		return var ? var->set(value) : false;
	}

	PUGI__FN bool xpath_variable_set::set(const char_t* name, double value)
	{
		xpath_variable* var = add(name, xpath_type_number);
		return var ? var->set(value) : false;
	}

	PUGI__FN bool xpath_variable_set::set(const char_t* name, const char_t* value)
	{
		xpath_variable* var = add(name, xpath_type_string);
		return var ? var->set(value) : false;
	}

	PUGI__FN bool xpath_variable_set::set(const char_t* name, const xpath_node_set& value)
	{
		xpath_variable* var = add(name, xpath_type_node_set);
		return var ? var->set(value) : false;
	}

	PUGI__FN xpath_variable* xpath_variable_set::get(const char_t* name)
	{
		return find(name);
	}

	PUGI__FN const xpath_variable* xpath_variable_set::get(const char_t* name) const
	{
		return find(name);
	}

	PUGI__FN xpath_query::xpath_query(const char_t* query, xpath_variable_set* variables) :
		_impl(0)
	{
		impl::xpath_query_impl* qimpl = impl::xpath_query_impl::create();

		if (!qimpl)
		{
	#ifdef PUGIXML_NO_EXCEPTIONS
			_result.error = "Out of memory";
	#else
			throw std::bad_alloc();
	#endif
		}
		else
		{
			impl::buffer_holder impl_holder(qimpl, impl::xpath_query_impl::destroy);

			qimpl->root = impl::xpath_parser::parse(query, variables, &qimpl->alloc, &_result);

			if (qimpl->root)
			{
				_impl = static_cast<impl::xpath_query_impl*>(impl_holder.release());
				_result.error = 0;
			}
		}
	}

	PUGI__FN xpath_query::~xpath_query()
	{
		impl::xpath_query_impl::destroy(_impl);
	}

	PUGI__FN xpath_value_type xpath_query::return_type() const
	{
		if (!_impl)
			return xpath_type_none;

		return static_cast<impl::xpath_query_impl*>(_impl)->root->rettype();
	}

	PUGI__FN bool xpath_query::evaluate_boolean(const xpath_node& n) const
	{
		if (!_impl)
			return false;

		impl::xpath_context c(n, 1, 1);
		impl::xpath_stack_data sd;

	#ifdef PUGIXML_NO_EXCEPTIONS
		if (setjmp(sd.error_handler))
			return false;
	#endif

		return static_cast<impl::xpath_query_impl*>(_impl)->root->eval_boolean(c, sd.stack);
	}

	PUGI__FN double xpath_query::evaluate_number(const xpath_node& n) const
	{
		if (!_impl)
			return impl::gen_nan();

		impl::xpath_context c(n, 1, 1);
		impl::xpath_stack_data sd;

	#ifdef PUGIXML_NO_EXCEPTIONS
		if (setjmp(sd.error_handler))
			return impl::gen_nan();
	#endif

		return static_cast<impl::xpath_query_impl*>(_impl)->root->eval_number(c, sd.stack);
	}

	#ifndef PUGIXML_NO_STL
	PUGI__FN string_t xpath_query::evaluate_string(const xpath_node& n) const
	{
		impl::xpath_stack_data sd;

		return impl::evaluate_string_impl(static_cast<impl::xpath_query_impl*>(_impl), n, sd).c_str();
	}
	#endif

	PUGI__FN size_t xpath_query::evaluate_string(char_t* buffer, size_t capacity, const xpath_node& n) const
	{
		impl::xpath_stack_data sd;

		impl::xpath_string r = impl::evaluate_string_impl(static_cast<impl::xpath_query_impl*>(_impl), n, sd);

		size_t full_size = r.length() + 1;

		if (capacity > 0)
		{
			size_t size = (full_size < capacity) ? full_size : capacity;
			assert(size > 0);

			memcpy(buffer, r.c_str(), (size - 1) * sizeof(char_t));
			buffer[size - 1] = 0;
		}

		return full_size;
	}

	PUGI__FN xpath_node_set xpath_query::evaluate_node_set(const xpath_node& n) const
	{
		if (!_impl)
			return xpath_node_set();

		impl::xpath_ast_node* root = static_cast<impl::xpath_query_impl*>(_impl)->root;

		if (root->rettype() != xpath_type_node_set)
		{
	#ifdef PUGIXML_NO_EXCEPTIONS
			return xpath_node_set();
	#else
			xpath_parse_result res;
			res.error = "Expression does not evaluate to node set";

			throw xpath_exception(res);
	#endif
		}

		impl::xpath_context c(n, 1, 1);
		impl::xpath_stack_data sd;

	#ifdef PUGIXML_NO_EXCEPTIONS
		if (setjmp(sd.error_handler))
			return xpath_node_set();
	#endif

		impl::xpath_node_set_raw r = root->eval_node_set(c, sd.stack);

		return xpath_node_set(r.begin(), r.end(), r.type());
	}

	PUGI__FN const xpath_parse_result& xpath_query::result() const
	{
		return _result;
	}

	PUGI__FN static void unspecified_bool_xpath_query(xpath_query***)
	{
	}

	PUGI__FN xpath_query::operator xpath_query::unspecified_bool_type() const
	{
		return _impl ? unspecified_bool_xpath_query : 0;
	}

	PUGI__FN bool xpath_query::operator!() const
	{
		return !_impl;
	}

	PUGI__FN xpath_node xml_node::select_single_node(const char_t* query, xpath_variable_set* variables) const
	{
		xpath_query q(query, variables);
		return select_single_node(q);
	}

	PUGI__FN xpath_node xml_node::select_single_node(const xpath_query& query) const
	{
		xpath_node_set s = query.evaluate_node_set(*this);
		return s.empty() ? xpath_node() : s.first();
	}

	PUGI__FN xpath_node_set xml_node::select_nodes(const char_t* query, xpath_variable_set* variables) const
	{
		xpath_query q(query, variables);
		return select_nodes(q);
	}

	PUGI__FN xpath_node_set xml_node::select_nodes(const xpath_query& query) const
	{
		return query.evaluate_node_set(*this);
	}
}

#endif

#ifdef __BORLANDC__
	#pragma option pop
#endif

// Intel C++ does not properly keep warning state for function templates,
// so popping warning state at the end of translation unit leads to warnings in the middle.
#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
	#pragma warning(pop)
#endif

// Undefine all local macros (makes sure we're not leaking macros in header-only mode)
#undef PUGI__NO_INLINE
#undef PUGI__STATIC_ASSERT
#undef PUGI__DMC_VOLATILE
#undef PUGI__MSVC_CRT_VERSION
#undef PUGI__NS_BEGIN
#undef PUGI__NS_END
#undef PUGI__FN
#undef PUGI__FN_NO_INLINE
#undef PUGI__IS_CHARTYPE_IMPL
#undef PUGI__IS_CHARTYPE
#undef PUGI__IS_CHARTYPEX
#undef PUGI__SKIPWS
#undef PUGI__OPTSET
#undef PUGI__PUSHNODE
#undef PUGI__POPNODE
#undef PUGI__SCANFOR
#undef PUGI__SCANWHILE
#undef PUGI__ENDSEG
#undef PUGI__THROW_ERROR
#undef PUGI__CHECK_ERROR

//#endif

/**
 * Copyright (c) 2006-2012 Arseny Kapoulkine
 *
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
 * files (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use,
 * copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following
 * conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 * 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */
