Changeset 33a694 for ThirdParty

Timestamp:

May 5, 2017, 2:17:23 PM (8 years ago)

Author:

Frederik Heber <frederik.heber@…>

Branches:

Action_Thermostats, Add_AtomRandomPerturbation, Add_RotateAroundBondAction, Add_SelectAtomByNameAction, Adding_Graph_to_ChangeBondActions, Adding_MD_integration_tests, Adding_StructOpt_integration_tests, AutomationFragmentation_failures, Candidate_v1.6.1, ChangeBugEmailaddress, ChangingTestPorts, ChemicalSpaceEvaluator, Docu_Python_wait, EmpiricalPotential_contain_HomologyGraph_documentation, Enhance_userguide, Enhanced_StructuralOptimization, Enhanced_StructuralOptimization_continued, Example_ManyWaysToTranslateAtom, Exclude_Hydrogens_annealWithBondGraph, Fix_ChronosMutex, Fix_StatusMsg, Fix_StepWorldTime_single_argument, Fix_Verbose_Codepatterns, ForceAnnealing_goodresults, ForceAnnealing_oldresults, ForceAnnealing_tocheck, ForceAnnealing_with_BondGraph, ForceAnnealing_with_BondGraph_continued, ForceAnnealing_with_BondGraph_continued_betteresults, ForceAnnealing_with_BondGraph_contraction-expansion, GeometryObjects, Gui_displays_atomic_force_velocity, IndependentFragmentGrids_IntegrationTest, JobMarket_RobustOnKillsSegFaults, JobMarket_StableWorkerPool, PythonUI_with_named_parameters, QtGui_reactivate_TimeChanged_changes, Recreated_GuiChecks, RotateToPrincipalAxisSystem_UndoRedo, StoppableMakroAction, TremoloParser_IncreasedPrecision, TremoloParser_MultipleTimesteps, Ubuntu_1604_changes

Children:

dde50d

Parents:

6798a5

git-author:

Frederik Heber <heber@…> (04/05/17 17:17:45)

git-committer:

Frederik Heber <frederik.heber@…> (05/05/17 14:17:23)

Message:

Updated pugixml in ThirdParty/vmg and src from 1.0 to 1.8.

• this solved compilation issues with 1.0 seemingly from too recent gcc and c++11/14/.. changes.

Location:

ThirdParty/vmg/src/thirdparty/pugixml

Files:

• pugiconfig.hpp (modified) (4 diffs)
• pugixml.cpp (modified) (377 diffs)
• pugixml.hpp (modified) (64 diffs)
• readme.txt (modified) (2 diffs)

ThirdParty/vmg/src/thirdparty/pugixml/pugiconfig.hpp

@@ -1 +1 @@
 /**
- * pugixml parser - version 1.0
+ * pugixml parser - version 1.8
  * --------------------------------------------------------
- * Copyright (C) 2006-2010, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
+ * Copyright (C) 2006-2016, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
  * Report bugs and download new versions at http://pugixml.org/
  *
@@ -18 +18 @@
 // #define PUGIXML_WCHAR_MODE
 
+// Uncomment this to enable compact mode
+// #define PUGIXML_COMPACT
+
 // Uncomment this to disable XPath
 // #define PUGIXML_NO_XPATH
 
 // Uncomment this to disable STL
-// Note: you can't use XPath with PUGIXML_NO_STL
 // #define PUGIXML_NO_STL
 
 // Uncomment this to disable exceptions
-// Note: you can't use XPath with PUGIXML_NO_EXCEPTIONS
 // #define PUGIXML_NO_EXCEPTIONS
 
@@ -35 +36 @@
 // In absence of PUGIXML_CLASS/PUGIXML_FUNCTION definitions PUGIXML_API is used instead
 
+// Tune these constants to adjust memory-related behavior
+// #define PUGIXML_MEMORY_PAGE_SIZE 32768
+// #define PUGIXML_MEMORY_OUTPUT_STACK 10240
+// #define PUGIXML_MEMORY_XPATH_PAGE_SIZE 4096
+
+// Uncomment this to switch to header-only version
+// #define PUGIXML_HEADER_ONLY
+
+// Uncomment this to enable long long support
+// #define PUGIXML_HAS_LONG_LONG
+
 #endif
 
 /**
- * Copyright (c) 2006-2010 Arseny Kapoulkine
+ * Copyright (c) 2006-2016 Arseny Kapoulkine
  *
  * Permission is hereby granted, free of charge, to any person
@@ -51 +63 @@
  * The above copyright notice and this permission notice shall be
  * included in all copies or substantial portions of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES

ThirdParty/vmg/src/thirdparty/pugixml/pugixml.cpp

@@ -1 +1 @@
 /**
- * pugixml parser - version 1.0
+ * pugixml parser - version 1.8
  * --------------------------------------------------------
- * Copyright (C) 2006-2010, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
+ * Copyright (C) 2006-2016, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
  * Report bugs and download new versions at http://pugixml.org/
  *
@@ -12 +12 @@
  */
 
+#ifndef SOURCE_PUGIXML_CPP
+#define SOURCE_PUGIXML_CPP
+
 #include "pugixml.hpp"
 
@@ -18 +21 @@
 #include <string.h>
 #include <assert.h>
-#include <setjmp.h>
-#include <wchar.h>
+#include <limits.h>
+
+#ifdef PUGIXML_WCHAR_MODE
+#       include <wchar.h>
+#endif
 
 #ifndef PUGIXML_NO_XPATH
 #       include <math.h>
 #       include <float.h>
+#       ifdef PUGIXML_NO_EXCEPTIONS
+#               include <setjmp.h>
+#       endif
 #endif
 
@@ -36 +45 @@
 
 #ifdef _MSC_VER
+#       pragma warning(push)
 #       pragma warning(disable: 4127) // conditional expression is constant
 #       pragma warning(disable: 4324) // structure was padded due to __declspec(align())
@@ -41 +51 @@
 #       pragma warning(disable: 4702) // unreachable code
 #       pragma warning(disable: 4996) // this function or variable may be unsafe
+#       pragma warning(disable: 4793) // function compiled as native: presence of '_setjmp' makes a function unmanaged
 #endif
 
 #ifdef __INTEL_COMPILER
-#       pragma warning(disable: 177) // function was declared but never referenced 
+#       pragma warning(disable: 177) // function was declared but never referenced
 #       pragma warning(disable: 279) // controlling expression is constant
 #       pragma warning(disable: 1478 1786) // function was declared "deprecated"
+#       pragma warning(disable: 1684) // conversion from pointer to same-sized integral type
 #endif
 
+#if defined(__BORLANDC__) && defined(PUGIXML_HEADER_ONLY)
+#       pragma warn -8080 // symbol is declared but never used; disabling this inside push/pop bracket does not make the warning go away
+#endif
+
 #ifdef __BORLANDC__
+#       pragma option push
 #       pragma warn -8008 // condition is always false
 #       pragma warn -8066 // unreachable code
@@ -55 +72 @@
 
 #ifdef __SNC__
+// Using diag_push/diag_pop does not disable the warnings inside templates due to a compiler bug
 #       pragma diag_suppress=178 // function was declared but never referenced
 #       pragma diag_suppress=237 // controlling expression is constant
 #endif
 
-// uintptr_t
-#if !defined(_MSC_VER) || _MSC_VER >= 1600
-#       include <stdint.h>
-#else
-#       if _MSC_VER < 1300
-// No native uintptr_t in MSVC6
-typedef size_t uintptr_t;
-#       endif
-typedef unsigned __int8 uint8_t;
-typedef unsigned __int16 uint16_t;
-typedef unsigned __int32 uint32_t;
-typedef __int32 int32_t;
-#endif
-
 // Inlining controls
 #if defined(_MSC_VER) && _MSC_VER >= 1300
-#       define PUGIXML_NO_INLINE __declspec(noinline)
+#       define PUGI__NO_INLINE __declspec(noinline)
 #elif defined(__GNUC__)
-#       define PUGIXML_NO_INLINE __attribute__((noinline))
+#       define PUGI__NO_INLINE __attribute__((noinline))
 #else
-#       define PUGIXML_NO_INLINE 
+#       define PUGI__NO_INLINE
 #endif
 
+// Branch weight controls
+#if defined(__GNUC__)
+#       define PUGI__UNLIKELY(cond) __builtin_expect(cond, 0)
+#else
+#       define PUGI__UNLIKELY(cond) (cond)
+#endif
+
 // Simple static assertion
-#define STATIC_ASSERT(cond) { static const char condition_failed[(cond) ? 1 : -1] = {0}; (void)condition_failed[0]; }
+#define PUGI__STATIC_ASSERT(cond) { static const char condition_failed[(cond) ? 1 : -1] = {0}; (void)condition_failed[0]; }
 
 // Digital Mars C++ bug workaround for passing char loaded from memory via stack
 #ifdef __DMC__
-#       define DMC_VOLATILE volatile
+#       define PUGI__DMC_VOLATILE volatile
 #else
-#       define DMC_VOLATILE
+#       define PUGI__DMC_VOLATILE
 #endif
 
-using namespace pugi;
+// Borland C++ bug workaround for not defining ::memcpy depending on header include order (can't always use std::memcpy because some compilers don't have it at all)
+#if defined(__BORLANDC__) && !defined(__MEM_H_USING_LIST)
+using std::memcpy;
+using std::memmove;
+using std::memset;
+#endif
+
+// Some MinGW versions have headers that erroneously omit LLONG_MIN/LLONG_MAX/ULLONG_MAX definitions in strict ANSI mode
+#if defined(PUGIXML_HAS_LONG_LONG) && defined(__MINGW32__) && defined(__STRICT_ANSI__) && !defined(LLONG_MAX) && !defined(LLONG_MIN) && !defined(ULLONG_MAX)
+#       define LLONG_MAX 9223372036854775807LL
+#       define LLONG_MIN (-LLONG_MAX-1)
+#       define ULLONG_MAX (2ULL*LLONG_MAX+1)
+#endif
+
+// In some environments MSVC is a compiler but the CRT lacks certain MSVC-specific features
+#if defined(_MSC_VER) && !defined(__S3E__)
+#       define PUGI__MSVC_CRT_VERSION _MSC_VER
+#endif
+
+#ifdef PUGIXML_HEADER_ONLY
+#       define PUGI__NS_BEGIN namespace pugi { namespace impl {
+#       define PUGI__NS_END } }
+#       define PUGI__FN inline
+#       define PUGI__FN_NO_INLINE inline
+#else
+#       if defined(_MSC_VER) && _MSC_VER < 1300 // MSVC6 seems to have an amusing bug with anonymous namespaces inside namespaces
+#               define PUGI__NS_BEGIN namespace pugi { namespace impl {
+#               define PUGI__NS_END } }
+#       else
+#               define PUGI__NS_BEGIN namespace pugi { namespace impl { namespace {
+#               define PUGI__NS_END } } }
+#       endif
+#       define PUGI__FN
+#       define PUGI__FN_NO_INLINE PUGI__NO_INLINE
+#endif
+
+// uintptr_t
+#if (defined(_MSC_VER) && _MSC_VER < 1600) || (defined(__BORLANDC__) && __BORLANDC__ < 0x561)
+namespace pugi
+{
+#       ifndef _UINTPTR_T_DEFINED
+        typedef size_t uintptr_t;
+#       endif
+
+        typedef unsigned __int8 uint8_t;
+        typedef unsigned __int16 uint16_t;
+        typedef unsigned __int32 uint32_t;
+}
+#else
+#       include <stdint.h>
+#endif
 
 // Memory allocation
-namespace
-{
-        void* default_allocate(size_t size)
+PUGI__NS_BEGIN
+        PUGI__FN void* default_allocate(size_t size)
         {
                 return malloc(size);
         }
 
-        void default_deallocate(void* ptr)
+        PUGI__FN void default_deallocate(void* ptr)
         {
                 free(ptr);
         }
 
-        allocation_function global_allocate = default_allocate;
-        deallocation_function global_deallocate = default_deallocate;
-}
+        template <typename T>
+        struct xml_memory_management_function_storage
+        {
+                static allocation_function allocate;
+                static deallocation_function deallocate;
+        };
+
+        // Global allocation functions are stored in class statics so that in header mode linker deduplicates them
+        // Without a template<> we'll get multiple definitions of the same static
+        template <typename T> allocation_function xml_memory_management_function_storage<T>::allocate = default_allocate;
+        template <typename T> deallocation_function xml_memory_management_function_storage<T>::deallocate = default_deallocate;
+
+        typedef xml_memory_management_function_storage<int> xml_memory;
+PUGI__NS_END
 
 // String utilities
-namespace
-{
+PUGI__NS_BEGIN
         // Get string length
-        size_t strlength(const char_t* s)
+        PUGI__FN size_t strlength(const char_t* s)
         {
                 assert(s);
@@ -127 +197 @@
 
         // Compare two strings
-        bool strequal(const char_t* src, const char_t* dst)
+        PUGI__FN bool strequal(const char_t* src, const char_t* dst)
         {
                 assert(src && dst);
@@ -139 +209 @@
 
         // Compare lhs with [rhs_begin, rhs_end)
-        bool strequalrange(const char_t* lhs, const char_t* rhs, size_t count)
+        PUGI__FN bool strequalrange(const char_t* lhs, const char_t* rhs, size_t count)
         {
                 for (size_t i = 0; i < count; ++i)
                         if (lhs[i] != rhs[i])
                                 return false;
-        
+
                 return lhs[count] == 0;
         }
-        
-#ifdef PUGIXML_WCHAR_MODE
-        // Convert string to wide string, assuming all symbols are ASCII
-        void widen_ascii(wchar_t* dest, const char* source)
-        {
-                for (const char* i = source; *i; ++i) *dest++ = *i;
-                *dest = 0;
-        }
-#endif
-}
-
-#if !defined(PUGIXML_NO_STL) || !defined(PUGIXML_NO_XPATH)
-// auto_ptr-like buffer holder for exception recovery
-namespace
-{
-        struct buffer_holder
-        {
-                void* data;
-                void (*deleter)(void*);
-
-                buffer_holder(void* data, void (*deleter)(void*)): data(data), deleter(deleter)
-                {
-                }
-
-                ~buffer_holder()
+
+        // Get length of wide string, even if CRT lacks wide character support
+        PUGI__FN size_t strlength_wide(const wchar_t* s)
+        {
+                assert(s);
+
+        #ifdef PUGIXML_WCHAR_MODE
+                return wcslen(s);
+        #else
+                const wchar_t* end = s;
+                while (*end) end++;
+                return static_cast<size_t>(end - s);
+        #endif
+        }
+PUGI__NS_END
+
+// auto_ptr-like object for exception recovery
+PUGI__NS_BEGIN
+        template <typename T> struct auto_deleter
+        {
+                typedef void (*D)(T*);
+
+                T* data;
+                D deleter;
+
+                auto_deleter(T* data_, D deleter_): data(data_), deleter(deleter_)
+                {
+                }
+
+                ~auto_deleter()
                 {
                         if (data) deleter(data);
                 }
 
-                void* release()
-                {
-                        void* result = data;
+                T* release()
+                {
+                        T* result = data;
                         data = 0;
                         return result;
                 }
         };
-}
+PUGI__NS_END
+
+#ifdef PUGIXML_COMPACT
+PUGI__NS_BEGIN
+        class compact_hash_table
+        {
+        public:
+                compact_hash_table(): _items(0), _capacity(0), _count(0)
+                {
+                }
+
+                void clear()
+                {
+                        if (_items)
+                        {
+                                xml_memory::deallocate(_items);
+                                _items = 0;
+                                _capacity = 0;
+                                _count = 0;
+                        }
+                }
+
+                void** find(const void* key)
+                {
+                        assert(key);
+
+                        if (_capacity == 0) return 0;
+
+                        size_t hashmod = _capacity - 1;
+                        size_t bucket = hash(key) & hashmod;
+
+                        for (size_t probe = 0; probe <= hashmod; ++probe)
+                        {
+                                item_t& probe_item = _items[bucket];
+
+                                if (probe_item.key == key)
+                                        return &probe_item.value;
+
+                                if (probe_item.key == 0)
+                                        return 0;
+
+                                // hash collision, quadratic probing
+                                bucket = (bucket + probe + 1) & hashmod;
+                        }
+
+                        assert(false && "Hash table is full");
+                        return 0;
+                }
+
+                void** insert(const void* key)
+                {
+                        assert(key);
+                        assert(_capacity != 0 && _count < _capacity - _capacity / 4);
+
+                        size_t hashmod = _capacity - 1;
+                        size_t bucket = hash(key) & hashmod;
+
+                        for (size_t probe = 0; probe <= hashmod; ++probe)
+                        {
+                                item_t& probe_item = _items[bucket];
+
+                                if (probe_item.key == 0)
+                                {
+                                        probe_item.key = key;
+                                        _count++;
+                                        return &probe_item.value;
+                                }
+
+                                if (probe_item.key == key)
+                                        return &probe_item.value;
+
+                                // hash collision, quadratic probing
+                                bucket = (bucket + probe + 1) & hashmod;
+                        }
+
+                        assert(false && "Hash table is full");
+                        return 0;
+                }
+
+                bool reserve()
+                {
+                        if (_count + 16 >= _capacity - _capacity / 4)
+                                return rehash();
+
+                        return true;
+                }
+
+        private:
+                struct item_t
+                {
+                        const void* key;
+                        void* value;
+                };
+
+                item_t* _items;
+                size_t _capacity;
+
+                size_t _count;
+
+                bool rehash();
+
+                static unsigned int hash(const void* key)
+                {
+                        unsigned int h = static_cast<unsigned int>(reinterpret_cast<uintptr_t>(key));
+
+                        // MurmurHash3 32-bit finalizer
+                        h ^= h >> 16;
+                        h *= 0x85ebca6bu;
+                        h ^= h >> 13;
+                        h *= 0xc2b2ae35u;
+                        h ^= h >> 16;
+
+                        return h;
+                }
+        };
+
+        PUGI__FN_NO_INLINE bool compact_hash_table::rehash()
+        {
+                compact_hash_table rt;
+                rt._capacity = (_capacity == 0) ? 32 : _capacity * 2;
+                rt._items = static_cast<item_t*>(xml_memory::allocate(sizeof(item_t) * rt._capacity));
+
+                if (!rt._items)
+                        return false;
+
+                memset(rt._items, 0, sizeof(item_t) * rt._capacity);
+
+                for (size_t i = 0; i < _capacity; ++i)
+                        if (_items[i].key)
+                                *rt.insert(_items[i].key) = _items[i].value;
+
+                if (_items)
+                        xml_memory::deallocate(_items);
+
+                _capacity = rt._capacity;
+                _items = rt._items;
+
+                assert(_count == rt._count);
+
+                return true;
+        }
+
+PUGI__NS_END
 #endif
 
-namespace
-{
-        static const size_t xml_memory_page_size = 32768;
-
-        static const uintptr_t xml_memory_page_alignment = 32;
-        static const uintptr_t xml_memory_page_pointer_mask = ~(xml_memory_page_alignment - 1);
-        static const uintptr_t xml_memory_page_name_allocated_mask = 16;
-        static const uintptr_t xml_memory_page_value_allocated_mask = 8;
-        static const uintptr_t xml_memory_page_type_mask = 7;
+PUGI__NS_BEGIN
+#ifdef PUGIXML_COMPACT
+        static const uintptr_t xml_memory_block_alignment = 4;
+#else
+        static const uintptr_t xml_memory_block_alignment = sizeof(void*);
+#endif
+
+        // extra metadata bits
+        static const uintptr_t xml_memory_page_contents_shared_mask = 64;
+        static const uintptr_t xml_memory_page_name_allocated_mask = 32;
+        static const uintptr_t xml_memory_page_value_allocated_mask = 16;
+        static const uintptr_t xml_memory_page_type_mask = 15;
+
+        // combined masks for string uniqueness
+        static const uintptr_t xml_memory_page_name_allocated_or_shared_mask = xml_memory_page_name_allocated_mask | xml_memory_page_contents_shared_mask;
+        static const uintptr_t xml_memory_page_value_allocated_or_shared_mask = xml_memory_page_value_allocated_mask | xml_memory_page_contents_shared_mask;
+
+#ifdef PUGIXML_COMPACT
+        #define PUGI__GETHEADER_IMPL(object, page, flags) // unused
+        #define PUGI__GETPAGE_IMPL(header) (header).get_page()
+#else
+        #define PUGI__GETHEADER_IMPL(object, page, flags) (((reinterpret_cast<char*>(object) - reinterpret_cast<char*>(page)) << 8) | (flags))
+        // this macro casts pointers through void* to avoid 'cast increases required alignment of target type' warnings
+        #define PUGI__GETPAGE_IMPL(header) static_cast<impl::xml_memory_page*>(const_cast<void*>(static_cast<const void*>(reinterpret_cast<const char*>(&header) - (header >> 8))))
+#endif
+
+        #define PUGI__GETPAGE(n) PUGI__GETPAGE_IMPL((n)->header)
+        #define PUGI__NODETYPE(n) static_cast<xml_node_type>((n)->header & impl::xml_memory_page_type_mask)
 
         struct xml_allocator;
@@ -202 +438 @@
                 static xml_memory_page* construct(void* memory)
                 {
-                        if (!memory) return 0; //$ redundant, left for performance
-
                         xml_memory_page* result = static_cast<xml_memory_page*>(memory);
 
                         result->allocator = 0;
-                        result->memory = 0;
                         result->prev = 0;
                         result->next = 0;
@@ -213 +446 @@
                         result->freed_size = 0;
 
+                #ifdef PUGIXML_COMPACT
+                        result->compact_string_base = 0;
+                        result->compact_shared_parent = 0;
+                        result->compact_page_marker = 0;
+                #endif
+
                         return result;
                 }
 
                 xml_allocator* allocator;
-
-                void* memory;
 
                 xml_memory_page* prev;
@@ -226 +463 @@
                 size_t freed_size;
 
-                char data[1];
+        #ifdef PUGIXML_COMPACT
+                char_t* compact_string_base;
+                void* compact_shared_parent;
+                uint32_t* compact_page_marker;
+        #endif
         };
+
+        static const size_t xml_memory_page_size =
+        #ifdef PUGIXML_MEMORY_PAGE_SIZE
+                (PUGIXML_MEMORY_PAGE_SIZE)
+        #else
+                32768
+        #endif
+                - sizeof(xml_memory_page);
 
         struct xml_memory_string_header
@@ -239 +488 @@
                 xml_allocator(xml_memory_page* root): _root(root), _busy_size(root->busy_size)
                 {
+                #ifdef PUGIXML_COMPACT
+                        _hash = 0;
+                #endif
                 }
 
                 xml_memory_page* allocate_page(size_t data_size)
                 {
-                        size_t size = offsetof(xml_memory_page, data) + data_size;
+                        size_t size = sizeof(xml_memory_page) + data_size;
 
                         // allocate block with some alignment, leaving memory for worst-case padding
-                        void* memory = global_allocate(size + xml_memory_page_alignment);
+                        void* memory = xml_memory::allocate(size);
                         if (!memory) return 0;
 
-                        // align upwards to page boundary
-                        void* page_memory = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(memory) + (xml_memory_page_alignment - 1)) & ~(xml_memory_page_alignment - 1));
-
                         // prepare page structure
-                        xml_memory_page* page = xml_memory_page::construct(page_memory);
-
-                        page->memory = memory;
+                        xml_memory_page* page = xml_memory_page::construct(memory);
+                        assert(page);
+
                         page->allocator = _root->allocator;
 
@@ -263 +512 @@
                 static void deallocate_page(xml_memory_page* page)
                 {
-                        global_deallocate(page->memory);
+                        xml_memory::deallocate(page);
                 }
 
@@ -270 +519 @@
                 void* allocate_memory(size_t size, xml_memory_page*& out_page)
                 {
-                        if (_busy_size + size > xml_memory_page_size) return allocate_memory_oob(size, out_page);
-
-                        void* buf = _root->data + _busy_size;
+                        if (PUGI__UNLIKELY(_busy_size + size > xml_memory_page_size))
+                                return allocate_memory_oob(size, out_page);
+
+                        void* buf = reinterpret_cast<char*>(_root) + sizeof(xml_memory_page) + _busy_size;
 
                         _busy_size += size;
@@ -281 +531 @@
                 }
 
+        #ifdef PUGIXML_COMPACT
+                void* allocate_object(size_t size, xml_memory_page*& out_page)
+                {
+                        void* result = allocate_memory(size + sizeof(uint32_t), out_page);
+                        if (!result) return 0;
+
+                        // adjust for marker
+                        ptrdiff_t offset = static_cast<char*>(result) - reinterpret_cast<char*>(out_page->compact_page_marker);
+
+                        if (PUGI__UNLIKELY(static_cast<uintptr_t>(offset) >= 256 * xml_memory_block_alignment))
+                        {
+                                // insert new marker
+                                uint32_t* marker = static_cast<uint32_t*>(result);
+
+                                *marker = static_cast<uint32_t>(reinterpret_cast<char*>(marker) - reinterpret_cast<char*>(out_page));
+                                out_page->compact_page_marker = marker;
+
+                                // since we don't reuse the page space until we reallocate it, we can just pretend that we freed the marker block
+                                // this will make sure deallocate_memory correctly tracks the size
+                                out_page->freed_size += sizeof(uint32_t);
+
+                                return marker + 1;
+                        }
+                        else
+                        {
+                                // roll back uint32_t part
+                                _busy_size -= sizeof(uint32_t);
+
+                                return result;
+                        }
+                }
+        #else
+                void* allocate_object(size_t size, xml_memory_page*& out_page)
+                {
+                        return allocate_memory(size, out_page);
+                }
+        #endif
+
                 void deallocate_memory(void* ptr, size_t size, xml_memory_page* page)
                 {
                         if (page == _root) page->busy_size = _busy_size;
 
-                        assert(ptr >= page->data && ptr < page->data + page->busy_size);
+                        assert(ptr >= reinterpret_cast<char*>(page) + sizeof(xml_memory_page) && ptr < reinterpret_cast<char*>(page) + sizeof(xml_memory_page) + page->busy_size);
                         (void)!ptr;
 
@@ -298 +586 @@
 
                                         // top page freed, just reset sizes
-                                        page->busy_size = page->freed_size = 0;
+                                        page->busy_size = 0;
+                                        page->freed_size = 0;
+
+                                #ifdef PUGIXML_COMPACT
+                                        // reset compact state to maximize efficiency
+                                        page->compact_string_base = 0;
+                                        page->compact_shared_parent = 0;
+                                        page->compact_page_marker = 0;
+                                #endif
+
                                         _busy_size = 0;
                                 }
@@ -318 +615 @@
                 char_t* allocate_string(size_t length)
                 {
+                        static const size_t max_encoded_offset = (1 << 16) * xml_memory_block_alignment;
+
+                        PUGI__STATIC_ASSERT(xml_memory_page_size <= max_encoded_offset);
+
                         // allocate memory for string and header block
                         size_t size = sizeof(xml_memory_string_header) + length * sizeof(char_t);
-                        
-                        // round size up to pointer alignment boundary
-                        size_t full_size = (size + (sizeof(void*) - 1)) & ~(sizeof(void*) - 1);
+
+                        // round size up to block alignment boundary
+                        size_t full_size = (size + (xml_memory_block_alignment - 1)) & ~(xml_memory_block_alignment - 1);
 
                         xml_memory_page* page;
@@ -330 +631 @@
 
                         // setup header
-                        ptrdiff_t page_offset = reinterpret_cast<char*>(header) - page->data;
-
-                        assert(page_offset >= 0 && page_offset < (1 << 16));
-                        header->page_offset = static_cast<uint16_t>(page_offset);
+                        ptrdiff_t page_offset = reinterpret_cast<char*>(header) - reinterpret_cast<char*>(page) - sizeof(xml_memory_page);
+
+                        assert(page_offset % xml_memory_block_alignment == 0);
+                        assert(page_offset >= 0 && static_cast<size_t>(page_offset) < max_encoded_offset);
+                        header->page_offset = static_cast<uint16_t>(static_cast<size_t>(page_offset) / xml_memory_block_alignment);
 
                         // full_size == 0 for large strings that occupy the whole page
-                        assert(full_size < (1 << 16) || (page->busy_size == full_size && page_offset == 0));
-                        header->full_size = static_cast<uint16_t>(full_size < (1 << 16) ? full_size : 0);
-
-                        return reinterpret_cast<char_t*>(header + 1);
+                        assert(full_size % xml_memory_block_alignment == 0);
+                        assert(full_size < max_encoded_offset || (page->busy_size == full_size && page_offset == 0));
+                        header->full_size = static_cast<uint16_t>(full_size < max_encoded_offset ? full_size / xml_memory_block_alignment : 0);
+
+                        // round-trip through void* to avoid 'cast increases required alignment of target type' warning
+                        // header is guaranteed a pointer-sized alignment, which should be enough for char_t
+                        return static_cast<char_t*>(static_cast<void*>(header + 1));
                 }
 
                 void deallocate_string(char_t* string)
                 {
+                        // this function casts pointers through void* to avoid 'cast increases required alignment of target type' warnings
+                        // we're guaranteed the proper (pointer-sized) alignment on the input string if it was allocated via allocate_string
+
                         // get header
-                        xml_memory_string_header* header = reinterpret_cast<xml_memory_string_header*>(string) - 1;
+                        xml_memory_string_header* header = static_cast<xml_memory_string_header*>(static_cast<void*>(string)) - 1;
+                        assert(header);
 
                         // deallocate
-                        size_t page_offset = offsetof(xml_memory_page, data) + header->page_offset;
-                        xml_memory_page* page = reinterpret_cast<xml_memory_page*>(reinterpret_cast<char*>(header) - page_offset);
+                        size_t page_offset = sizeof(xml_memory_page) + header->page_offset * xml_memory_block_alignment;
+                        xml_memory_page* page = reinterpret_cast<xml_memory_page*>(static_cast<void*>(reinterpret_cast<char*>(header) - page_offset));
 
                         // if full_size == 0 then this string occupies the whole page
-                        size_t full_size = header->full_size == 0 ? page->busy_size : header->full_size;
+                        size_t full_size = header->full_size == 0 ? page->busy_size : header->full_size * xml_memory_block_alignment;
 
                         deallocate_memory(header, full_size, page);
+                }
+
+                bool reserve()
+                {
+                #ifdef PUGIXML_COMPACT
+                        return _hash->reserve();
+                #else
+                        return true;
+                #endif
                 }
 
                 xml_memory_page* _root;
                 size_t _busy_size;
+
+        #ifdef PUGIXML_COMPACT
+                compact_hash_table* _hash;
+        #endif
         };
 
-        PUGIXML_NO_INLINE void* xml_allocator::allocate_memory_oob(size_t size, xml_memory_page*& out_page)
+        PUGI__FN_NO_INLINE void* xml_allocator::allocate_memory_oob(size_t size, xml_memory_page*& out_page)
         {
                 const size_t large_allocation_threshold = xml_memory_page_size / 4;
 
                 xml_memory_page* page = allocate_page(size <= large_allocation_threshold ? xml_memory_page_size : size);
+                out_page = page;
+
                 if (!page) return 0;
 
@@ -390 +714 @@
                         _root->prev->next = page;
                         _root->prev = page;
-                }
-
-                // allocate inside page
-                page->busy_size = size;
-
-                out_page = page;
-                return page->data;
-        }
-}
-
+
+                        page->busy_size = size;
+                }
+
+                return reinterpret_cast<char*>(page) + sizeof(xml_memory_page);
+        }
+PUGI__NS_END
+
+#ifdef PUGIXML_COMPACT
+PUGI__NS_BEGIN
+        static const uintptr_t compact_alignment_log2 = 2;
+        static const uintptr_t compact_alignment = 1 << compact_alignment_log2;
+
+        class compact_header
+        {
+        public:
+                compact_header(xml_memory_page* page, unsigned int flags)
+                {
+                        PUGI__STATIC_ASSERT(xml_memory_block_alignment == compact_alignment);
+
+                        ptrdiff_t offset = (reinterpret_cast<char*>(this) - reinterpret_cast<char*>(page->compact_page_marker));
+                        assert(offset % compact_alignment == 0 && static_cast<uintptr_t>(offset) < 256 * compact_alignment);
+
+                        _page = static_cast<unsigned char>(offset >> compact_alignment_log2);
+                        _flags = static_cast<unsigned char>(flags);
+                }
+
+                void operator&=(uintptr_t mod)
+                {
+                        _flags &= static_cast<unsigned char>(mod);
+                }
+
+                void operator|=(uintptr_t mod)
+                {
+                        _flags |= static_cast<unsigned char>(mod);
+                }
+
+                uintptr_t operator&(uintptr_t mod) const
+                {
+                        return _flags & mod;
+                }
+
+                xml_memory_page* get_page() const
+                {
+                        // round-trip through void* to silence 'cast increases required alignment of target type' warnings
+                        const char* page_marker = reinterpret_cast<const char*>(this) - (_page << compact_alignment_log2);
+                        const char* page = page_marker - *reinterpret_cast<const uint32_t*>(static_cast<const void*>(page_marker));
+
+                        return const_cast<xml_memory_page*>(reinterpret_cast<const xml_memory_page*>(static_cast<const void*>(page)));
+                }
+
+        private:
+                unsigned char _page;
+                unsigned char _flags;
+        };
+
+        PUGI__FN xml_memory_page* compact_get_page(const void* object, int header_offset)
+        {
+                const compact_header* header = reinterpret_cast<const compact_header*>(static_cast<const char*>(object) - header_offset);
+
+                return header->get_page();
+        }
+
+        template <int header_offset, typename T> PUGI__FN_NO_INLINE T* compact_get_value(const void* object)
+        {
+                return static_cast<T*>(*compact_get_page(object, header_offset)->allocator->_hash->find(object));
+        }
+
+        template <int header_offset, typename T> PUGI__FN_NO_INLINE void compact_set_value(const void* object, T* value)
+        {
+                *compact_get_page(object, header_offset)->allocator->_hash->insert(object) = value;
+        }
+
+        template <typename T, int header_offset, int start = -126> class compact_pointer
+        {
+        public:
+                compact_pointer(): _data(0)
+                {
+                }
+
+                void operator=(const compact_pointer& rhs)
+                {
+                        *this = rhs + 0;
+                }
+
+                void operator=(T* value)
+                {
+                        if (value)
+                        {
+                                // value is guaranteed to be compact-aligned; 'this' is not
+                                // our decoding is based on 'this' aligned to compact alignment downwards (see operator T*)
+                                // so for negative offsets (e.g. -3) we need to adjust the diff by compact_alignment - 1 to
+                                // compensate for arithmetic shift rounding for negative values
+                                ptrdiff_t diff = reinterpret_cast<char*>(value) - reinterpret_cast<char*>(this);
+                                ptrdiff_t offset = ((diff + int(compact_alignment - 1)) >> compact_alignment_log2) - start;
+
+                                if (static_cast<uintptr_t>(offset) <= 253)
+                                        _data = static_cast<unsigned char>(offset + 1);
+                                else
+                                {
+                                        compact_set_value<header_offset>(this, value);
+
+                                        _data = 255;
+                                }
+                        }
+                        else
+                                _data = 0;
+                }
+
+                operator T*() const
+                {
+                        if (_data)
+                        {
+                                if (_data < 255)
+                                {
+                                        uintptr_t base = reinterpret_cast<uintptr_t>(this) & ~(compact_alignment - 1);
+
+                                        return reinterpret_cast<T*>(base + ((_data - 1 + start) << compact_alignment_log2));
+                                }
+                                else
+                                        return compact_get_value<header_offset, T>(this);
+                        }
+                        else
+                                return 0;
+                }
+
+                T* operator->() const
+                {
+                        return *this;
+                }
+
+        private:
+                unsigned char _data;
+        };
+
+        template <typename T, int header_offset> class compact_pointer_parent
+        {
+        public:
+                compact_pointer_parent(): _data(0)
+                {
+                }
+
+                void operator=(const compact_pointer_parent& rhs)
+                {
+                        *this = rhs + 0;
+                }
+
+                void operator=(T* value)
+                {
+                        if (value)
+                        {
+                                // value is guaranteed to be compact-aligned; 'this' is not
+                                // our decoding is based on 'this' aligned to compact alignment downwards (see operator T*)
+                                // so for negative offsets (e.g. -3) we need to adjust the diff by compact_alignment - 1 to
+                                // compensate for arithmetic shift behavior for negative values
+                                ptrdiff_t diff = reinterpret_cast<char*>(value) - reinterpret_cast<char*>(this);
+                                ptrdiff_t offset = ((diff + int(compact_alignment - 1)) >> compact_alignment_log2) + 65533;
+
+                                if (static_cast<uintptr_t>(offset) <= 65533)
+                                {
+                                        _data = static_cast<unsigned short>(offset + 1);
+                                }
+                                else
+                                {
+                                        xml_memory_page* page = compact_get_page(this, header_offset);
+
+                                        if (PUGI__UNLIKELY(page->compact_shared_parent == 0))
+                                                page->compact_shared_parent = value;
+
+                                        if (page->compact_shared_parent == value)
+                                        {
+                                                _data = 65534;
+                                        }
+                                        else
+                                        {
+                                                compact_set_value<header_offset>(this, value);
+
+                                                _data = 65535;
+                                        }
+                                }
+                        }
+                        else
+                        {
+                                _data = 0;
+                        }
+                }
+
+                operator T*() const
+                {
+                        if (_data)
+                        {
+                                if (_data < 65534)
+                                {
+                                        uintptr_t base = reinterpret_cast<uintptr_t>(this) & ~(compact_alignment - 1);
+
+                                        return reinterpret_cast<T*>(base + ((_data - 1 - 65533) << compact_alignment_log2));
+                                }
+                                else if (_data == 65534)
+                                        return static_cast<T*>(compact_get_page(this, header_offset)->compact_shared_parent);
+                                else
+                                        return compact_get_value<header_offset, T>(this);
+                        }
+                        else
+                                return 0;
+                }
+
+                T* operator->() const
+                {
+                        return *this;
+                }
+
+        private:
+                uint16_t _data;
+        };
+
+        template <int header_offset, int base_offset> class compact_string
+        {
+        public:
+                compact_string(): _data(0)
+                {
+                }
+
+                void operator=(const compact_string& rhs)
+                {
+                        *this = rhs + 0;
+                }
+
+                void operator=(char_t* value)
+                {
+                        if (value)
+                        {
+                                xml_memory_page* page = compact_get_page(this, header_offset);
+
+                                if (PUGI__UNLIKELY(page->compact_string_base == 0))
+                                        page->compact_string_base = value;
+
+                                ptrdiff_t offset = value - page->compact_string_base;
+
+                                if (static_cast<uintptr_t>(offset) < (65535 << 7))
+                                {
+                                        // round-trip through void* to silence 'cast increases required alignment of target type' warnings
+                                        uint16_t* base = reinterpret_cast<uint16_t*>(static_cast<void*>(reinterpret_cast<char*>(this) - base_offset));
+
+                                        if (*base == 0)
+                                        {
+                                                *base = static_cast<uint16_t>((offset >> 7) + 1);
+                                                _data = static_cast<unsigned char>((offset & 127) + 1);
+                                        }
+                                        else
+                                        {
+                                                ptrdiff_t remainder = offset - ((*base - 1) << 7);
+
+                                                if (static_cast<uintptr_t>(remainder) <= 253)
+                                                {
+                                                        _data = static_cast<unsigned char>(remainder + 1);
+                                                }
+                                                else
+                                                {
+                                                        compact_set_value<header_offset>(this, value);
+
+                                                        _data = 255;
+                                                }
+                                        }
+                                }
+                                else
+                                {
+                                        compact_set_value<header_offset>(this, value);
+
+                                        _data = 255;
+                                }
+                        }
+                        else
+                        {
+                                _data = 0;
+                        }
+                }
+
+                operator char_t*() const
+                {
+                        if (_data)
+                        {
+                                if (_data < 255)
+                                {
+                                        xml_memory_page* page = compact_get_page(this, header_offset);
+
+                                        // round-trip through void* to silence 'cast increases required alignment of target type' warnings
+                                        const uint16_t* base = reinterpret_cast<const uint16_t*>(static_cast<const void*>(reinterpret_cast<const char*>(this) - base_offset));
+                                        assert(*base);
+
+                                        ptrdiff_t offset = ((*base - 1) << 7) + (_data - 1);
+
+                                        return page->compact_string_base + offset;
+                                }
+                                else
+                                {
+                                        return compact_get_value<header_offset, char_t>(this);
+                                }
+                        }
+                        else
+                                return 0;
+                }
+
+        private:
+                unsigned char _data;
+        };
+PUGI__NS_END
+#endif
+
+#ifdef PUGIXML_COMPACT
 namespace pugi
 {
-        /// A 'name=value' XML attribute structure.
         struct xml_attribute_struct
         {
-                /// Default ctor
-                xml_attribute_struct(xml_memory_page* page): header(reinterpret_cast<uintptr_t>(page)), name(0), value(0), prev_attribute_c(0), next_attribute(0)
-                {
-                }
-
-                uintptr_t header;
-
-                char_t* name;   ///< Pointer to attribute name.
-                char_t* value;  ///< Pointer to attribute value.
-
-                xml_attribute_struct* prev_attribute_c; ///< Previous attribute (cyclic list)
-                xml_attribute_struct* next_attribute;   ///< Next attribute
+                xml_attribute_struct(impl::xml_memory_page* page): header(page, 0), namevalue_base(0)
+                {
+                        PUGI__STATIC_ASSERT(sizeof(xml_attribute_struct) == 8);
+                }
+
+                impl::compact_header header;
+
+                uint16_t namevalue_base;
+
+                impl::compact_string<4, 2> name;
+                impl::compact_string<5, 3> value;
+
+                impl::compact_pointer<xml_attribute_struct, 6> prev_attribute_c;
+                impl::compact_pointer<xml_attribute_struct, 7, 0> next_attribute;
         };
 
-        /// An XML document tree node.
         struct xml_node_struct
         {
-                /// Default ctor
-                /// \param type - node type
-                xml_node_struct(xml_memory_page* page, xml_node_type type): header(reinterpret_cast<uintptr_t>(page) | (type - 1)), parent(0), name(0), value(0), first_child(0), prev_sibling_c(0), next_sibling(0), first_attribute(0)
-                {
-                }
-
-                uintptr_t header;
-
-                xml_node_struct*                parent;                                 ///< Pointer to parent
-
-                char_t*                                 name;                                   ///< Pointer to element name.
-                char_t*                                 value;                                  ///< Pointer to any associated string data.
-
-                xml_node_struct*                first_child;                    ///< First child
-                
-                xml_node_struct*                prev_sibling_c;                 ///< Left brother (cyclic list)
-                xml_node_struct*                next_sibling;                   ///< Right brother
-                
-                xml_attribute_struct*   first_attribute;                ///< First attribute
+                xml_node_struct(impl::xml_memory_page* page, xml_node_type type): header(page, type), namevalue_base(0)
+                {
+                        PUGI__STATIC_ASSERT(sizeof(xml_node_struct) == 12);
+                }
+
+                impl::compact_header header;
+
+                uint16_t namevalue_base;
+
+                impl::compact_string<4, 2> name;
+                impl::compact_string<5, 3> value;
+
+                impl::compact_pointer_parent<xml_node_struct, 6> parent;
+
+                impl::compact_pointer<xml_node_struct, 8, 0> first_child;
+
+                impl::compact_pointer<xml_node_struct,  9>    prev_sibling_c;
+                impl::compact_pointer<xml_node_struct, 10, 0> next_sibling;
+
+                impl::compact_pointer<xml_attribute_struct, 11, 0> first_attribute;
         };
 }
-
-namespace
+#else
+namespace pugi
 {
+        struct xml_attribute_struct
+        {
+                xml_attribute_struct(impl::xml_memory_page* page): name(0), value(0), prev_attribute_c(0), next_attribute(0)
+                {
+                        header = PUGI__GETHEADER_IMPL(this, page, 0);
+                }
+
+                uintptr_t header;
+
+                char_t* name;
+                char_t* value;
+
+                xml_attribute_struct* prev_attribute_c;
+                xml_attribute_struct* next_attribute;
+        };
+
+        struct xml_node_struct
+        {
+                xml_node_struct(impl::xml_memory_page* page, xml_node_type type): name(0), value(0), parent(0), first_child(0), prev_sibling_c(0), next_sibling(0), first_attribute(0)
+                {
+                        header = PUGI__GETHEADER_IMPL(this, page, type);
+                }
+
+                uintptr_t header;
+
+                char_t* name;
+                char_t* value;
+
+                xml_node_struct* parent;
+
+                xml_node_struct* first_child;
+
+                xml_node_struct* prev_sibling_c;
+                xml_node_struct* next_sibling;
+
+                xml_attribute_struct* first_attribute;
+        };
+}
+#endif
+
+PUGI__NS_BEGIN
+        struct xml_extra_buffer
+        {
+                char_t* buffer;
+                xml_extra_buffer* next;
+        };
+
         struct xml_document_struct: public xml_node_struct, public xml_allocator
         {
-                xml_document_struct(xml_memory_page* page): xml_node_struct(page, node_document), xml_allocator(page), buffer(0)
+                xml_document_struct(xml_memory_page* page): xml_node_struct(page, node_document), xml_allocator(page), buffer(0), extra_buffers(0)
                 {
                 }
 
                 const char_t* buffer;
+
+                xml_extra_buffer* extra_buffers;
+
+        #ifdef PUGIXML_COMPACT
+                compact_hash_table hash;
+        #endif
         };
 
-        static inline xml_allocator& get_allocator(const xml_node_struct* node)
-        {
-                assert(node);
-
-                return *reinterpret_cast<xml_memory_page*>(node->header & xml_memory_page_pointer_mask)->allocator;
-        }
-}
+        template <typename Object> inline xml_allocator& get_allocator(const Object* object)
+        {
+                assert(object);
+
+                return *PUGI__GETPAGE(object)->allocator;
+        }
+
+        template <typename Object> inline xml_document_struct& get_document(const Object* object)
+        {
+                assert(object);
+
+                return *static_cast<xml_document_struct*>(PUGI__GETPAGE(object)->allocator);
+        }
+PUGI__NS_END
 
 // Low-level DOM operations
-namespace
-{
+PUGI__NS_BEGIN
         inline xml_attribute_struct* allocate_attribute(xml_allocator& alloc)
         {
                 xml_memory_page* page;
-                void* memory = alloc.allocate_memory(sizeof(xml_attribute_struct), page);
+                void* memory = alloc.allocate_object(sizeof(xml_attribute_struct), page);
+                if (!memory) return 0;
 
                 return new (memory) xml_attribute_struct(page);
@@ -477 +1161 @@
         {
                 xml_memory_page* page;
-                void* memory = alloc.allocate_memory(sizeof(xml_node_struct), page);
+                void* memory = alloc.allocate_object(sizeof(xml_node_struct), page);
+                if (!memory) return 0;
 
                 return new (memory) xml_node_struct(page, type);
@@ -484 +1169 @@
         inline void destroy_attribute(xml_attribute_struct* a, xml_allocator& alloc)
         {
-                uintptr_t header = a->header;
-
-                if (header & xml_memory_page_name_allocated_mask) alloc.deallocate_string(a->name);
-                if (header & xml_memory_page_value_allocated_mask) alloc.deallocate_string(a->value);
-
-                alloc.deallocate_memory(a, sizeof(xml_attribute_struct), reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask));
+                if (a->header & impl::xml_memory_page_name_allocated_mask)
+                        alloc.deallocate_string(a->name);
+
+                if (a->header & impl::xml_memory_page_value_allocated_mask)
+                        alloc.deallocate_string(a->value);
+
+                alloc.deallocate_memory(a, sizeof(xml_attribute_struct), PUGI__GETPAGE(a));
         }
 
         inline void destroy_node(xml_node_struct* n, xml_allocator& alloc)
         {
-                uintptr_t header = n->header;
-
-                if (header & xml_memory_page_name_allocated_mask) alloc.deallocate_string(n->name);
-                if (header & xml_memory_page_value_allocated_mask) alloc.deallocate_string(n->value);
+                if (n->header & impl::xml_memory_page_name_allocated_mask)
+                        alloc.deallocate_string(n->name);
+
+                if (n->header & impl::xml_memory_page_value_allocated_mask)
+                        alloc.deallocate_string(n->value);
 
                 for (xml_attribute_struct* attr = n->first_attribute; attr; )
@@ -517 +1204 @@
                 }
 
-                alloc.deallocate_memory(n, sizeof(xml_node_struct), reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask));
-        }
-
-        PUGIXML_NO_INLINE xml_node_struct* append_node(xml_node_struct* node, xml_allocator& alloc, xml_node_type type = node_element)
-        {
+                alloc.deallocate_memory(n, sizeof(xml_node_struct), PUGI__GETPAGE(n));
+        }
+
+        inline void append_node(xml_node_struct* child, xml_node_struct* node)
+        {
+                child->parent = node;
+
+                xml_node_struct* head = node->first_child;
+
+                if (head)
+                {
+                        xml_node_struct* tail = head->prev_sibling_c;
+
+                        tail->next_sibling = child;
+                        child->prev_sibling_c = tail;
+                        head->prev_sibling_c = child;
+                }
+                else
+                {
+                        node->first_child = child;
+                        child->prev_sibling_c = child;
+                }
+        }
+
+        inline void prepend_node(xml_node_struct* child, xml_node_struct* node)
+        {
+                child->parent = node;
+
+                xml_node_struct* head = node->first_child;
+
+                if (head)
+                {
+                        child->prev_sibling_c = head->prev_sibling_c;
+                        head->prev_sibling_c = child;
+                }
+                else
+                        child->prev_sibling_c = child;
+
+                child->next_sibling = head;
+                node->first_child = child;
+        }
+
+        inline void insert_node_after(xml_node_struct* child, xml_node_struct* node)
+        {
+                xml_node_struct* parent = node->parent;
+
+                child->parent = parent;
+
+                if (node->next_sibling)
+                        node->next_sibling->prev_sibling_c = child;
+                else
+                        parent->first_child->prev_sibling_c = child;
+
+                child->next_sibling = node->next_sibling;
+                child->prev_sibling_c = node;
+
+                node->next_sibling = child;
+        }
+
+        inline void insert_node_before(xml_node_struct* child, xml_node_struct* node)
+        {
+                xml_node_struct* parent = node->parent;
+
+                child->parent = parent;
+
+                if (node->prev_sibling_c->next_sibling)
+                        node->prev_sibling_c->next_sibling = child;
+                else
+                        parent->first_child = child;
+
+                child->prev_sibling_c = node->prev_sibling_c;
+                child->next_sibling = node;
+
+                node->prev_sibling_c = child;
+        }
+
+        inline void remove_node(xml_node_struct* node)
+        {
+                xml_node_struct* parent = node->parent;
+
+                if (node->next_sibling)
+                        node->next_sibling->prev_sibling_c = node->prev_sibling_c;
+                else
+                        parent->first_child->prev_sibling_c = node->prev_sibling_c;
+
+                if (node->prev_sibling_c->next_sibling)
+                        node->prev_sibling_c->next_sibling = node->next_sibling;
+                else
+                        parent->first_child = node->next_sibling;
+
+                node->parent = 0;
+                node->prev_sibling_c = 0;
+                node->next_sibling = 0;
+        }
+
+        inline void append_attribute(xml_attribute_struct* attr, xml_node_struct* node)
+        {
+                xml_attribute_struct* head = node->first_attribute;
+
+                if (head)
+                {
+                        xml_attribute_struct* tail = head->prev_attribute_c;
+
+                        tail->next_attribute = attr;
+                        attr->prev_attribute_c = tail;
+                        head->prev_attribute_c = attr;
+                }
+                else
+                {
+                        node->first_attribute = attr;
+                        attr->prev_attribute_c = attr;
+                }
+        }
+
+        inline void prepend_attribute(xml_attribute_struct* attr, xml_node_struct* node)
+        {
+                xml_attribute_struct* head = node->first_attribute;
+
+                if (head)
+                {
+                        attr->prev_attribute_c = head->prev_attribute_c;
+                        head->prev_attribute_c = attr;
+                }
+                else
+                        attr->prev_attribute_c = attr;
+
+                attr->next_attribute = head;
+                node->first_attribute = attr;
+        }
+
+        inline void insert_attribute_after(xml_attribute_struct* attr, xml_attribute_struct* place, xml_node_struct* node)
+        {
+                if (place->next_attribute)
+                        place->next_attribute->prev_attribute_c = attr;
+                else
+                        node->first_attribute->prev_attribute_c = attr;
+
+                attr->next_attribute = place->next_attribute;
+                attr->prev_attribute_c = place;
+                place->next_attribute = attr;
+        }
+
+        inline void insert_attribute_before(xml_attribute_struct* attr, xml_attribute_struct* place, xml_node_struct* node)
+        {
+                if (place->prev_attribute_c->next_attribute)
+                        place->prev_attribute_c->next_attribute = attr;
+                else
+                        node->first_attribute = attr;
+
+                attr->prev_attribute_c = place->prev_attribute_c;
+                attr->next_attribute = place;
+                place->prev_attribute_c = attr;
+        }
+
+        inline void remove_attribute(xml_attribute_struct* attr, xml_node_struct* node)
+        {
+                if (attr->next_attribute)
+                        attr->next_attribute->prev_attribute_c = attr->prev_attribute_c;
+                else
+                        node->first_attribute->prev_attribute_c = attr->prev_attribute_c;
+
+                if (attr->prev_attribute_c->next_attribute)
+                        attr->prev_attribute_c->next_attribute = attr->next_attribute;
+                else
+                        node->first_attribute = attr->next_attribute;
+
+                attr->prev_attribute_c = 0;
+                attr->next_attribute = 0;
+        }
+
+        PUGI__FN_NO_INLINE xml_node_struct* append_new_node(xml_node_struct* node, xml_allocator& alloc, xml_node_type type = node_element)
+        {
+                if (!alloc.reserve()) return 0;
+
                 xml_node_struct* child = allocate_node(alloc, type);
                 if (!child) return 0;
 
-                child->parent = node;
-
-                xml_node_struct* first_child = node->first_child;
-                        
-                if (first_child)
-                {
-                        xml_node_struct* last_child = first_child->prev_sibling_c;
-
-                        last_child->next_sibling = child;
-                        child->prev_sibling_c = last_child;
-                        first_child->prev_sibling_c = child;
-                }
-                else
-                {
-                        node->first_child = child;
-                        child->prev_sibling_c = child;
-                }
-                        
+                append_node(child, node);
+
                 return child;
         }
 
-        PUGIXML_NO_INLINE xml_attribute_struct* append_attribute_ll(xml_node_struct* node, xml_allocator& alloc)
-        {
-                xml_attribute_struct* a = allocate_attribute(alloc);
-                if (!a) return 0;
-
-                xml_attribute_struct* first_attribute = node->first_attribute;
-
-                if (first_attribute)
-                {
-                        xml_attribute_struct* last_attribute = first_attribute->prev_attribute_c;
-
-                        last_attribute->next_attribute = a;
-                        a->prev_attribute_c = last_attribute;
-                        first_attribute->prev_attribute_c = a;
-                }
-                else
-                {
-                        node->first_attribute = a;
-                        a->prev_attribute_c = a;
-                }
-                        
-                return a;
-        }
-}
+        PUGI__FN_NO_INLINE xml_attribute_struct* append_new_attribute(xml_node_struct* node, xml_allocator& alloc)
+        {
+                if (!alloc.reserve()) return 0;
+
+                xml_attribute_struct* attr = allocate_attribute(alloc);
+                if (!attr) return 0;
+
+                append_attribute(attr, node);
+
+                return attr;
+        }
+PUGI__NS_END
 
 // Helper classes for code generation
-namespace
-{
+PUGI__NS_BEGIN
         struct opt_false
         {
@@ -583 +1410 @@
                 enum { value = 1 };
         };
-}
+PUGI__NS_END
 
 // Unicode utilities
-namespace
-{
+PUGI__NS_BEGIN
         inline uint16_t endian_swap(uint16_t value)
         {
@@ -692 +1518 @@
                 static value_type high(value_type result, uint32_t ch)
                 {
-                        uint32_t msh = (uint32_t)(ch - 0x10000) >> 10;
-                        uint32_t lsh = (uint32_t)(ch - 0x10000) & 0x3ff;
+                        uint32_t msh = static_cast<uint32_t>(ch - 0x10000) >> 10;
+                        uint32_t lsh = static_cast<uint32_t>(ch - 0x10000) & 0x3ff;
 
                         result[0] = static_cast<uint16_t>(0xD800 + msh);
@@ -748 +1574 @@
         };
 
-        template <size_t size> struct wchar_selector;
-
-        template <> struct wchar_selector<2>
-        {
-                typedef uint16_t type;
-                typedef utf16_counter counter;
-                typedef utf16_writer writer;
+        struct latin1_writer
+        {
+                typedef uint8_t* value_type;
+
+                static value_type low(value_type result, uint32_t ch)
+                {
+                        *result = static_cast<uint8_t>(ch > 255 ? '?' : ch);
+
+                        return result + 1;
+                }
+
+                static value_type high(value_type result, uint32_t ch)
+                {
+                        (void)ch;
+
+                        *result = '?';
+
+                        return result + 1;
+                }
         };
 
-        template <> struct wchar_selector<4>
-        {
-                typedef uint32_t type;
-                typedef utf32_counter counter;
-                typedef utf32_writer writer;
-        };
-
-        typedef wchar_selector<sizeof(wchar_t)>::counter wchar_counter;
-        typedef wchar_selector<sizeof(wchar_t)>::writer wchar_writer;
-
-        template <typename Traits, typename opt_swap = opt_false> struct utf_decoder
-        {
-                static inline typename Traits::value_type decode_utf8_block(const uint8_t* data, size_t size, typename Traits::value_type result)
+        struct utf8_decoder
+        {
+                typedef uint8_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const uint8_t* data, size_t size, typename Traits::value_type result, Traits)
                 {
                         const uint8_t utf8_byte_mask = 0x3f;
@@ -787 +1617 @@
                                         if ((reinterpret_cast<uintptr_t>(data) & 3) == 0)
                                         {
-                                                while (size >= 4 && (*reinterpret_cast<const uint32_t*>(data) & 0x80808080) == 0)
+                                                // round-trip through void* to silence 'cast increases required alignment of target type' warnings
+                                                while (size >= 4 && (*static_cast<const uint32_t*>(static_cast<const void*>(data)) & 0x80808080) == 0)
                                                 {
                                                         result = Traits::low(result, data[0]);
@@ -799 +1630 @@
                                 }
                                 // 110xxxxx -> U+0080..U+07FF
-                                else if ((unsigned)(lead - 0xC0) < 0x20 && size >= 2 && (data[1] & 0xc0) == 0x80)
+                                else if (static_cast<unsigned int>(lead - 0xC0) < 0x20 && size >= 2 && (data[1] & 0xc0) == 0x80)
                                 {
                                         result = Traits::low(result, ((lead & ~0xC0) << 6) | (data[1] & utf8_byte_mask));
@@ -806 +1637 @@
                                 }
                                 // 1110xxxx -> U+0800-U+FFFF
-                                else if ((unsigned)(lead - 0xE0) < 0x10 && size >= 3 && (data[1] & 0xc0) == 0x80 && (data[2] & 0xc0) == 0x80)
+                                else if (static_cast<unsigned int>(lead - 0xE0) < 0x10 && size >= 3 && (data[1] & 0xc0) == 0x80 && (data[2] & 0xc0) == 0x80)
                                 {
                                         result = Traits::low(result, ((lead & ~0xE0) << 12) | ((data[1] & utf8_byte_mask) << 6) | (data[2] & utf8_byte_mask));
@@ -813 +1644 @@
                                 }
                                 // 11110xxx -> U+10000..U+10FFFF
-                                else if ((unsigned)(lead - 0xF0) < 0x08 && size >= 4 && (data[1] & 0xc0) == 0x80 && (data[2] & 0xc0) == 0x80 && (data[3] & 0xc0) == 0x80)
+                                else if (static_cast<unsigned int>(lead - 0xF0) < 0x08 && size >= 4 && (data[1] & 0xc0) == 0x80 && (data[2] & 0xc0) == 0x80 && (data[3] & 0xc0) == 0x80)
                                 {
                                         result = Traits::high(result, ((lead & ~0xF0) << 18) | ((data[1] & utf8_byte_mask) << 12) | ((data[2] & utf8_byte_mask) << 6) | (data[3] & utf8_byte_mask));
@@ -829 +1660 @@
                         return result;
                 }
-
-                static inline typename Traits::value_type decode_utf16_block(const uint16_t* data, size_t size, typename Traits::value_type result)
-                {
-                        const uint16_t* end = data + size;
-
-                        while (data < end)
+        };
+
+        template <typename opt_swap> struct utf16_decoder
+        {
+                typedef uint16_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const uint16_t* data, size_t size, typename Traits::value_type result, Traits)
+                {
+                        while (size)
                         {
                                 uint16_t lead = opt_swap::value ? endian_swap(*data) : *data;
@@ -843 +1677 @@
                                         result = Traits::low(result, lead);
                                         data += 1;
+                                        size -= 1;
                                 }
                                 // U+E000..U+FFFF
-                                else if ((unsigned)(lead - 0xE000) < 0x2000)
+                                else if (static_cast<unsigned int>(lead - 0xE000) < 0x2000)
                                 {
                                         result = Traits::low(result, lead);
                                         data += 1;
+                                        size -= 1;
                                 }
                                 // surrogate pair lead
-                                else if ((unsigned)(lead - 0xD800) < 0x400 && data + 1 < end)
+                                else if (static_cast<unsigned int>(lead - 0xD800) < 0x400 && size >= 2)
                                 {
                                         uint16_t next = opt_swap::value ? endian_swap(data[1]) : data[1];
 
-                                        if ((unsigned)(next - 0xDC00) < 0x400)
+                                        if (static_cast<unsigned int>(next - 0xDC00) < 0x400)
                                         {
                                                 result = Traits::high(result, 0x10000 + ((lead & 0x3ff) << 10) + (next & 0x3ff));
                                                 data += 2;
+                                                size -= 2;
                                         }
                                         else
                                         {
                                                 data += 1;
+                                                size -= 1;
                                         }
                                 }
@@ -868 +1706 @@
                                 {
                                         data += 1;
+                                        size -= 1;
                                 }
                         }
@@ -873 +1712 @@
                         return result;
                 }
-
-                static inline typename Traits::value_type decode_utf32_block(const uint32_t* data, size_t size, typename Traits::value_type result)
-                {
-                        const uint32_t* end = data + size;
-
-                        while (data < end)
+        };
+
+        template <typename opt_swap> struct utf32_decoder
+        {
+                typedef uint32_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const uint32_t* data, size_t size, typename Traits::value_type result, Traits)
+                {
+                        while (size)
                         {
                                 uint32_t lead = opt_swap::value ? endian_swap(*data) : *data;
@@ -887 +1729 @@
                                         result = Traits::low(result, lead);
                                         data += 1;
+                                        size -= 1;
                                 }
                                 // U+10000..U+10FFFF
@@ -893 +1736 @@
                                         result = Traits::high(result, lead);
                                         data += 1;
+                                        size -= 1;
                                 }
                         }
@@ -900 +1744 @@
         };
 
-        template <typename T> inline void convert_utf_endian_swap(T* result, const T* data, size_t length)
-        {
-                for (size_t i = 0; i < length; ++i) result[i] = endian_swap(data[i]);
-        }
-
-        inline void convert_wchar_endian_swap(wchar_t* result, const wchar_t* data, size_t length)
-        {
-                for (size_t i = 0; i < length; ++i) result[i] = static_cast<wchar_t>(endian_swap(static_cast<wchar_selector<sizeof(wchar_t)>::type>(data[i])));
-        }
-}
-
-namespace
-{       
+        struct latin1_decoder
+        {
+                typedef uint8_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const uint8_t* data, size_t size, typename Traits::value_type result, Traits)
+                {
+                        while (size)
+                        {
+                                result = Traits::low(result, *data);
+                                data += 1;
+                                size -= 1;
+                        }
+
+                        return result;
+                }
+        };
+
+        template <size_t size> struct wchar_selector;
+
+        template <> struct wchar_selector<2>
+        {
+                typedef uint16_t type;
+                typedef utf16_counter counter;
+                typedef utf16_writer writer;
+                typedef utf16_decoder<opt_false> decoder;
+        };
+
+        template <> struct wchar_selector<4>
+        {
+                typedef uint32_t type;
+                typedef utf32_counter counter;
+                typedef utf32_writer writer;
+                typedef utf32_decoder<opt_false> decoder;
+        };
+
+        typedef wchar_selector<sizeof(wchar_t)>::counter wchar_counter;
+        typedef wchar_selector<sizeof(wchar_t)>::writer wchar_writer;
+
+        struct wchar_decoder
+        {
+                typedef wchar_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const wchar_t* data, size_t size, typename Traits::value_type result, Traits traits)
+                {
+                        typedef wchar_selector<sizeof(wchar_t)>::decoder decoder;
+
+                        return decoder::process(reinterpret_cast<const typename decoder::type*>(data), size, result, traits);
+                }
+        };
+
+#ifdef PUGIXML_WCHAR_MODE
+        PUGI__FN void convert_wchar_endian_swap(wchar_t* result, const wchar_t* data, size_t length)
+        {
+                for (size_t i = 0; i < length; ++i)
+                        result[i] = static_cast<wchar_t>(endian_swap(static_cast<wchar_selector<sizeof(wchar_t)>::type>(data[i])));
+        }
+#endif
+PUGI__NS_END
+
+PUGI__NS_BEGIN
         enum chartype_t
         {
@@ -925 +1816 @@
         };
 
-        const unsigned char chartype_table[256] =
+        static const unsigned char chartype_table[256] =
         {
                 55,  0,   0,   0,   0,   0,   0,   0,      0,   12,  12,  0,   0,   63,  0,   0,   // 0-15
@@ -954 +1845 @@
                 ctx_symbol = 16                   // Any symbol > 127, a-z, A-Z, 0-9, _, -, .
         };
-        
-        const unsigned char chartypex_table[256] =
+
+        static const unsigned char chartypex_table[256] =
         {
                 3,  3,  3,  3,  3,  3,  3,  3,     3,  0,  2,  3,  3,  2,  3,  3,     // 0-15
@@ -976 +1867 @@
                 20, 20, 20, 20, 20, 20, 20, 20,    20, 20, 20, 20, 20, 20, 20, 20
         };
-        
+
 #ifdef PUGIXML_WCHAR_MODE
-        #define IS_CHARTYPE_IMPL(c, ct, table) ((static_cast<unsigned int>(c) < 128 ? table[static_cast<unsigned int>(c)] : table[128]) & (ct))
+        #define PUGI__IS_CHARTYPE_IMPL(c, ct, table) ((static_cast<unsigned int>(c) < 128 ? table[static_cast<unsigned int>(c)] : table[128]) & (ct))
 #else
-        #define IS_CHARTYPE_IMPL(c, ct, table) (table[static_cast<unsigned char>(c)] & (ct))
+        #define PUGI__IS_CHARTYPE_IMPL(c, ct, table) (table[static_cast<unsigned char>(c)] & (ct))
 #endif
 
-        #define IS_CHARTYPE(c, ct) IS_CHARTYPE_IMPL(c, ct, chartype_table)
-        #define IS_CHARTYPEX(c, ct) IS_CHARTYPE_IMPL(c, ct, chartypex_table)
-
-        bool is_little_endian()
+        #define PUGI__IS_CHARTYPE(c, ct) PUGI__IS_CHARTYPE_IMPL(c, ct, chartype_table)
+        #define PUGI__IS_CHARTYPEX(c, ct) PUGI__IS_CHARTYPE_IMPL(c, ct, chartypex_table)
+
+        PUGI__FN bool is_little_endian()
         {
                 unsigned int ui = 1;
@@ -993 +1884 @@
         }
 
-        xml_encoding get_wchar_encoding()
-        {
-                STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);
+        PUGI__FN xml_encoding get_wchar_encoding()
+        {
+                PUGI__STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);
 
                 if (sizeof(wchar_t) == 2)
                         return is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
-                else 
+                else
                         return is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
         }
 
-        xml_encoding guess_buffer_encoding(uint8_t d0, uint8_t d1, uint8_t d2, uint8_t d3)
-        {
+        PUGI__FN bool parse_declaration_encoding(const uint8_t* data, size_t size, const uint8_t*& out_encoding, size_t& out_length)
+        {
+        #define PUGI__SCANCHAR(ch) { if (offset >= size || data[offset] != ch) return false; offset++; }
+        #define PUGI__SCANCHARTYPE(ct) { while (offset < size && PUGI__IS_CHARTYPE(data[offset], ct)) offset++; }
+
+                // check if we have a non-empty XML declaration
+                if (size < 6 || !((data[0] == '<') & (data[1] == '?') & (data[2] == 'x') & (data[3] == 'm') & (data[4] == 'l') && PUGI__IS_CHARTYPE(data[5], ct_space)))
+                        return false;
+
+                // scan XML declaration until the encoding field
+                for (size_t i = 6; i + 1 < size; ++i)
+                {
+                        // declaration can not contain ? in quoted values
+                        if (data[i] == '?')
+                                return false;
+
+                        if (data[i] == 'e' && data[i + 1] == 'n')
+                        {
+                                size_t offset = i;
+
+                                // encoding follows the version field which can't contain 'en' so this has to be the encoding if XML is well formed
+                                PUGI__SCANCHAR('e'); PUGI__SCANCHAR('n'); PUGI__SCANCHAR('c'); PUGI__SCANCHAR('o');
+                                PUGI__SCANCHAR('d'); PUGI__SCANCHAR('i'); PUGI__SCANCHAR('n'); PUGI__SCANCHAR('g');
+
+                                // S? = S?
+                                PUGI__SCANCHARTYPE(ct_space);
+                                PUGI__SCANCHAR('=');
+                                PUGI__SCANCHARTYPE(ct_space);
+
+                                // the only two valid delimiters are ' and "
+                                uint8_t delimiter = (offset < size && data[offset] == '"') ? '"' : '\'';
+
+                                PUGI__SCANCHAR(delimiter);
+
+                                size_t start = offset;
+
+                                out_encoding = data + offset;
+
+                                PUGI__SCANCHARTYPE(ct_symbol);
+
+                                out_length = offset - start;
+
+                                PUGI__SCANCHAR(delimiter);
+
+                                return true;
+                        }
+                }
+
+                return false;
+
+        #undef PUGI__SCANCHAR
+        #undef PUGI__SCANCHARTYPE
+        }
+
+        PUGI__FN xml_encoding guess_buffer_encoding(const uint8_t* data, size_t size)
+        {
+                // skip encoding autodetection if input buffer is too small
+                if (size < 4) return encoding_utf8;
+
+                uint8_t d0 = data[0], d1 = data[1], d2 = data[2], d3 = data[3];
+
                 // look for BOM in first few bytes
                 if (d0 == 0 && d1 == 0 && d2 == 0xfe && d3 == 0xff) return encoding_utf32_be;
@@ -1017 +1967 @@
                 if (d0 == 0 && d1 == 0x3c && d2 == 0 && d3 == 0x3f) return encoding_utf16_be;
                 if (d0 == 0x3c && d1 == 0 && d2 == 0x3f && d3 == 0) return encoding_utf16_le;
-                if (d0 == 0x3c && d1 == 0x3f && d2 == 0x78 && d3 == 0x6d) return encoding_utf8;
 
                 // look for utf16 < followed by node name (this may fail, but is better than utf8 since it's zero terminated so early)
@@ -1023 +1972 @@
                 if (d0 == 0x3c && d1 == 0) return encoding_utf16_le;
 
-                // no known BOM detected, assume utf8
+                // no known BOM detected; parse declaration
+                const uint8_t* enc = 0;
+                size_t enc_length = 0;
+
+                if (d0 == 0x3c && d1 == 0x3f && d2 == 0x78 && d3 == 0x6d && parse_declaration_encoding(data, size, enc, enc_length))
+                {
+                        // iso-8859-1 (case-insensitive)
+                        if (enc_length == 10
+                                && (enc[0] | ' ') == 'i' && (enc[1] | ' ') == 's' && (enc[2] | ' ') == 'o'
+                                && enc[3] == '-' && enc[4] == '8' && enc[5] == '8' && enc[6] == '5' && enc[7] == '9'
+                                && enc[8] == '-' && enc[9] == '1')
+                                return encoding_latin1;
+
+                        // latin1 (case-insensitive)
+                        if (enc_length == 6
+                                && (enc[0] | ' ') == 'l' && (enc[1] | ' ') == 'a' && (enc[2] | ' ') == 't'
+                                && (enc[3] | ' ') == 'i' && (enc[4] | ' ') == 'n'
+                                && enc[5] == '1')
+                                return encoding_latin1;
+                }
+
                 return encoding_utf8;
         }
 
-        xml_encoding get_buffer_encoding(xml_encoding encoding, const void* contents, size_t size)
+        PUGI__FN xml_encoding get_buffer_encoding(xml_encoding encoding, const void* contents, size_t size)
         {
                 // replace wchar encoding with utf implementation
@@ -1041 +2010 @@
                 if (encoding != encoding_auto) return encoding;
 
-                // skip encoding autodetection if input buffer is too small
-                if (size < 4) return encoding_utf8;
-
                 // try to guess encoding (based on XML specification, Appendix F.1)
                 const uint8_t* data = static_cast<const uint8_t*>(contents);
 
-                DMC_VOLATILE uint8_t d0 = data[0], d1 = data[1], d2 = data[2], d3 = data[3];
-
-                return guess_buffer_encoding(d0, d1, d2, d3);
-        }
-
-        bool get_mutable_buffer(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
-        {
+                return guess_buffer_encoding(data, size);
+        }
+
+        PUGI__FN bool get_mutable_buffer(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
+        {
+                size_t length = size / sizeof(char_t);
+
                 if (is_mutable)
                 {
                         out_buffer = static_cast<char_t*>(const_cast<void*>(contents));
+                        out_length = length;
                 }
                 else
                 {
-                        void* buffer = global_allocate(size > 0 ? size : 1);
+                        char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
                         if (!buffer) return false;
 
-                        memcpy(buffer, contents, size);
-
-                        out_buffer = static_cast<char_t*>(buffer);
-                }
-
-                out_length = size / sizeof(char_t);
+                        if (contents)
+                                memcpy(buffer, contents, length * sizeof(char_t));
+                        else
+                                assert(length == 0);
+
+                        buffer[length] = 0;
+
+                        out_buffer = buffer;
+                        out_length = length + 1;
+                }
 
                 return true;
@@ -1074 +2045 @@
 
 #ifdef PUGIXML_WCHAR_MODE
-        inline bool need_endian_swap_utf(xml_encoding le, xml_encoding re)
+        PUGI__FN bool need_endian_swap_utf(xml_encoding le, xml_encoding re)
         {
                 return (le == encoding_utf16_be && re == encoding_utf16_le) || (le == encoding_utf16_le && re == encoding_utf16_be) ||
-                       (le == encoding_utf32_be && re == encoding_utf32_le) || (le == encoding_utf32_le && re == encoding_utf32_be);
-        }
-
-        bool convert_buffer_endian_swap(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
+                           (le == encoding_utf32_be && re == encoding_utf32_le) || (le == encoding_utf32_le && re == encoding_utf32_be);
+        }
+
+        PUGI__FN bool convert_buffer_endian_swap(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
         {
                 const char_t* data = static_cast<const char_t*>(contents);
-        
+                size_t length = size / sizeof(char_t);
+
                 if (is_mutable)
                 {
-                        out_buffer = const_cast<char_t*>(data);
+                        char_t* buffer = const_cast<char_t*>(data);
+
+                        convert_wchar_endian_swap(buffer, data, length);
+
+                        out_buffer = buffer;
+                        out_length = length;
                 }
                 else
                 {
-                        out_buffer = static_cast<char_t*>(global_allocate(size > 0 ? size : 1));
-                        if (!out_buffer) return false;
-                }
-
-                out_length = size / sizeof(char_t);
-
-                convert_wchar_endian_swap(out_buffer, data, out_length);
+                        char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                        if (!buffer) return false;
+
+                        convert_wchar_endian_swap(buffer, data, length);
+                        buffer[length] = 0;
+
+                        out_buffer = buffer;
+                        out_length = length + 1;
+                }
 
                 return true;
         }
 
-        bool convert_buffer_utf8(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size)
-        {
-                const uint8_t* data = static_cast<const uint8_t*>(contents);
+        template <typename D> PUGI__FN bool convert_buffer_generic(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, D)
+        {
+                const typename D::type* data = static_cast<const typename D::type*>(contents);
+                size_t data_length = size / sizeof(typename D::type);
 
                 // first pass: get length in wchar_t units
-                out_length = utf_decoder<wchar_counter>::decode_utf8_block(data, size, 0);
+                size_t length = D::process(data, data_length, 0, wchar_counter());
 
                 // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
-
-                // second pass: convert utf8 input to wchar_t
-                wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
-                wchar_writer::value_type out_end = utf_decoder<wchar_writer>::decode_utf8_block(data, size, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
+                char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                if (!buffer) return false;
+
+                // second pass: convert utf16 input to wchar_t
+                wchar_writer::value_type obegin = reinterpret_cast<wchar_writer::value_type>(buffer);
+                wchar_writer::value_type oend = D::process(data, data_length, obegin, wchar_writer());
+
+                assert(oend == obegin + length);
+                *oend = 0;
+
+                out_buffer = buffer;
+                out_length = length + 1;
 
                 return true;
         }
 
-        template <typename opt_swap> bool convert_buffer_utf16(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
-        {
-                const uint16_t* data = static_cast<const uint16_t*>(contents);
-                size_t length = size / sizeof(uint16_t);
-
-                // first pass: get length in wchar_t units
-                out_length = utf_decoder<wchar_counter, opt_swap>::decode_utf16_block(data, length, 0);
-
-                // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
-
-                // second pass: convert utf16 input to wchar_t
-                wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
-                wchar_writer::value_type out_end = utf_decoder<wchar_writer, opt_swap>::decode_utf16_block(data, length, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
-
-                return true;
-        }
-
-        template <typename opt_swap> bool convert_buffer_utf32(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
-        {
-                const uint32_t* data = static_cast<const uint32_t*>(contents);
-                size_t length = size / sizeof(uint32_t);
-
-                // first pass: get length in wchar_t units
-                out_length = utf_decoder<wchar_counter, opt_swap>::decode_utf32_block(data, length, 0);
-
-                // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
-
-                // second pass: convert utf32 input to wchar_t
-                wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
-                wchar_writer::value_type out_end = utf_decoder<wchar_writer, opt_swap>::decode_utf32_block(data, length, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
-
-                return true;
-        }
-
-        bool convert_buffer(char_t*& out_buffer, size_t& out_length, xml_encoding encoding, const void* contents, size_t size, bool is_mutable)
+        PUGI__FN bool convert_buffer(char_t*& out_buffer, size_t& out_length, xml_encoding encoding, const void* contents, size_t size, bool is_mutable)
         {
                 // get native encoding
@@ -1172 +2111 @@
 
                 // fast path: no conversion required
-                if (encoding == wchar_encoding) return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
+                if (encoding == wchar_encoding)
+                        return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
 
                 // only endian-swapping is required
-                if (need_endian_swap_utf(encoding, wchar_encoding)) return convert_buffer_endian_swap(out_buffer, out_length, contents, size, is_mutable);
+                if (need_endian_swap_utf(encoding, wchar_encoding))
+                        return convert_buffer_endian_swap(out_buffer, out_length, contents, size, is_mutable);
 
                 // source encoding is utf8
-                if (encoding == encoding_utf8) return convert_buffer_utf8(out_buffer, out_length, contents, size);
+                if (encoding == encoding_utf8)
+                        return convert_buffer_generic(out_buffer, out_length, contents, size, utf8_decoder());
 
                 // source encoding is utf16
@@ -1186 +2128 @@
 
                         return (native_encoding == encoding) ?
-                                convert_buffer_utf16(out_buffer, out_length, contents, size, opt_false()) :
-                                convert_buffer_utf16(out_buffer, out_length, contents, size, opt_true());
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf16_decoder<opt_false>()) :
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf16_decoder<opt_true>());
                 }
 
@@ -1196 +2138 @@
 
                         return (native_encoding == encoding) ?
-                                convert_buffer_utf32(out_buffer, out_length, contents, size, opt_false()) :
-                                convert_buffer_utf32(out_buffer, out_length, contents, size, opt_true());
-                }
-
-                assert(!"Invalid encoding");
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf32_decoder<opt_false>()) :
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf32_decoder<opt_true>());
+                }
+
+                // source encoding is latin1
+                if (encoding == encoding_latin1)
+                        return convert_buffer_generic(out_buffer, out_length, contents, size, latin1_decoder());
+
+                assert(false && "Invalid encoding");
                 return false;
         }
 #else
-        template <typename opt_swap> bool convert_buffer_utf16(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
-        {
-                const uint16_t* data = static_cast<const uint16_t*>(contents);
-                size_t length = size / sizeof(uint16_t);
+        template <typename D> PUGI__FN bool convert_buffer_generic(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, D)
+        {
+                const typename D::type* data = static_cast<const typename D::type*>(contents);
+                size_t data_length = size / sizeof(typename D::type);
 
                 // first pass: get length in utf8 units
-                out_length = utf_decoder<utf8_counter, opt_swap>::decode_utf16_block(data, length, 0);
+                size_t length = D::process(data, data_length, 0, utf8_counter());
 
                 // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
+                char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                if (!buffer) return false;
 
                 // second pass: convert utf16 input to utf8
-                uint8_t* out_begin = reinterpret_cast<uint8_t*>(out_buffer);
-                uint8_t* out_end = utf_decoder<utf8_writer, opt_swap>::decode_utf16_block(data, length, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
+                uint8_t* obegin = reinterpret_cast<uint8_t*>(buffer);
+                uint8_t* oend = D::process(data, data_length, obegin, utf8_writer());
+
+                assert(oend == obegin + length);
+                *oend = 0;
+
+                out_buffer = buffer;
+                out_length = length + 1;
 
                 return true;
         }
 
-        template <typename opt_swap> bool convert_buffer_utf32(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
-        {
-                const uint32_t* data = static_cast<const uint32_t*>(contents);
-                size_t length = size / sizeof(uint32_t);
+        PUGI__FN size_t get_latin1_7bit_prefix_length(const uint8_t* data, size_t size)
+        {
+                for (size_t i = 0; i < size; ++i)
+                        if (data[i] > 127)
+                                return i;
+
+                return size;
+        }
+
+        PUGI__FN bool convert_buffer_latin1(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
+        {
+                const uint8_t* data = static_cast<const uint8_t*>(contents);
+                size_t data_length = size;
+
+                // get size of prefix that does not need utf8 conversion
+                size_t prefix_length = get_latin1_7bit_prefix_length(data, data_length);
+                assert(prefix_length <= data_length);
+
+                const uint8_t* postfix = data + prefix_length;
+                size_t postfix_length = data_length - prefix_length;
+
+                // if no conversion is needed, just return the original buffer
+                if (postfix_length == 0) return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
 
                 // first pass: get length in utf8 units
-                out_length = utf_decoder<utf8_counter, opt_swap>::decode_utf32_block(data, length, 0);
+                size_t length = prefix_length + latin1_decoder::process(postfix, postfix_length, 0, utf8_counter());
 
                 // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
-
-                // second pass: convert utf32 input to utf8
-                uint8_t* out_begin = reinterpret_cast<uint8_t*>(out_buffer);
-                uint8_t* out_end = utf_decoder<utf8_writer, opt_swap>::decode_utf32_block(data, length, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
+                char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                if (!buffer) return false;
+
+                // second pass: convert latin1 input to utf8
+                memcpy(buffer, data, prefix_length);
+
+                uint8_t* obegin = reinterpret_cast<uint8_t*>(buffer);
+                uint8_t* oend = latin1_decoder::process(postfix, postfix_length, obegin + prefix_length, utf8_writer());
+
+                assert(oend == obegin + length);
+                *oend = 0;
+
+                out_buffer = buffer;
+                out_length = length + 1;
 
                 return true;
         }
 
-        bool convert_buffer(char_t*& out_buffer, size_t& out_length, xml_encoding encoding, const void* contents, size_t size, bool is_mutable)
+        PUGI__FN bool convert_buffer(char_t*& out_buffer, size_t& out_length, xml_encoding encoding, const void* contents, size_t size, bool is_mutable)
         {
                 // fast path: no conversion required
-                if (encoding == encoding_utf8) return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
+                if (encoding == encoding_utf8)
+                        return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
 
                 // source encoding is utf16
@@ -1259 +2233 @@
 
                         return (native_encoding == encoding) ?
-                                convert_buffer_utf16(out_buffer, out_length, contents, size, opt_false()) :
-                                convert_buffer_utf16(out_buffer, out_length, contents, size, opt_true());
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf16_decoder<opt_false>()) :
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf16_decoder<opt_true>());
                 }
 
@@ -1269 +2243 @@
 
                         return (native_encoding == encoding) ?
-                                convert_buffer_utf32(out_buffer, out_length, contents, size, opt_false()) :
-                                convert_buffer_utf32(out_buffer, out_length, contents, size, opt_true());
-                }
-
-                assert(!"Invalid encoding");
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf32_decoder<opt_false>()) :
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf32_decoder<opt_true>());
+                }
+
+                // source encoding is latin1
+                if (encoding == encoding_latin1)
+                        return convert_buffer_latin1(out_buffer, out_length, contents, size, is_mutable);
+
+                assert(false && "Invalid encoding");
                 return false;
         }
 #endif
 
-        size_t as_utf8_begin(const wchar_t* str, size_t length)
-        {
-                STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);
-
+        PUGI__FN size_t as_utf8_begin(const wchar_t* str, size_t length)
+        {
                 // get length in utf8 characters
-                return sizeof(wchar_t) == 2 ?
-                        utf_decoder<utf8_counter>::decode_utf16_block(reinterpret_cast<const uint16_t*>(str), length, 0) :
-                        utf_decoder<utf8_counter>::decode_utf32_block(reinterpret_cast<const uint32_t*>(str), length, 0);
-    }
-
-    void as_utf8_end(char* buffer, size_t size, const wchar_t* str, size_t length)
-    {
-                STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);
-
-        // convert to utf8
-        uint8_t* begin = reinterpret_cast<uint8_t*>(buffer);
-        uint8_t* end = sizeof(wchar_t) == 2 ?
-            utf_decoder<utf8_writer>::decode_utf16_block(reinterpret_cast<const uint16_t*>(str), length, begin) :
-            utf_decoder<utf8_writer>::decode_utf32_block(reinterpret_cast<const uint32_t*>(str), length, begin);
-    
-        assert(begin + size == end);
-        (void)!end;
-
-                // zero-terminate
-                buffer[size] = 0;
-        }
-    
+                return wchar_decoder::process(str, length, 0, utf8_counter());
+        }
+
+        PUGI__FN void as_utf8_end(char* buffer, size_t size, const wchar_t* str, size_t length)
+        {
+                // convert to utf8
+                uint8_t* begin = reinterpret_cast<uint8_t*>(buffer);
+                uint8_t* end = wchar_decoder::process(str, length, begin, utf8_writer());
+
+                assert(begin + size == end);
+                (void)!end;
+                (void)!size;
+        }
+
 #ifndef PUGIXML_NO_STL
-    std::string as_utf8_impl(const wchar_t* str, size_t length)
-    {
+        PUGI__FN std::string as_utf8_impl(const wchar_t* str, size_t length)
+        {
                 // first pass: get length in utf8 characters
-        size_t size = as_utf8_begin(str, length);
+                size_t size = as_utf8_begin(str, length);
 
                 // allocate resulting string
@@ -1318 +2286 @@
                 if (size > 0) as_utf8_end(&result[0], size, str, length);
 
-                return result;
-    }
-
-        std::wstring as_wide_impl(const char* str, size_t size)
+                return result;
+        }
+
+        PUGI__FN std::basic_string<wchar_t> as_wide_impl(const char* str, size_t size)
         {
                 const uint8_t* data = reinterpret_cast<const uint8_t*>(str);
 
                 // first pass: get length in wchar_t units
-                size_t length = utf_decoder<wchar_counter>::decode_utf8_block(data, size, 0);
+                size_t length = utf8_decoder::process(data, size, 0, wchar_counter());
 
                 // allocate resulting string
-                std::wstring result;
+                std::basic_string<wchar_t> result;
                 result.resize(length);
 
@@ -1336 +2304 @@
                 {
                         wchar_writer::value_type begin = reinterpret_cast<wchar_writer::value_type>(&result[0]);
-                        wchar_writer::value_type end = utf_decoder<wchar_writer>::decode_utf8_block(data, size, begin);
+                        wchar_writer::value_type end = utf8_decoder::process(data, size, begin, wchar_writer());
 
                         assert(begin + length == end);
@@ -1346 +2314 @@
 #endif
 
-        inline bool strcpy_insitu_allow(size_t length, uintptr_t allocated, char_t* target)
-        {
-                assert(target);
+        template <typename Header>
+        inline bool strcpy_insitu_allow(size_t length, const Header& header, uintptr_t header_mask, char_t* target)
+        {
+                // never reuse shared memory
+                if (header & xml_memory_page_contents_shared_mask) return false;
+
                 size_t target_length = strlength(target);
 
                 // always reuse document buffer memory if possible
-                if (!allocated) return target_length >= length;
+                if ((header & header_mask) == 0) return target_length >= length;
 
                 // reuse heap memory if waste is not too great
@@ -1360 +2331 @@
         }
 
-        bool strcpy_insitu(char_t*& dest, uintptr_t& header, uintptr_t header_mask, const char_t* source)
-        {
-                size_t source_length = strlength(source);
-
+        template <typename String, typename Header>
+        PUGI__FN bool strcpy_insitu(String& dest, Header& header, uintptr_t header_mask, const char_t* source, size_t source_length)
+        {
                 if (source_length == 0)
                 {
                         // empty string and null pointer are equivalent, so just deallocate old memory
-                        xml_allocator* alloc = reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask)->allocator;
+                        xml_allocator* alloc = PUGI__GETPAGE_IMPL(header)->allocator;
 
                         if (header & header_mask) alloc->deallocate_string(dest);
-                        
+
                         // mark the string as not allocated
                         dest = 0;
@@ -1377 +2347 @@
                         return true;
                 }
-                else if (dest && strcpy_insitu_allow(source_length, header & header_mask, dest))
+                else if (dest && strcpy_insitu_allow(source_length, header, header_mask, dest))
                 {
                         // we can reuse old buffer, so just copy the new data (including zero terminator)
-                        memcpy(dest, source, (source_length + 1) * sizeof(char_t));
-                        
+                        memcpy(dest, source, source_length * sizeof(char_t));
+                        dest[source_length] = 0;
+
                         return true;
                 }
                 else
                 {
-                        xml_allocator* alloc = reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask)->allocator;
+                        xml_allocator* alloc = PUGI__GETPAGE_IMPL(header)->allocator;
+
+                        if (!alloc->reserve()) return false;
 
                         // allocate new buffer
@@ -1393 +2366 @@
 
                         // copy the string (including zero terminator)
-                        memcpy(buf, source, (source_length + 1) * sizeof(char_t));
+                        memcpy(buf, source, source_length * sizeof(char_t));
+                        buf[source_length] = 0;
 
                         // deallocate old buffer (*after* the above to protect against overlapping memory and/or allocation failures)
                         if (header & header_mask) alloc->deallocate_string(dest);
-                        
+
                         // the string is now allocated, so set the flag
                         dest = buf;
@@ -1410 +2384 @@
                 char_t* end;
                 size_t size;
-                        
+
                 gap(): end(0), size(0)
                 {
                 }
-                        
+
                 // Push new gap, move s count bytes further (skipping the gap).
                 // Collapse previous gap.
@@ -1425 +2399 @@
                                 memmove(end - size, end, reinterpret_cast<char*>(s) - reinterpret_cast<char*>(end));
                         }
-                                
+
                         s += count; // end of current gap
-                                
+
                         // "merge" two gaps
                         end = s;
                         size += count;
                 }
-                        
+
                 // Collapse all gaps, return past-the-end pointer
                 char_t* flush(char_t* s)
@@ -1447 +2421 @@
                 }
         };
-        
-        char_t* strconv_escape(char_t* s, gap& g)
+
+        PUGI__FN char_t* strconv_escape(char_t* s, gap& g)
         {
                 char_t* stre = s + 1;
@@ -1479 +2453 @@
                                                 ch = *++stre;
                                         }
-                                        
+
                                         ++stre;
                                 }
@@ -1490 +2464 @@
                                         for (;;)
                                         {
-                                                if (static_cast<unsigned int>(ch - '0') <= 9)
+                                                if (static_cast<unsigned int>(static_cast<unsigned int>(ch) - '0') <= 9)
                                                         ucsc = 10 * ucsc + (ch - '0');
                                                 else if (ch == ';')
@@ -1499 +2473 @@
                                                 ch = *++stre;
                                         }
-                                        
+
                                         ++stre;
                                 }
@@ -1508 +2482 @@
                                 s = reinterpret_cast<char_t*>(utf8_writer::any(reinterpret_cast<uint8_t*>(s), ucsc));
                         #endif
-                                        
+
                                 g.push(s, stre - s);
                                 return stre;
                         }
+
                         case 'a':       // &a
                         {
@@ -1522 +2497 @@
                                                 *s++ = '&';
                                                 ++stre;
-                                                        
+
                                                 g.push(s, stre - s);
                                                 return stre;
@@ -1540 +2515 @@
                                 break;
                         }
+
                         case 'g': // &g
                         {
@@ -1546 +2522 @@
                                         *s++ = '>';
                                         ++stre;
-                                        
+
                                         g.push(s, stre - s);
                                         return stre;
@@ -1552 +2528 @@
                                 break;
                         }
+
                         case 'l': // &l
                         {
@@ -1558 +2535 @@
                                         *s++ = '<';
                                         ++stre;
-                                                
+
                                         g.push(s, stre - s);
                                         return stre;
@@ -1564 +2541 @@
                                 break;
                         }
+
                         case 'q': // &q
                         {
@@ -1570 +2548 @@
                                         *s++ = '"';
                                         ++stre;
-                                        
+
                                         g.push(s, stre - s);
                                         return stre;
@@ -1576 +2554 @@
                                 break;
                         }
-                }
-                
+
+                        default:
+                                break;
+                }
+
                 return stre;
         }
 
-        // Utility macro for last character handling
-        #define ENDSWITH(c, e) ((c) == (e) || ((c) == 0 && endch == (e)))
-
-        char_t* strconv_comment(char_t* s, char_t endch)
+        // Parser utilities
+        #define PUGI__ENDSWITH(c, e)        ((c) == (e) || ((c) == 0 && endch == (e)))
+        #define PUGI__SKIPWS()              { while (PUGI__IS_CHARTYPE(*s, ct_space)) ++s; }
+        #define PUGI__OPTSET(OPT)           ( optmsk & (OPT) )
+        #define PUGI__PUSHNODE(TYPE)        { cursor = append_new_node(cursor, *alloc, TYPE); if (!cursor) PUGI__THROW_ERROR(status_out_of_memory, s); }
+        #define PUGI__POPNODE()             { cursor = cursor->parent; }
+        #define PUGI__SCANFOR(X)            { while (*s != 0 && !(X)) ++s; }
+        #define PUGI__SCANWHILE(X)          { while (X) ++s; }
+        #define PUGI__SCANWHILE_UNROLL(X)   { for (;;) { char_t ss = s[0]; if (PUGI__UNLIKELY(!(X))) { break; } ss = s[1]; if (PUGI__UNLIKELY(!(X))) { s += 1; break; } ss = s[2]; if (PUGI__UNLIKELY(!(X))) { s += 2; break; } ss = s[3]; if (PUGI__UNLIKELY(!(X))) { s += 3; break; } s += 4; } }
+        #define PUGI__ENDSEG()              { ch = *s; *s = 0; ++s; }
+        #define PUGI__THROW_ERROR(err, m)   return error_offset = m, error_status = err, static_cast<char_t*>(0)
+        #define PUGI__CHECK_ERROR(err, m)   { if (*s == 0) PUGI__THROW_ERROR(err, m); }
+
+        PUGI__FN char_t* strconv_comment(char_t* s, char_t endch)
         {
                 gap g;
-                
+
                 while (true)
                 {
-                        while (!IS_CHARTYPE(*s, ct_parse_comment)) ++s;
-                
+                        PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_comment));
+
                         if (*s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
                         {
                                 *s++ = '\n'; // replace first one with 0x0a
-                                
+
                                 if (*s == '\n') g.push(s, 1);
                         }
-                        else if (s[0] == '-' && s[1] == '-' && ENDSWITH(s[2], '>')) // comment ends here
+                        else if (s[0] == '-' && s[1] == '-' && PUGI__ENDSWITH(s[2], '>')) // comment ends here
                         {
                                 *g.flush(s) = 0;
-                                
+
                                 return s + (s[2] == '>' ? 3 : 2);
                         }
@@ -1612 +2603 @@
         }
 
-        char_t* strconv_cdata(char_t* s, char_t endch)
+        PUGI__FN char_t* strconv_cdata(char_t* s, char_t endch)
         {
                 gap g;
-                        
+
                 while (true)
                 {
-                        while (!IS_CHARTYPE(*s, ct_parse_cdata)) ++s;
-                        
+                        PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_cdata));
+
                         if (*s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
                         {
                                 *s++ = '\n'; // replace first one with 0x0a
-                                
+
                                 if (*s == '\n') g.push(s, 1);
                         }
-                        else if (s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>')) // CDATA ends here
+                        else if (s[0] == ']' && s[1] == ']' && PUGI__ENDSWITH(s[2], '>')) // CDATA ends here
                         {
                                 *g.flush(s) = 0;
-                                
+
                                 return s + 1;
                         }
@@ -1639 +2630 @@
                 }
         }
-        
+
         typedef char_t* (*strconv_pcdata_t)(char_t*);
-                
-        template <typename opt_eol, typename opt_escape> struct strconv_pcdata_impl
+
+        template <typename opt_trim, typename opt_eol, typename opt_escape> struct strconv_pcdata_impl
         {
                 static char_t* parse(char_t* s)
                 {
                         gap g;
-                        
+
+                        char_t* begin = s;
+
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_pcdata)) ++s;
-                                        
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_pcdata));
+
                                 if (*s == '<') // PCDATA ends here
                                 {
-                                        *g.flush(s) = 0;
-                                        
+                                        char_t* end = g.flush(s);
+
+                                        if (opt_trim::value)
+                                                while (end > begin && PUGI__IS_CHARTYPE(end[-1], ct_space))
+                                                        --end;
+
+                                        *end = 0;
+
                                         return s + 1;
                                 }
@@ -1661 +2660 @@
                                 {
                                         *s++ = '\n'; // replace first one with 0x0a
-                                        
+
                                         if (*s == '\n') g.push(s, 1);
                                 }
@@ -1670 +2669 @@
                                 else if (*s == 0)
                                 {
+                                        char_t* end = g.flush(s);
+
+                                        if (opt_trim::value)
+                                                while (end > begin && PUGI__IS_CHARTYPE(end[-1], ct_space))
+                                                        --end;
+
+                                        *end = 0;
+
                                         return s;
                                 }
@@ -1676 +2683 @@
                 }
         };
-        
-        strconv_pcdata_t get_strconv_pcdata(unsigned int optmask)
-        {
-                STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20);
-
-                switch ((optmask >> 4) & 3) // get bitmask for flags (eol escapes)
-                {
-                case 0: return strconv_pcdata_impl<opt_false, opt_false>::parse;
-                case 1: return strconv_pcdata_impl<opt_false, opt_true>::parse;
-                case 2: return strconv_pcdata_impl<opt_true, opt_false>::parse;
-                case 3: return strconv_pcdata_impl<opt_true, opt_true>::parse;
-                default: return 0; // should not get here
+
+        PUGI__FN strconv_pcdata_t get_strconv_pcdata(unsigned int optmask)
+        {
+                PUGI__STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_trim_pcdata == 0x0800);
+
+                switch (((optmask >> 4) & 3) | ((optmask >> 9) & 4)) // get bitmask for flags (eol escapes trim)
+                {
+                case 0: return strconv_pcdata_impl<opt_false, opt_false, opt_false>::parse;
+                case 1: return strconv_pcdata_impl<opt_false, opt_false, opt_true>::parse;
+                case 2: return strconv_pcdata_impl<opt_false, opt_true, opt_false>::parse;
+                case 3: return strconv_pcdata_impl<opt_false, opt_true, opt_true>::parse;
+                case 4: return strconv_pcdata_impl<opt_true, opt_false, opt_false>::parse;
+                case 5: return strconv_pcdata_impl<opt_true, opt_false, opt_true>::parse;
+                case 6: return strconv_pcdata_impl<opt_true, opt_true, opt_false>::parse;
+                case 7: return strconv_pcdata_impl<opt_true, opt_true, opt_true>::parse;
+                default: assert(false); return 0; // should not get here
                 }
         }
 
         typedef char_t* (*strconv_attribute_t)(char_t*, char_t);
-        
+
         template <typename opt_escape> struct strconv_attribute_impl
         {
@@ -1700 +2711 @@
 
                         // trim leading whitespaces
-                        if (IS_CHARTYPE(*s, ct_space))
+                        if (PUGI__IS_CHARTYPE(*s, ct_space))
                         {
                                 char_t* str = s;
-                                
+
                                 do ++str;
-                                while (IS_CHARTYPE(*str, ct_space));
-                                
+                                while (PUGI__IS_CHARTYPE(*str, ct_space));
+
                                 g.push(s, str - s);
                         }
@@ -1712 +2723 @@
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_attr_ws | ct_space)) ++s;
-                                
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_attr_ws | ct_space));
+
                                 if (*s == end_quote)
                                 {
                                         char_t* str = g.flush(s);
-                                        
+
                                         do *str-- = 0;
-                                        while (IS_CHARTYPE(*str, ct_space));
-                                
+                                        while (PUGI__IS_CHARTYPE(*str, ct_space));
+
                                         return s + 1;
                                 }
-                                else if (IS_CHARTYPE(*s, ct_space))
+                                else if (PUGI__IS_CHARTYPE(*s, ct_space))
                                 {
                                         *s++ = ' ';
-                
-                                        if (IS_CHARTYPE(*s, ct_space))
+
+                                        if (PUGI__IS_CHARTYPE(*s, ct_space))
                                         {
                                                 char_t* str = s + 1;
-                                                while (IS_CHARTYPE(*str, ct_space)) ++str;
-                                                
+                                                while (PUGI__IS_CHARTYPE(*str, ct_space)) ++str;
+
                                                 g.push(s, str - s);
                                         }
@@ -1753 +2764 @@
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_attr_ws)) ++s;
-                                
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_attr_ws));
+
                                 if (*s == end_quote)
                                 {
                                         *g.flush(s) = 0;
-                                
+
                                         return s + 1;
                                 }
-                                else if (IS_CHARTYPE(*s, ct_space))
+                                else if (PUGI__IS_CHARTYPE(*s, ct_space))
                                 {
                                         if (*s == '\r')
                                         {
                                                 *s++ = ' ';
-                                
+
                                                 if (*s == '\n') g.push(s, 1);
                                         }
@@ -1789 +2800 @@
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_attr)) ++s;
-                                
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_attr));
+
                                 if (*s == end_quote)
                                 {
                                         *g.flush(s) = 0;
-                                
+
                                         return s + 1;
                                 }
@@ -1800 +2811 @@
                                 {
                                         *s++ = '\n';
-                                        
+
                                         if (*s == '\n') g.push(s, 1);
                                 }
@@ -1821 +2832 @@
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_attr)) ++s;
-                                
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_attr));
+
                                 if (*s == end_quote)
                                 {
                                         *g.flush(s) = 0;
-                                
+
                                         return s + 1;
                                 }
@@ -1842 +2853 @@
         };
 
-        strconv_attribute_t get_strconv_attribute(unsigned int optmask)
-        {
-                STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_wconv_attribute == 0x40 && parse_wnorm_attribute == 0x80);
-                
+        PUGI__FN strconv_attribute_t get_strconv_attribute(unsigned int optmask)
+        {
+                PUGI__STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_wconv_attribute == 0x40 && parse_wnorm_attribute == 0x80);
+
                 switch ((optmask >> 4) & 15) // get bitmask for flags (wconv wnorm eol escapes)
                 {
@@ -1864 +2875 @@
                 case 14: return strconv_attribute_impl<opt_false>::parse_wnorm;
                 case 15: return strconv_attribute_impl<opt_true>::parse_wnorm;
-                default: return 0; // should not get here
+                default: assert(false); return 0; // should not get here
                 }
         }
@@ -1879 +2890 @@
         struct xml_parser
         {
-                xml_allocator alloc;
+                xml_allocator* alloc;
                 char_t* error_offset;
-                jmp_buf error_handler;
-                
-                // Parser utilities.
-                #define SKIPWS()                        { while (IS_CHARTYPE(*s, ct_space)) ++s; }
-                #define OPTSET(OPT)                     ( optmsk & OPT )
-                #define PUSHNODE(TYPE)          { cursor = append_node(cursor, alloc, TYPE); if (!cursor) THROW_ERROR(status_out_of_memory, s); }
-                #define POPNODE()                       { cursor = cursor->parent; }
-                #define SCANFOR(X)                      { while (*s != 0 && !(X)) ++s; }
-                #define SCANWHILE(X)            { while ((X)) ++s; }
-                #define ENDSEG()                        { ch = *s; *s = 0; ++s; }
-                #define THROW_ERROR(err, m)     error_offset = m, longjmp(error_handler, err)
-                #define CHECK_ERROR(err, m)     { if (*s == 0) THROW_ERROR(err, m); }
-                
-                xml_parser(const xml_allocator& alloc): alloc(alloc), error_offset(0)
+                xml_parse_status error_status;
+
+                xml_parser(xml_allocator* alloc_): alloc(alloc_), error_offset(0), error_status(status_ok)
                 {
                 }
@@ -1911 +2911 @@
                                 // quoted string
                                 char_t ch = *s++;
-                                SCANFOR(*s == ch);
-                                if (!*s) THROW_ERROR(status_bad_doctype, s);
+                                PUGI__SCANFOR(*s == ch);
+                                if (!*s) PUGI__THROW_ERROR(status_bad_doctype, s);
 
                                 s++;
@@ -1920 +2920 @@
                                 // <? ... ?>
                                 s += 2;
-                                SCANFOR(s[0] == '?' && s[1] == '>'); // no need for ENDSWITH because ?> can't terminate proper doctype
-                                if (!*s) THROW_ERROR(status_bad_doctype, s);
+                                PUGI__SCANFOR(s[0] == '?' && s[1] == '>'); // no need for ENDSWITH because ?> can't terminate proper doctype
+                                if (!*s) PUGI__THROW_ERROR(status_bad_doctype, s);
 
                                 s += 2;
@@ -1928 +2928 @@
                         {
                                 s += 4;
-                                SCANFOR(s[0] == '-' && s[1] == '-' && s[2] == '>'); // no need for ENDSWITH because --> can't terminate proper doctype
-                                if (!*s) THROW_ERROR(status_bad_doctype, s);
-
-                                s += 4;
-                        }
-                        else THROW_ERROR(status_bad_doctype, s);
+                                PUGI__SCANFOR(s[0] == '-' && s[1] == '-' && s[2] == '>'); // no need for ENDSWITH because --> can't terminate proper doctype
+                                if (!*s) PUGI__THROW_ERROR(status_bad_doctype, s);
+
+                                s += 3;
+                        }
+                        else PUGI__THROW_ERROR(status_bad_doctype, s);
 
                         return s;
@@ -1940 +2940 @@
                 char_t* parse_doctype_ignore(char_t* s)
                 {
+                        size_t depth = 0;
+
                         assert(s[0] == '<' && s[1] == '!' && s[2] == '[');
-                        s++;
+                        s += 3;
 
                         while (*s)
@@ -1948 +2950 @@
                                 {
                                         // nested ignore section
-                                        s = parse_doctype_ignore(s);
+                                        s += 3;
+                                        depth++;
                                 }
                                 else if (s[0] == ']' && s[1] == ']' && s[2] == '>')
@@ -1955 +2958 @@
                                         s += 3;
 
-                                        return s;
+                                        if (depth == 0)
+                                                return s;
+
+                                        depth--;
                                 }
                                 else s++;
                         }
 
-                        THROW_ERROR(status_bad_doctype, s);
-
-                        return s;
-                }
-
-                char_t* parse_doctype_group(char_t* s, char_t endch, bool toplevel)
-                {
-                        assert(s[0] == '<' && s[1] == '!');
-                        s++;
+                        PUGI__THROW_ERROR(status_bad_doctype, s);
+                }
+
+                char_t* parse_doctype_group(char_t* s, char_t endch)
+                {
+                        size_t depth = 0;
+
+                        assert((s[0] == '<' || s[0] == 0) && s[1] == '!');
+                        s += 2;
 
                         while (*s)
@@ -1978 +2984 @@
                                                 // ignore
                                                 s = parse_doctype_ignore(s);
+                                                if (!s) return s;
                                         }
                                         else
                                         {
                                                 // some control group
-                                                s = parse_doctype_group(s, endch, false);
+                                                s += 2;
+                                                depth++;
                                         }
                                 }
@@ -1989 +2997 @@
                                         // unknown tag (forbidden), or some primitive group
                                         s = parse_doctype_primitive(s);
+                                        if (!s) return s;
                                 }
                                 else if (*s == '>')
                                 {
+                                        if (depth == 0)
+                                                return s;
+
+                                        depth--;
                                         s++;
-
-                                        return s;
                                 }
                                 else s++;
                         }
 
-                        if (!toplevel || endch != '>') THROW_ERROR(status_bad_doctype, s);
+                        if (depth != 0 || endch != '>') PUGI__THROW_ERROR(status_bad_doctype, s);
 
                         return s;
@@ -2017 +3028 @@
                                         ++s;
 
-                                        if (OPTSET(parse_comments))
+                                        if (PUGI__OPTSET(parse_comments))
                                         {
-                                                PUSHNODE(node_comment); // Append a new node on the tree.
+                                                PUGI__PUSHNODE(node_comment); // Append a new node on the tree.
                                                 cursor->value = s; // Save the offset.
                                         }
 
-                                        if (OPTSET(parse_eol) && OPTSET(parse_comments))
+                                        if (PUGI__OPTSET(parse_eol) && PUGI__OPTSET(parse_comments))
                                         {
                                                 s = strconv_comment(s, endch);
 
-                                                if (!s) THROW_ERROR(status_bad_comment, cursor->value);
+                                                if (!s) PUGI__THROW_ERROR(status_bad_comment, cursor->value);
                                         }
                                         else
                                         {
                                                 // Scan for terminating '-->'.
-                                                SCANFOR(s[0] == '-' && s[1] == '-' && ENDSWITH(s[2], '>'));
-                                                CHECK_ERROR(status_bad_comment, s);
-
-                                                if (OPTSET(parse_comments))
+                                                PUGI__SCANFOR(s[0] == '-' && s[1] == '-' && PUGI__ENDSWITH(s[2], '>'));
+                                                PUGI__CHECK_ERROR(status_bad_comment, s);
+
+                                                if (PUGI__OPTSET(parse_comments))
                                                         *s = 0; // Zero-terminate this segment at the first terminating '-'.
 
@@ -2041 +3052 @@
                                         }
                                 }
-                                else THROW_ERROR(status_bad_comment, s);
+                                else PUGI__THROW_ERROR(status_bad_comment, s);
                         }
                         else if (*s == '[')
@@ -2050 +3061 @@
                                         ++s;
 
-                                        if (OPTSET(parse_cdata))
+                                        if (PUGI__OPTSET(parse_cdata))
                                         {
-                                                PUSHNODE(node_cdata); // Append a new node on the tree.
+                                                PUGI__PUSHNODE(node_cdata); // Append a new node on the tree.
                                                 cursor->value = s; // Save the offset.
 
-                                                if (OPTSET(parse_eol))
+                                                if (PUGI__OPTSET(parse_eol))
                                                 {
                                                         s = strconv_cdata(s, endch);
 
-                                                        if (!s) THROW_ERROR(status_bad_cdata, cursor->value);
+                                                        if (!s) PUGI__THROW_ERROR(status_bad_cdata, cursor->value);
                                                 }
                                                 else
                                                 {
                                                         // Scan for terminating ']]>'.
-                                                        SCANFOR(s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>'));
-                                                        CHECK_ERROR(status_bad_cdata, s);
+                                                        PUGI__SCANFOR(s[0] == ']' && s[1] == ']' && PUGI__ENDSWITH(s[2], '>'));
+                                                        PUGI__CHECK_ERROR(status_bad_cdata, s);
 
                                                         *s++ = 0; // Zero-terminate this segment.
@@ -2073 +3084 @@
                                         {
                                                 // Scan for terminating ']]>'.
-                                                SCANFOR(s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>'));
-                                                CHECK_ERROR(status_bad_cdata, s);
+                                                PUGI__SCANFOR(s[0] == ']' && s[1] == ']' && PUGI__ENDSWITH(s[2], '>'));
+                                                PUGI__CHECK_ERROR(status_bad_cdata, s);
 
                                                 ++s;
@@ -2081 +3092 @@
                                         s += (s[1] == '>' ? 2 : 1); // Step over the last ']>'.
                                 }
-                                else THROW_ERROR(status_bad_cdata, s);
-                        }
-                        else if (s[0] == 'D' && s[1] == 'O' && s[2] == 'C' && s[3] == 'T' && s[4] == 'Y' && s[5] == 'P' && ENDSWITH(s[6], 'E'))
+                                else PUGI__THROW_ERROR(status_bad_cdata, s);
+                        }
+                        else if (s[0] == 'D' && s[1] == 'O' && s[2] == 'C' && s[3] == 'T' && s[4] == 'Y' && s[5] == 'P' && PUGI__ENDSWITH(s[6], 'E'))
                         {
                                 s -= 2;
 
-                if (cursor->parent) THROW_ERROR(status_bad_doctype, s);
-
-                char_t* mark = s + 9;
-
-                                s = parse_doctype_group(s, endch, true);
-
-                if (OPTSET(parse_doctype))
-                {
-                    while (IS_CHARTYPE(*mark, ct_space)) ++mark;
-
-                    PUSHNODE(node_doctype);
-
-                    cursor->value = mark;
-
-                    assert((s[0] == 0 && endch == '>') || s[-1] == '>');
-                    s[*s == 0 ? 0 : -1] = 0;
-
-                    POPNODE();
-                }
-                        }
-                        else if (*s == 0 && endch == '-') THROW_ERROR(status_bad_comment, s);
-                        else if (*s == 0 && endch == '[') THROW_ERROR(status_bad_cdata, s);
-                        else THROW_ERROR(status_unrecognized_tag, s);
+                                if (cursor->parent) PUGI__THROW_ERROR(status_bad_doctype, s);
+
+                                char_t* mark = s + 9;
+
+                                s = parse_doctype_group(s, endch);
+                                if (!s) return s;
+
+                                assert((*s == 0 && endch == '>') || *s == '>');
+                                if (*s) *s++ = 0;
+
+                                if (PUGI__OPTSET(parse_doctype))
+                                {
+                                        while (PUGI__IS_CHARTYPE(*mark, ct_space)) ++mark;
+
+                                        PUGI__PUSHNODE(node_doctype);
+
+                                        cursor->value = mark;
+                                }
+                        }
+                        else if (*s == 0 && endch == '-') PUGI__THROW_ERROR(status_bad_comment, s);
+                        else if (*s == 0 && endch == '[') PUGI__THROW_ERROR(status_bad_cdata, s);
+                        else PUGI__THROW_ERROR(status_unrecognized_tag, s);
 
                         return s;
@@ -2126 +3136 @@
                         char_t* target = s;
 
-                        if (!IS_CHARTYPE(*s, ct_start_symbol)) THROW_ERROR(status_bad_pi, s);
-
-                        SCANWHILE(IS_CHARTYPE(*s, ct_symbol));
-                        CHECK_ERROR(status_bad_pi, s);
+                        if (!PUGI__IS_CHARTYPE(*s, ct_start_symbol)) PUGI__THROW_ERROR(status_bad_pi, s);
+
+                        PUGI__SCANWHILE(PUGI__IS_CHARTYPE(*s, ct_symbol));
+                        PUGI__CHECK_ERROR(status_bad_pi, s);
 
                         // determine node type; stricmp / strcasecmp is not portable
                         bool declaration = (target[0] | ' ') == 'x' && (target[1] | ' ') == 'm' && (target[2] | ' ') == 'l' && target + 3 == s;
 
-                        if (declaration ? OPTSET(parse_declaration) : OPTSET(parse_pi))
+                        if (declaration ? PUGI__OPTSET(parse_declaration) : PUGI__OPTSET(parse_pi))
                         {
                                 if (declaration)
                                 {
                                         // disallow non top-level declarations
-                                        if (cursor->parent) THROW_ERROR(status_bad_pi, s);
-
-                                        PUSHNODE(node_declaration);
+                                        if (cursor->parent) PUGI__THROW_ERROR(status_bad_pi, s);
+
+                                        PUGI__PUSHNODE(node_declaration);
                                 }
                                 else
                                 {
-                                        PUSHNODE(node_pi);
+                                        PUGI__PUSHNODE(node_pi);
                                 }
 
                                 cursor->name = target;
 
-                                ENDSEG();
+                                PUGI__ENDSEG();
 
                                 // parse value/attributes
@@ -2156 +3166 @@
                                 {
                                         // empty node
-                                        if (!ENDSWITH(*s, '>')) THROW_ERROR(status_bad_pi, s);
+                                        if (!PUGI__ENDSWITH(*s, '>')) PUGI__THROW_ERROR(status_bad_pi, s);
                                         s += (*s == '>');
 
-                                        POPNODE();
-                                }
-                                else if (IS_CHARTYPE(ch, ct_space))
-                                {
-                                        SKIPWS();
+                                        PUGI__POPNODE();
+                                }
+                                else if (PUGI__IS_CHARTYPE(ch, ct_space))
+                                {
+                                        PUGI__SKIPWS();
 
                                         // scan for tag end
                                         char_t* value = s;
 
-                                        SCANFOR(s[0] == '?' && ENDSWITH(s[1], '>'));
-                                        CHECK_ERROR(status_bad_pi, s);
+                                        PUGI__SCANFOR(s[0] == '?' && PUGI__ENDSWITH(s[1], '>'));
+                                        PUGI__CHECK_ERROR(status_bad_pi, s);
 
                                         if (declaration)
@@ -2183 +3193 @@
                                                 // store value and step over >
                                                 cursor->value = value;
-                                                POPNODE();
-
-                                                ENDSEG();
+
+                                                PUGI__POPNODE();
+
+                                                PUGI__ENDSEG();
 
                                                 s += (*s == '>');
                                         }
                                 }
-                                else THROW_ERROR(status_bad_pi, s);
+                                else PUGI__THROW_ERROR(status_bad_pi, s);
                         }
                         else
                         {
                                 // scan for tag end
-                                SCANFOR(s[0] == '?' && ENDSWITH(s[1], '>'));
-                                CHECK_ERROR(status_bad_pi, s);
+                                PUGI__SCANFOR(s[0] == '?' && PUGI__ENDSWITH(s[1], '>'));
+                                PUGI__CHECK_ERROR(status_bad_pi, s);
 
                                 s += (s[1] == '>' ? 2 : 1);
@@ -2207 +3218 @@
                 }
 
-                void parse(char_t* s, xml_node_struct* xmldoc, unsigned int optmsk, char_t endch)
+                char_t* parse_tree(char_t* s, xml_node_struct* root, unsigned int optmsk, char_t endch)
                 {
                         strconv_attribute_t strconv_attribute = get_strconv_attribute(optmsk);
                         strconv_pcdata_t strconv_pcdata = get_strconv_pcdata(optmsk);
-                        
+
                         char_t ch = 0;
-                        xml_node_struct* cursor = xmldoc;
+                        xml_node_struct* cursor = root;
                         char_t* mark = s;
 
@@ -2223 +3234 @@
 
                                 LOC_TAG:
-                                        if (IS_CHARTYPE(*s, ct_start_symbol)) // '<#...'
+                                        if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) // '<#...'
                                         {
-                                                PUSHNODE(node_element); // Append a new node to the tree.
+                                                PUGI__PUSHNODE(node_element); // Append a new node to the tree.
 
                                                 cursor->name = s;
 
-                                                SCANWHILE(IS_CHARTYPE(*s, ct_symbol)); // Scan for a terminator.
-                                                ENDSEG(); // Save char in 'ch', terminate & step over.
+                                                PUGI__SCANWHILE_UNROLL(PUGI__IS_CHARTYPE(ss, ct_symbol)); // Scan for a terminator.
+                                                PUGI__ENDSEG(); // Save char in 'ch', terminate & step over.
 
                                                 if (ch == '>')
@@ -2236 +3247 @@
                                                         // end of tag
                                                 }
-                                                else if (IS_CHARTYPE(ch, ct_space))
+                                                else if (PUGI__IS_CHARTYPE(ch, ct_space))
                                                 {
                                                 LOC_ATTRIBUTES:
-                                                    while (true)
-                                                    {
-                                                                SKIPWS(); // Eat any whitespace.
-                                                
-                                                                if (IS_CHARTYPE(*s, ct_start_symbol)) // <... #...
+                                                        while (true)
+                                                        {
+                                                                PUGI__SKIPWS(); // Eat any whitespace.
+
+                                                                if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) // <... #...
                                                                 {
-                                                                        xml_attribute_struct* a = append_attribute_ll(cursor, alloc); // Make space for this attribute.
-                                                                        if (!a) THROW_ERROR(status_out_of_memory, s);
+                                                                        xml_attribute_struct* a = append_new_attribute(cursor, *alloc); // Make space for this attribute.
+                                                                        if (!a) PUGI__THROW_ERROR(status_out_of_memory, s);
 
                                                                         a->name = s; // Save the offset.
 
-                                                                        SCANWHILE(IS_CHARTYPE(*s, ct_symbol)); // Scan for a terminator.
-                                                                        CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance
-
-                                                                        ENDSEG(); // Save char in 'ch', terminate & step over.
-                                                                        CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance
-
-                                                                        if (IS_CHARTYPE(ch, ct_space))
+                                                                        PUGI__SCANWHILE_UNROLL(PUGI__IS_CHARTYPE(ss, ct_symbol)); // Scan for a terminator.
+                                                                        PUGI__ENDSEG(); // Save char in 'ch', terminate & step over.
+
+                                                                        if (PUGI__IS_CHARTYPE(ch, ct_space))
                                                                         {
-                                                                                SKIPWS(); // Eat any whitespace.
-                                                                                CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance
+                                                                                PUGI__SKIPWS(); // Eat any whitespace.
 
                                                                                 ch = *s;
                                                                                 ++s;
                                                                         }
-                                                                        
+
                                                                         if (ch == '=') // '<... #=...'
                                                                         {
-                                                                                SKIPWS(); // Eat any whitespace.
+                                                                                PUGI__SKIPWS(); // Eat any whitespace.
 
                                                                                 if (*s == '"' || *s == '\'') // '<... #="...'
@@ -2276 +3283 @@
 
                                                                                         s = strconv_attribute(s, ch);
-                                                                                
-                                                                                        if (!s) THROW_ERROR(status_bad_attribute, a->value);
+
+                                                                                        if (!s) PUGI__THROW_ERROR(status_bad_attribute, a->value);
 
                                                                                         // After this line the loop continues from the start;
                                                                                         // Whitespaces, / and > are ok, symbols and EOF are wrong,
                                                                                         // everything else will be detected
-                                                                                        if (IS_CHARTYPE(*s, ct_start_symbol)) THROW_ERROR(status_bad_attribute, s);
+                                                                                        if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) PUGI__THROW_ERROR(status_bad_attribute, s);
                                                                                 }
-                                                                                else THROW_ERROR(status_bad_attribute, s);
+                                                                                else PUGI__THROW_ERROR(status_bad_attribute, s);
                                                                         }
-                                                                        else THROW_ERROR(status_bad_attribute, s);
+                                                                        else PUGI__THROW_ERROR(status_bad_attribute, s);
                                                                 }
                                                                 else if (*s == '/')
                                                                 {
                                                                         ++s;
-                                                                        
+
                                                                         if (*s == '>')
                                                                         {
-                                                                                POPNODE();
+                                                                                PUGI__POPNODE();
                                                                                 s++;
                                                                                 break;
@@ -2300 +3307 @@
                                                                         else if (*s == 0 && endch == '>')
                                                                         {
-                                                                                POPNODE();
+                                                                                PUGI__POPNODE();
                                                                                 break;
                                                                         }
-                                                                        else THROW_ERROR(status_bad_start_element, s);
+                                                                        else PUGI__THROW_ERROR(status_bad_start_element, s);
                                                                 }
                                                                 else if (*s == '>')
@@ -2315 +3322 @@
                                                                         break;
                                                                 }
-                                                                else THROW_ERROR(status_bad_start_element, s);
+                                                                else PUGI__THROW_ERROR(status_bad_start_element, s);
                                                         }
 
@@ -2322 +3329 @@
                                                 else if (ch == '/') // '<#.../'
                                                 {
-                                                        if (!ENDSWITH(*s, '>')) THROW_ERROR(status_bad_start_element, s);
-
-                                                        POPNODE(); // Pop.
+                                                        if (!PUGI__ENDSWITH(*s, '>')) PUGI__THROW_ERROR(status_bad_start_element, s);
+
+                                                        PUGI__POPNODE(); // Pop.
 
                                                         s += (*s == '>');
@@ -2332 +3339 @@
                                                         // we stepped over null terminator, backtrack & handle closing tag
                                                         --s;
-                                                        
-                                                        if (endch != '>') THROW_ERROR(status_bad_start_element, s);
+
+                                                        if (endch != '>') PUGI__THROW_ERROR(status_bad_start_element, s);
                                                 }
-                                                else THROW_ERROR(status_bad_start_element, s);
+                                                else PUGI__THROW_ERROR(status_bad_start_element, s);
                                         }
                                         else if (*s == '/')
@@ -2341 +3348 @@
                                                 ++s;
 
+                                                mark = s;
+
                                                 char_t* name = cursor->name;
-                                                if (!name) THROW_ERROR(status_end_element_mismatch, s);
-                                                
-                                                while (IS_CHARTYPE(*s, ct_symbol))
+                                                if (!name) PUGI__THROW_ERROR(status_end_element_mismatch, mark);
+
+                                                while (PUGI__IS_CHARTYPE(*s, ct_symbol))
                                                 {
-                                                        if (*s++ != *name++) THROW_ERROR(status_end_element_mismatch, s);
+                                                        if (*s++ != *name++) PUGI__THROW_ERROR(status_end_element_mismatch, mark);
                                                 }
 
                                                 if (*name)
                                                 {
-                                                        if (*s == 0 && name[0] == endch && name[1] == 0) THROW_ERROR(status_bad_end_element, s);
-                                                        else THROW_ERROR(status_end_element_mismatch, s);
+                                                        if (*s == 0 && name[0] == endch && name[1] == 0) PUGI__THROW_ERROR(status_bad_end_element, s);
+                                                        else PUGI__THROW_ERROR(status_end_element_mismatch, mark);
                                                 }
-                                                        
-                                                POPNODE(); // Pop.
-
-                                                SKIPWS();
+
+                                                PUGI__POPNODE(); // Pop.
+
+                                                PUGI__SKIPWS();
 
                                                 if (*s == 0)
                                                 {
-                                                        if (endch != '>') THROW_ERROR(status_bad_end_element, s);
+                                                        if (endch != '>') PUGI__THROW_ERROR(status_bad_end_element, s);
                                                 }
                                                 else
                                                 {
-                                                        if (*s != '>') THROW_ERROR(status_bad_end_element, s);
+                                                        if (*s != '>') PUGI__THROW_ERROR(status_bad_end_element, s);
                                                         ++s;
                                                 }
@@ -2372 +3381 @@
                                         {
                                                 s = parse_question(s, cursor, optmsk, endch);
+                                                if (!s) return s;
 
                                                 assert(cursor);
-                                                if ((cursor->header & xml_memory_page_type_mask) + 1 == node_declaration) goto LOC_ATTRIBUTES;
+                                                if (PUGI__NODETYPE(cursor) == node_declaration) goto LOC_ATTRIBUTES;
                                         }
                                         else if (*s == '!') // '<!...'
                                         {
                                                 s = parse_exclamation(s, cursor, optmsk, endch);
+                                                if (!s) return s;
                                         }
-                                        else if (*s == 0 && endch == '?') THROW_ERROR(status_bad_pi, s);
-                                        else THROW_ERROR(status_unrecognized_tag, s);
+                                        else if (*s == 0 && endch == '?') PUGI__THROW_ERROR(status_bad_pi, s);
+                                        else PUGI__THROW_ERROR(status_unrecognized_tag, s);
                                 }
                                 else
@@ -2387 +3398 @@
                                         mark = s; // Save this offset while searching for a terminator.
 
-                                        SKIPWS(); // Eat whitespace if no genuine PCDATA here.
-
-                                        if ((!OPTSET(parse_ws_pcdata) || mark == s) && (*s == '<' || !*s))
+                                        PUGI__SKIPWS(); // Eat whitespace if no genuine PCDATA here.
+
+                                        if (*s == '<' || !*s)
                                         {
-                                                continue;
+                                                // We skipped some whitespace characters because otherwise we would take the tag branch instead of PCDATA one
+                                                assert(mark != s);
+
+                                                if (!PUGI__OPTSET(parse_ws_pcdata | parse_ws_pcdata_single) || PUGI__OPTSET(parse_trim_pcdata))
+                                                {
+                                                        continue;
+                                                }
+                                                else if (PUGI__OPTSET(parse_ws_pcdata_single))
+                                                {
+                                                        if (s[0] != '<' || s[1] != '/' || cursor->first_child) continue;
+                                                }
                                         }
 
-                                        s = mark;
-                                                        
-                                        if (cursor->parent)
+                                        if (!PUGI__OPTSET(parse_trim_pcdata))
+                                                s = mark;
+
+                                        if (cursor->parent || PUGI__OPTSET(parse_fragment))
                                         {
-                                                PUSHNODE(node_pcdata); // Append a new node on the tree.
-                                                cursor->value = s; // Save the offset.
+                                                if (PUGI__OPTSET(parse_embed_pcdata) && cursor->parent && !cursor->first_child && !cursor->value)
+                                                {
+                                                        cursor->value = s; // Save the offset.
+                                                }
+                                                else
+                                                {
+                                                        PUGI__PUSHNODE(node_pcdata); // Append a new node on the tree.
+
+                                                        cursor->value = s; // Save the offset.
+
+                                                        PUGI__POPNODE(); // Pop since this is a standalone.
+                                                }
 
                                                 s = strconv_pcdata(s);
-                                                                
-                                                POPNODE(); // Pop since this is a standalone.
-                                                
+
                                                 if (!*s) break;
                                         }
                                         else
                                         {
-                                                SCANFOR(*s == '<'); // '...<'
+                                                PUGI__SCANFOR(*s == '<'); // '...<'
                                                 if (!*s) break;
-                                                
+
                                                 ++s;
                                         }
@@ -2421 +3451 @@
 
                         // check that last tag is closed
-                        if (cursor != xmldoc) THROW_ERROR(status_end_element_mismatch, s);
-                }
-
-                static xml_parse_result parse(char_t* buffer, size_t length, xml_node_struct* root, unsigned int optmsk)
-                {
-                        xml_document_struct* xmldoc = static_cast<xml_document_struct*>(root);
-
-                        // store buffer for offset_debug
-                        xmldoc->buffer = buffer;
-
+                        if (cursor != root) PUGI__THROW_ERROR(status_end_element_mismatch, s);
+
+                        return s;
+                }
+
+        #ifdef PUGIXML_WCHAR_MODE
+                static char_t* parse_skip_bom(char_t* s)
+                {
+                        unsigned int bom = 0xfeff;
+                        return (s[0] == static_cast<wchar_t>(bom)) ? s + 1 : s;
+                }
+        #else
+                static char_t* parse_skip_bom(char_t* s)
+                {
+                        return (s[0] == '\xef' && s[1] == '\xbb' && s[2] == '\xbf') ? s + 3 : s;
+                }
+        #endif
+
+                static bool has_element_node_siblings(xml_node_struct* node)
+                {
+                        while (node)
+                        {
+                                if (PUGI__NODETYPE(node) == node_element) return true;
+
+                                node = node->next_sibling;
+                        }
+
+                        return false;
+                }
+
+                static xml_parse_result parse(char_t* buffer, size_t length, xml_document_struct* xmldoc, xml_node_struct* root, unsigned int optmsk)
+                {
                         // early-out for empty documents
-                        if (length == 0) return make_parse_result(status_ok);
+                        if (length == 0)
+                                return make_parse_result(PUGI__OPTSET(parse_fragment) ? status_ok : status_no_document_element);
+
+                        // get last child of the root before parsing
+                        xml_node_struct* last_root_child = root->first_child ? root->first_child->prev_sibling_c + 0 : 0;
 
                         // create parser on stack
-                        xml_parser parser(*xmldoc);
+                        xml_parser parser(static_cast<xml_allocator*>(xmldoc));
 
                         // save last character and make buffer zero-terminated (speeds up parsing)
                         char_t endch = buffer[length - 1];
                         buffer[length - 1] = 0;
-                        
+
+                        // skip BOM to make sure it does not end up as part of parse output
+                        char_t* buffer_data = parse_skip_bom(buffer);
+
                         // perform actual parsing
-                        int error = setjmp(parser.error_handler);
-
-                        if (error == 0)
-                        {
-                                parser.parse(buffer, xmldoc, optmsk, endch);
-                        }
-
-                        xml_parse_result result = make_parse_result(static_cast<xml_parse_status>(error), parser.error_offset ? parser.error_offset - buffer : 0);
+                        parser.parse_tree(buffer_data, root, optmsk, endch);
+
+                        xml_parse_result result = make_parse_result(parser.error_status, parser.error_offset ? parser.error_offset - buffer : 0);
                         assert(result.offset >= 0 && static_cast<size_t>(result.offset) <= length);
 
-                        // update allocator state
-                        *static_cast<xml_allocator*>(xmldoc) = parser.alloc;
-
-                        // since we removed last character, we have to handle the only possible false positive
-                        if (result && endch == '<')
-                        {
-                                // there's no possible well-formed document with < at the end
-                                return make_parse_result(status_unrecognized_tag, length);
+                        if (result)
+                        {
+                                // since we removed last character, we have to handle the only possible false positive (stray <)
+                                if (endch == '<')
+                                        return make_parse_result(status_unrecognized_tag, length - 1);
+
+                                // check if there are any element nodes parsed
+                                xml_node_struct* first_root_child_parsed = last_root_child ? last_root_child->next_sibling + 0 : root->first_child+ 0;
+
+                                if (!PUGI__OPTSET(parse_fragment) && !has_element_node_siblings(first_root_child_parsed))
+                                        return make_parse_result(status_no_document_element, length - 1);
+                        }
+                        else
+                        {
+                                // roll back offset if it occurs on a null terminator in the source buffer
+                                if (result.offset > 0 && static_cast<size_t>(result.offset) == length - 1 && endch == 0)
+                                        result.offset--;
                         }
 
@@ -2467 +3530 @@
 
         // Output facilities
-        xml_encoding get_write_native_encoding()
+        PUGI__FN xml_encoding get_write_native_encoding()
         {
         #ifdef PUGIXML_WCHAR_MODE
@@ -2476 +3539 @@
         }
 
-        xml_encoding get_write_encoding(xml_encoding encoding)
+        PUGI__FN xml_encoding get_write_encoding(xml_encoding encoding)
         {
                 // replace wchar encoding with utf implementation
@@ -2494 +3557 @@
         }
 
+        template <typename D, typename T> PUGI__FN size_t convert_buffer_output_generic(typename T::value_type dest, const char_t* data, size_t length, D, T)
+        {
+                PUGI__STATIC_ASSERT(sizeof(char_t) == sizeof(typename D::type));
+
+                typename T::value_type end = D::process(reinterpret_cast<const typename D::type*>(data), length, dest, T());
+
+                return static_cast<size_t>(end - dest) * sizeof(*dest);
+        }
+
+        template <typename D, typename T> PUGI__FN size_t convert_buffer_output_generic(typename T::value_type dest, const char_t* data, size_t length, D, T, bool opt_swap)
+        {
+                PUGI__STATIC_ASSERT(sizeof(char_t) == sizeof(typename D::type));
+
+                typename T::value_type end = D::process(reinterpret_cast<const typename D::type*>(data), length, dest, T());
+
+                if (opt_swap)
+                {
+                        for (typename T::value_type i = dest; i != end; ++i)
+                                *i = endian_swap(*i);
+                }
+
+                return static_cast<size_t>(end - dest) * sizeof(*dest);
+        }
+
 #ifdef PUGIXML_WCHAR_MODE
-        size_t get_valid_length(const char_t* data, size_t length)
-        {
-                assert(length > 0);
-
-                // discard last character if it's the lead of a surrogate pair 
-                return (sizeof(wchar_t) == 2 && (unsigned)(static_cast<uint16_t>(data[length - 1]) - 0xD800) < 0x400) ? length - 1 : length;
-        }
-
-        size_t convert_buffer(char* result, const char_t* data, size_t length, xml_encoding encoding)
+        PUGI__FN size_t get_valid_length(const char_t* data, size_t length)
+        {
+                if (length < 1) return 0;
+
+                // discard last character if it's the lead of a surrogate pair
+                return (sizeof(wchar_t) == 2 && static_cast<unsigned int>(static_cast<uint16_t>(data[length - 1]) - 0xD800) < 0x400) ? length - 1 : length;
+        }
+
+        PUGI__FN size_t convert_buffer_output(char_t* r_char, uint8_t* r_u8, uint16_t* r_u16, uint32_t* r_u32, const char_t* data, size_t length, xml_encoding encoding)
         {
                 // only endian-swapping is required
                 if (need_endian_swap_utf(encoding, get_wchar_encoding()))
                 {
-                        convert_wchar_endian_swap(reinterpret_cast<char_t*>(result), data, length);
+                        convert_wchar_endian_swap(r_char, data, length);
 
                         return length * sizeof(char_t);
                 }
-        
+
                 // convert to utf8
                 if (encoding == encoding_utf8)
-                {
-                        uint8_t* dest = reinterpret_cast<uint8_t*>(result);
-
-                        uint8_t* end = sizeof(wchar_t) == 2 ?
-                                utf_decoder<utf8_writer>::decode_utf16_block(reinterpret_cast<const uint16_t*>(data), length, dest) :
-                                utf_decoder<utf8_writer>::decode_utf32_block(reinterpret_cast<const uint32_t*>(data), length, dest);
-
-                        return static_cast<size_t>(end - dest);
-                }
+                        return convert_buffer_output_generic(r_u8, data, length, wchar_decoder(), utf8_writer());
 
                 // convert to utf16
                 if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
                 {
-                        uint16_t* dest = reinterpret_cast<uint16_t*>(result);
-
-                        // convert to native utf16
-                        uint16_t* end = utf_decoder<utf16_writer>::decode_utf32_block(reinterpret_cast<const uint32_t*>(data), length, dest);
-
-                        // swap if necessary
                         xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
-                        if (native_encoding != encoding) convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
-
-                        return static_cast<size_t>(end - dest) * sizeof(uint16_t);
+                        return convert_buffer_output_generic(r_u16, data, length, wchar_decoder(), utf16_writer(), native_encoding != encoding);
                 }
 
@@ -2544 +3615 @@
                 if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
                 {
-                        uint32_t* dest = reinterpret_cast<uint32_t*>(result);
-
-                        // convert to native utf32
-                        uint32_t* end = utf_decoder<utf32_writer>::decode_utf16_block(reinterpret_cast<const uint16_t*>(data), length, dest);
-
-                        // swap if necessary
                         xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
-                        if (native_encoding != encoding) convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
-
-                        return static_cast<size_t>(end - dest) * sizeof(uint32_t);
-                }
-
-                assert(!"Invalid encoding");
+                        return convert_buffer_output_generic(r_u32, data, length, wchar_decoder(), utf32_writer(), native_encoding != encoding);
+                }
+
+                // convert to latin1
+                if (encoding == encoding_latin1)
+                        return convert_buffer_output_generic(r_u8, data, length, wchar_decoder(), latin1_writer());
+
+                assert(false && "Invalid encoding");
                 return 0;
         }
 #else
-        size_t get_valid_length(const char_t* data, size_t length)
-        {
-                assert(length > 4);
+        PUGI__FN size_t get_valid_length(const char_t* data, size_t length)
+        {
+                if (length < 5) return 0;
 
                 for (size_t i = 1; i <= 4; ++i)
@@ -2577 +3644 @@
         }
 
-        size_t convert_buffer(char* result, const char_t* data, size_t length, xml_encoding encoding)
+        PUGI__FN size_t convert_buffer_output(char_t* /* r_char */, uint8_t* r_u8, uint16_t* r_u16, uint32_t* r_u32, const char_t* data, size_t length, xml_encoding encoding)
         {
                 if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
                 {
-                        uint16_t* dest = reinterpret_cast<uint16_t*>(result);
-
-                        // convert to native utf16
-                        uint16_t* end = utf_decoder<utf16_writer>::decode_utf8_block(reinterpret_cast<const uint8_t*>(data), length, dest);
-
-                        // swap if necessary
                         xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
-                        if (native_encoding != encoding) convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
-
-                        return static_cast<size_t>(end - dest) * sizeof(uint16_t);
+                        return convert_buffer_output_generic(r_u16, data, length, utf8_decoder(), utf16_writer(), native_encoding != encoding);
                 }
 
                 if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
                 {
-                        uint32_t* dest = reinterpret_cast<uint32_t*>(result);
-
-                        // convert to native utf32
-                        uint32_t* end = utf_decoder<utf32_writer>::decode_utf8_block(reinterpret_cast<const uint8_t*>(data), length, dest);
-
-                        // swap if necessary
                         xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
-                        if (native_encoding != encoding) convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
-
-                        return static_cast<size_t>(end - dest) * sizeof(uint32_t);
-                }
-
-                assert(!"Invalid encoding");
+                        return convert_buffer_output_generic(r_u32, data, length, utf8_decoder(), utf32_writer(), native_encoding != encoding);
+                }
+
+                if (encoding == encoding_latin1)
+                        return convert_buffer_output_generic(r_u8, data, length, utf8_decoder(), latin1_writer());
+
+                assert(false && "Invalid encoding");
                 return 0;
         }
@@ -2620 +3674 @@
 
         public:
-                xml_buffered_writer(xml_writer& writer, xml_encoding user_encoding): writer(writer), bufsize(0), encoding(get_write_encoding(user_encoding))
-                {
-                }
-
-                ~xml_buffered_writer()
-                {
-                        flush();
-                }
-
-                void flush()
+                xml_buffered_writer(xml_writer& writer_, xml_encoding user_encoding): writer(writer_), bufsize(0), encoding(get_write_encoding(user_encoding))
+                {
+                        PUGI__STATIC_ASSERT(bufcapacity >= 8);
+                }
+
+                size_t flush()
                 {
                         flush(buffer, bufsize);
                         bufsize = 0;
+                        return 0;
                 }
 
@@ -2645 +3696 @@
                         {
                                 // convert chunk
-                                size_t result = convert_buffer(scratch, data, size, encoding);
+                                size_t result = convert_buffer_output(scratch.data_char, scratch.data_u8, scratch.data_u16, scratch.data_u32, data, size, encoding);
                                 assert(result <= sizeof(scratch));
 
                                 // write data
-                                writer.write(scratch, result);
-                        }
-                }
-
-                void write(const char_t* data, size_t length)
-                {
-                        if (bufsize + length > bufcapacity)
-                        {
-                                // flush the remaining buffer contents
-                                flush();
-
-                                // handle large chunks
-                                if (length > bufcapacity)
-                                {
-                                        if (encoding == get_write_native_encoding())
-                                        {
-                                                // fast path, can just write data chunk
-                                                writer.write(data, length * sizeof(char_t));
-                                                return;
-                                        }
-
-                                        // need to convert in suitable chunks
-                                        while (length > bufcapacity)
-                                        {
-                                                // get chunk size by selecting such number of characters that are guaranteed to fit into scratch buffer
-                                                // and form a complete codepoint sequence (i.e. discard start of last codepoint if necessary)
-                                                size_t chunk_size = get_valid_length(data, bufcapacity);
-
-                                                // convert chunk and write
-                                                flush(data, chunk_size);
-
-                                                // iterate
-                                                data += chunk_size;
-                                                length -= chunk_size;
-                                        }
-
-                                        // small tail is copied below
-                                        bufsize = 0;
-                                }
+                                writer.write(scratch.data_u8, result);
+                        }
+                }
+
+                void write_direct(const char_t* data, size_t length)
+                {
+                        // flush the remaining buffer contents
+                        flush();
+
+                        // handle large chunks
+                        if (length > bufcapacity)
+                        {
+                                if (encoding == get_write_native_encoding())
+                                {
+                                        // fast path, can just write data chunk
+                                        writer.write(data, length * sizeof(char_t));
+                                        return;
+                                }
+
+                                // need to convert in suitable chunks
+                                while (length > bufcapacity)
+                                {
+                                        // get chunk size by selecting such number of characters that are guaranteed to fit into scratch buffer
+                                        // and form a complete codepoint sequence (i.e. discard start of last codepoint if necessary)
+                                        size_t chunk_size = get_valid_length(data, bufcapacity);
+                                        assert(chunk_size);
+
+                                        // convert chunk and write
+                                        flush(data, chunk_size);
+
+                                        // iterate
+                                        data += chunk_size;
+                                        length -= chunk_size;
+                                }
+
+                                // small tail is copied below
+                                bufsize = 0;
                         }
 
@@ -2694 +3743 @@
                 }
 
-                void write(const char_t* data)
-                {
-                        write(data, strlength(data));
+                void write_buffer(const char_t* data, size_t length)
+                {
+                        size_t offset = bufsize;
+
+                        if (offset + length <= bufcapacity)
+                        {
+                                memcpy(buffer + offset, data, length * sizeof(char_t));
+                                bufsize = offset + length;
+                        }
+                        else
+                        {
+                                write_direct(data, length);
+                        }
+                }
+
+                void write_string(const char_t* data)
+                {
+                        // write the part of the string that fits in the buffer
+                        size_t offset = bufsize;
+
+                        while (*data && offset < bufcapacity)
+                                buffer[offset++] = *data++;
+
+                        // write the rest
+                        if (offset < bufcapacity)
+                        {
+                                bufsize = offset;
+                        }
+                        else
+                        {
+                                // backtrack a bit if we have split the codepoint
+                                size_t length = offset - bufsize;
+                                size_t extra = length - get_valid_length(data - length, length);
+
+                                bufsize = offset - extra;
+
+                                write_direct(data - extra, strlength(data) + extra);
+                        }
                 }
 
                 void write(char_t d0)
                 {
-                        if (bufsize + 1 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        bufsize += 1;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 1) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        bufsize = offset + 1;
                 }
 
                 void write(char_t d0, char_t d1)
                 {
-                        if (bufsize + 2 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        bufsize += 2;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 2) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        bufsize = offset + 2;
                 }
 
                 void write(char_t d0, char_t d1, char_t d2)
                 {
-                        if (bufsize + 3 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        buffer[bufsize + 2] = d2;
-                        bufsize += 3;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 3) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        buffer[offset + 2] = d2;
+                        bufsize = offset + 3;
                 }
 
                 void write(char_t d0, char_t d1, char_t d2, char_t d3)
                 {
-                        if (bufsize + 4 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        buffer[bufsize + 2] = d2;
-                        buffer[bufsize + 3] = d3;
-                        bufsize += 4;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 4) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        buffer[offset + 2] = d2;
+                        buffer[offset + 3] = d3;
+                        bufsize = offset + 4;
                 }
 
                 void write(char_t d0, char_t d1, char_t d2, char_t d3, char_t d4)
                 {
-                        if (bufsize + 5 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        buffer[bufsize + 2] = d2;
-                        buffer[bufsize + 3] = d3;
-                        buffer[bufsize + 4] = d4;
-                        bufsize += 5;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 5) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        buffer[offset + 2] = d2;
+                        buffer[offset + 3] = d3;
+                        buffer[offset + 4] = d4;
+                        bufsize = offset + 5;
                 }
 
                 void write(char_t d0, char_t d1, char_t d2, char_t d3, char_t d4, char_t d5)
                 {
-                        if (bufsize + 6 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        buffer[bufsize + 2] = d2;
-                        buffer[bufsize + 3] = d3;
-                        buffer[bufsize + 4] = d4;
-                        buffer[bufsize + 5] = d5;
-                        bufsize += 6;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 6) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        buffer[offset + 2] = d2;
+                        buffer[offset + 3] = d3;
+                        buffer[offset + 4] = d4;
+                        buffer[offset + 5] = d5;
+                        bufsize = offset + 6;
                 }
 
@@ -2765 +3855 @@
                 // utf16 maximum expansion: x2 (-> utf32)
                 // utf32 maximum expansion: x1
-                enum { bufcapacity = 2048 };
+                enum
+                {
+                        bufcapacitybytes =
+                        #ifdef PUGIXML_MEMORY_OUTPUT_STACK
+                                PUGIXML_MEMORY_OUTPUT_STACK
+                        #else
+                                10240
+                        #endif
+                        ,
+                        bufcapacity = bufcapacitybytes / (sizeof(char_t) + 4)
+                };
 
                 char_t buffer[bufcapacity];
-                char scratch[4 * bufcapacity];
+
+                union
+                {
+                        uint8_t data_u8[4 * bufcapacity];
+                        uint16_t data_u16[2 * bufcapacity];
+                        uint32_t data_u32[bufcapacity];
+                        char_t data_char[bufcapacity];
+                } scratch;
 
                 xml_writer& writer;
@@ -2775 +3882 @@
         };
 
-        void write_bom(xml_writer& writer, xml_encoding encoding)
-        {
-                switch (encoding)
-                {
-                case encoding_utf8:
-                        writer.write("\xef\xbb\xbf", 3);
-                        break;
-
-                case encoding_utf16_be:
-                        writer.write("\xfe\xff", 2);
-                        break;
-
-                case encoding_utf16_le:
-                        writer.write("\xff\xfe", 2);
-                        break;
-
-                case encoding_utf32_be:
-                        writer.write("\x00\x00\xfe\xff", 4);
-                        break;
-
-                case encoding_utf32_le:
-                        writer.write("\xff\xfe\x00\x00", 4);
-                        break;
-
-                default:
-                        assert(!"Invalid encoding");
-                }
-        }
-
-        void text_output_escaped(xml_buffered_writer& writer, const char_t* s, chartypex_t type)
+        PUGI__FN void text_output_escaped(xml_buffered_writer& writer, const char_t* s, chartypex_t type)
         {
                 while (*s)
                 {
                         const char_t* prev = s;
-                        
+
                         // While *s is a usual symbol
-                        while (!IS_CHARTYPEX(*s, type)) ++s;
-                
-                        writer.write(prev, static_cast<size_t>(s - prev));
+                        PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPEX(ss, type));
+
+                        writer.write_buffer(prev, static_cast<size_t>(s - prev));
 
                         switch (*s)
@@ -2845 +3923 @@
         }
 
-        void text_output_cdata(xml_buffered_writer& writer, const char_t* s)
+        PUGI__FN void text_output(xml_buffered_writer& writer, const char_t* s, chartypex_t type, unsigned int flags)
+        {
+                if (flags & format_no_escapes)
+                        writer.write_string(s);
+                else
+                        text_output_escaped(writer, s, type);
+        }
+
+        PUGI__FN void text_output_cdata(xml_buffered_writer& writer, const char_t* s)
         {
                 do
@@ -2860 +3946 @@
                         if (*s) s += 2;
 
-                        writer.write(prev, static_cast<size_t>(s - prev));
+                        writer.write_buffer(prev, static_cast<size_t>(s - prev));
 
                         writer.write(']', ']', '>');
@@ -2867 +3953 @@
         }
 
-        void node_output_attributes(xml_buffered_writer& writer, const xml_node& node)
+        PUGI__FN void text_output_indent(xml_buffered_writer& writer, const char_t* indent, size_t indent_length, unsigned int depth)
+        {
+                switch (indent_length)
+                {
+                case 1:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write(indent[0]);
+                        break;
+                }
+
+                case 2:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write(indent[0], indent[1]);
+                        break;
+                }
+
+                case 3:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write(indent[0], indent[1], indent[2]);
+                        break;
+                }
+
+                case 4:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write(indent[0], indent[1], indent[2], indent[3]);
+                        break;
+                }
+
+                default:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write_buffer(indent, indent_length);
+                }
+                }
+        }
+
+        PUGI__FN void node_output_comment(xml_buffered_writer& writer, const char_t* s)
+        {
+                writer.write('<', '!', '-', '-');
+
+                while (*s)
+                {
+                        const char_t* prev = s;
+
+                        // look for -\0 or -- sequence - we can't output it since -- is illegal in comment body
+                        while (*s && !(s[0] == '-' && (s[1] == '-' || s[1] == 0))) ++s;
+
+                        writer.write_buffer(prev, static_cast<size_t>(s - prev));
+
+                        if (*s)
+                        {
+                                assert(*s == '-');
+
+                                writer.write('-', ' ');
+                                ++s;
+                        }
+                }
+
+                writer.write('-', '-', '>');
+        }
+
+        PUGI__FN void node_output_pi_value(xml_buffered_writer& writer, const char_t* s)
+        {
+                while (*s)
+                {
+                        const char_t* prev = s;
+
+                        // look for ?> sequence - we can't output it since ?> terminates PI
+                        while (*s && !(s[0] == '?' && s[1] == '>')) ++s;
+
+                        writer.write_buffer(prev, static_cast<size_t>(s - prev));
+
+                        if (*s)
+                        {
+                                assert(s[0] == '?' && s[1] == '>');
+
+                                writer.write('?', ' ', '>');
+                                s += 2;
+                        }
+                }
+        }
+
+        PUGI__FN void node_output_attributes(xml_buffered_writer& writer, xml_node_struct* node, const char_t* indent, size_t indent_length, unsigned int flags, unsigned int depth)
         {
                 const char_t* default_name = PUGIXML_TEXT(":anonymous");
 
-                for (xml_attribute a = node.first_attribute(); a; a = a.next_attribute())
-                {
-                        writer.write(' ');
-                        writer.write(a.name()[0] ? a.name() : default_name);
+                for (xml_attribute_struct* a = node->first_attribute; a; a = a->next_attribute)
+                {
+                        if ((flags & (format_indent_attributes | format_raw)) == format_indent_attributes)
+                        {
+                                writer.write('\n');
+
+                                text_output_indent(writer, indent, indent_length, depth + 1);
+                        }
+                        else
+                        {
+                                writer.write(' ');
+                        }
+
+                        writer.write_string(a->name ? a->name + 0 : default_name);
                         writer.write('=', '"');
 
-                        text_output_escaped(writer, a.value(), ctx_special_attr);
+                        if (a->value)
+                                text_output(writer, a->value, ctx_special_attr, flags);
 
                         writer.write('"');
@@ -2883 +4066 @@
         }
 
-        void node_output(xml_buffered_writer& writer, const xml_node& node, const char_t* indent, unsigned int flags, unsigned int depth)
+        PUGI__FN bool node_output_start(xml_buffered_writer& writer, xml_node_struct* node, const char_t* indent, size_t indent_length, unsigned int flags, unsigned int depth)
         {
                 const char_t* default_name = PUGIXML_TEXT(":anonymous");
-
-                if ((flags & format_indent) != 0 && (flags & format_raw) == 0)
-                        for (unsigned int i = 0; i < depth; ++i) writer.write(indent);
-
-                switch (node.type())
-                {
-                case node_document:
-                {
-                        for (xml_node n = node.first_child(); n; n = n.next_sibling())
-                                node_output(writer, n, indent, flags, depth);
-                        break;
-                }
-                        
-                case node_element:
-                {
-                        const char_t* name = node.name()[0] ? node.name() : default_name;
-
-                        writer.write('<');
-                        writer.write(name);
-
-                        node_output_attributes(writer, node);
-
-                        if (flags & format_raw)
-                        {
-                                if (!node.first_child())
-                                        writer.write(' ', '/', '>');
+                const char_t* name = node->name ? node->name + 0 : default_name;
+
+                writer.write('<');
+                writer.write_string(name);
+
+                if (node->first_attribute)
+                        node_output_attributes(writer, node, indent, indent_length, flags, depth);
+
+                // element nodes can have value if parse_embed_pcdata was used
+                if (!node->value)
+                {
+                        if (!node->first_child)
+                        {
+                                if (flags & format_no_empty_element_tags)
+                                {
+                                        writer.write('>', '<', '/');
+                                        writer.write_string(name);
+                                        writer.write('>');
+
+                                        return false;
+                                }
                                 else
                                 {
-                                        writer.write('>');
-
-                                        for (xml_node n = node.first_child(); n; n = n.next_sibling())
-                                                node_output(writer, n, indent, flags, depth + 1);
-
-                                        writer.write('<', '/');
-                                        writer.write(name);
-                                        writer.write('>');
-                                }
-                        }
-                        else if (!node.first_child())
-                                writer.write(' ', '/', '>', '\n');
-                        else if (node.first_child() == node.last_child() && (node.first_child().type() == node_pcdata || node.first_child().type() == node_cdata))
+                                        if ((flags & format_raw) == 0)
+                                                writer.write(' ');
+
+                                        writer.write('/', '>');
+
+                                        return false;
+                                }
+                        }
+                        else
                         {
                                 writer.write('>');
 
-                if (node.first_child().type() == node_pcdata)
-                    text_output_escaped(writer, node.first_child().value(), ctx_special_pcdata);
-                else
-                    text_output_cdata(writer, node.first_child().value());
-
+                                return true;
+                        }
+                }
+                else
+                {
+                        writer.write('>');
+
+                        text_output(writer, node->value, ctx_special_pcdata, flags);
+
+                        if (!node->first_child)
+                        {
                                 writer.write('<', '/');
-                                writer.write(name);
-                                writer.write('>', '\n');
+                                writer.write_string(name);
+                                writer.write('>');
+
+                                return false;
                         }
                         else
                         {
-                                writer.write('>', '\n');
-                                
-                                for (xml_node n = node.first_child(); n; n = n.next_sibling())
-                                        node_output(writer, n, indent, flags, depth + 1);
-
-                                if ((flags & format_indent) != 0 && (flags & format_raw) == 0)
-                                        for (unsigned int i = 0; i < depth; ++i) writer.write(indent);
-                                
-                                writer.write('<', '/');
-                                writer.write(name);
-                                writer.write('>', '\n');
-                        }
-
-                        break;
-                }
-                
-                case node_pcdata:
-                        text_output_escaped(writer, node.value(), ctx_special_pcdata);
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                case node_cdata:
-                        text_output_cdata(writer, node.value());
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                case node_comment:
-                        writer.write('<', '!', '-', '-');
-                        writer.write(node.value());
-                        writer.write('-', '-', '>');
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                case node_pi:
-                case node_declaration:
-                        writer.write('<', '?');
-                        writer.write(node.name()[0] ? node.name() : default_name);
-
-                        if (node.type() == node_declaration)
-                        {
-                                node_output_attributes(writer, node);
-                        }
-                        else if (node.value()[0])
-                        {
-                                writer.write(' ');
-                                writer.write(node.value());
-                        }
-
-                        writer.write('?', '>');
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                case node_doctype:
-                        writer.write('<', '!', 'D', 'O', 'C');
-                        writer.write('T', 'Y', 'P', 'E');
-
-            if (node.value()[0])
-            {
-                writer.write(' ');
-                writer.write(node.value());
-            }
-
-            writer.write('>');
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                default:
-                        assert(!"Invalid node type");
-                }
-        }
-
-        inline bool has_declaration(const xml_node& node)
-        {
-                for (xml_node child = node.first_child(); child; child = child.next_sibling())
-                {
-                        xml_node_type type = child.type();
+                                return true;
+                        }
+                }
+        }
+
+        PUGI__FN void node_output_end(xml_buffered_writer& writer, xml_node_struct* node)
+        {
+                const char_t* default_name = PUGIXML_TEXT(":anonymous");
+                const char_t* name = node->name ? node->name + 0 : default_name;
+
+                writer.write('<', '/');
+                writer.write_string(name);
+                writer.write('>');
+        }
+
+        PUGI__FN void node_output_simple(xml_buffered_writer& writer, xml_node_struct* node, unsigned int flags)
+        {
+                const char_t* default_name = PUGIXML_TEXT(":anonymous");
+
+                switch (PUGI__NODETYPE(node))
+                {
+                        case node_pcdata:
+                                text_output(writer, node->value ? node->value + 0 : PUGIXML_TEXT(""), ctx_special_pcdata, flags);
+                                break;
+
+                        case node_cdata:
+                                text_output_cdata(writer, node->value ? node->value + 0 : PUGIXML_TEXT(""));
+                                break;
+
+                        case node_comment:
+                                node_output_comment(writer, node->value ? node->value + 0 : PUGIXML_TEXT(""));
+                                break;
+
+                        case node_pi:
+                                writer.write('<', '?');
+                                writer.write_string(node->name ? node->name + 0 : default_name);
+
+                                if (node->value)
+                                {
+                                        writer.write(' ');
+                                        node_output_pi_value(writer, node->value);
+                                }
+
+                                writer.write('?', '>');
+                                break;
+
+                        case node_declaration:
+                                writer.write('<', '?');
+                                writer.write_string(node->name ? node->name + 0 : default_name);
+                                node_output_attributes(writer, node, PUGIXML_TEXT(""), 0, flags | format_raw, 0);
+                                writer.write('?', '>');
+                                break;
+
+                        case node_doctype:
+                                writer.write('<', '!', 'D', 'O', 'C');
+                                writer.write('T', 'Y', 'P', 'E');
+
+                                if (node->value)
+                                {
+                                        writer.write(' ');
+                                        writer.write_string(node->value);
+                                }
+
+                                writer.write('>');
+                                break;
+
+                        default:
+                                assert(false && "Invalid node type");
+                }
+        }
+
+        enum indent_flags_t
+        {
+                indent_newline = 1,
+                indent_indent = 2
+        };
+
+        PUGI__FN void node_output(xml_buffered_writer& writer, xml_node_struct* root, const char_t* indent, unsigned int flags, unsigned int depth)
+        {
+                size_t indent_length = ((flags & (format_indent | format_indent_attributes)) && (flags & format_raw) == 0) ? strlength(indent) : 0;
+                unsigned int indent_flags = indent_indent;
+
+                xml_node_struct* node = root;
+
+                do
+                {
+                        assert(node);
+
+                        // begin writing current node
+                        if (PUGI__NODETYPE(node) == node_pcdata || PUGI__NODETYPE(node) == node_cdata)
+                        {
+                                node_output_simple(writer, node, flags);
+
+                                indent_flags = 0;
+                        }
+                        else
+                        {
+                                if ((indent_flags & indent_newline) && (flags & format_raw) == 0)
+                                        writer.write('\n');
+
+                                if ((indent_flags & indent_indent) && indent_length)
+                                        text_output_indent(writer, indent, indent_length, depth);
+
+                                if (PUGI__NODETYPE(node) == node_element)
+                                {
+                                        indent_flags = indent_newline | indent_indent;
+
+                                        if (node_output_start(writer, node, indent, indent_length, flags, depth))
+                                        {
+                                                // element nodes can have value if parse_embed_pcdata was used
+                                                if (node->value)
+                                                        indent_flags = 0;
+
+                                                node = node->first_child;
+                                                depth++;
+                                                continue;
+                                        }
+                                }
+                                else if (PUGI__NODETYPE(node) == node_document)
+                                {
+                                        indent_flags = indent_indent;
+
+                                        if (node->first_child)
+                                        {
+                                                node = node->first_child;
+                                                continue;
+                                        }
+                                }
+                                else
+                                {
+                                        node_output_simple(writer, node, flags);
+
+                                        indent_flags = indent_newline | indent_indent;
+                                }
+                        }
+
+                        // continue to the next node
+                        while (node != root)
+                        {
+                                if (node->next_sibling)
+                                {
+                                        node = node->next_sibling;
+                                        break;
+                                }
+
+                                node = node->parent;
+
+                                // write closing node
+                                if (PUGI__NODETYPE(node) == node_element)
+                                {
+                                        depth--;
+
+                                        if ((indent_flags & indent_newline) && (flags & format_raw) == 0)
+                                                writer.write('\n');
+
+                                        if ((indent_flags & indent_indent) && indent_length)
+                                                text_output_indent(writer, indent, indent_length, depth);
+
+                                        node_output_end(writer, node);
+
+                                        indent_flags = indent_newline | indent_indent;
+                                }
+                        }
+                }
+                while (node != root);
+
+                if ((indent_flags & indent_newline) && (flags & format_raw) == 0)
+                        writer.write('\n');
+        }
+
+        PUGI__FN bool has_declaration(xml_node_struct* node)
+        {
+                for (xml_node_struct* child = node->first_child; child; child = child->next_sibling)
+                {
+                        xml_node_type type = PUGI__NODETYPE(child);
 
                         if (type == node_declaration) return true;
@@ -3025 +4306 @@
         }
 
-        inline bool allow_insert_child(xml_node_type parent, xml_node_type child)
+        PUGI__FN bool is_attribute_of(xml_attribute_struct* attr, xml_node_struct* node)
+        {
+                for (xml_attribute_struct* a = node->first_attribute; a; a = a->next_attribute)
+                        if (a == attr)
+                                return true;
+
+                return false;
+        }
+
+        PUGI__FN bool allow_insert_attribute(xml_node_type parent)
+        {
+                return parent == node_element || parent == node_declaration;
+        }
+
+        PUGI__FN bool allow_insert_child(xml_node_type parent, xml_node_type child)
         {
                 if (parent != node_document && parent != node_element) return false;
@@ -3034 +4329 @@
         }
 
-        void recursive_copy_skip(xml_node& dest, const xml_node& source, const xml_node& skip)
-        {
-                assert(dest.type() == source.type());
-
-                switch (source.type())
-                {
-                case node_element:
-                {
-                        dest.set_name(source.name());
-
-                        for (xml_attribute a = source.first_attribute(); a; a = a.next_attribute())
-                                dest.append_attribute(a.name()).set_value(a.value());
-
-                        for (xml_node c = source.first_child(); c; c = c.next_sibling())
-                        {
-                                if (c == skip) continue;
-
-                                xml_node cc = dest.append_child(c.type());
-                                assert(cc);
-
-                                recursive_copy_skip(cc, c, skip);
-                        }
-
-                        break;
-                }
-
-                case node_pcdata:
-                case node_cdata:
-                case node_comment:
-        case node_doctype:
-                        dest.set_value(source.value());
-                        break;
-
-                case node_pi:
-                        dest.set_name(source.name());
-                        dest.set_value(source.value());
-                        break;
-
-                case node_declaration:
-                {
-                        dest.set_name(source.name());
-
-                        for (xml_attribute a = source.first_attribute(); a; a = a.next_attribute())
-                                dest.append_attribute(a.name()).set_value(a.value());
-
-                        break;
-                }
-
-                default:
-                        assert(!"Invalid node type");
-                }
+        PUGI__FN bool allow_move(xml_node parent, xml_node child)
+        {
+                // check that child can be a child of parent
+                if (!allow_insert_child(parent.type(), child.type()))
+                        return false;
+
+                // check that node is not moved between documents
+                if (parent.root() != child.root())
+                        return false;
+
+                // check that new parent is not in the child subtree
+                xml_node cur = parent;
+
+                while (cur)
+                {
+                        if (cur == child)
+                                return false;
+
+                        cur = cur.parent();
+                }
+
+                return true;
+        }
+
+        template <typename String, typename Header>
+        PUGI__FN void node_copy_string(String& dest, Header& header, uintptr_t header_mask, char_t* source, Header& source_header, xml_allocator* alloc)
+        {
+                assert(!dest && (header & header_mask) == 0);
+
+                if (source)
+                {
+                        if (alloc && (source_header & header_mask) == 0)
+                        {
+                                dest = source;
+
+                                // since strcpy_insitu can reuse document buffer memory we need to mark both source and dest as shared
+                                header |= xml_memory_page_contents_shared_mask;
+                                source_header |= xml_memory_page_contents_shared_mask;
+                        }
+                        else
+                                strcpy_insitu(dest, header, header_mask, source, strlength(source));
+                }
+        }
+
+        PUGI__FN void node_copy_contents(xml_node_struct* dn, xml_node_struct* sn, xml_allocator* shared_alloc)
+        {
+                node_copy_string(dn->name, dn->header, xml_memory_page_name_allocated_mask, sn->name, sn->header, shared_alloc);
+                node_copy_string(dn->value, dn->header, xml_memory_page_value_allocated_mask, sn->value, sn->header, shared_alloc);
+
+                for (xml_attribute_struct* sa = sn->first_attribute; sa; sa = sa->next_attribute)
+                {
+                        xml_attribute_struct* da = append_new_attribute(dn, get_allocator(dn));
+
+                        if (da)
+                        {
+                                node_copy_string(da->name, da->header, xml_memory_page_name_allocated_mask, sa->name, sa->header, shared_alloc);
+                                node_copy_string(da->value, da->header, xml_memory_page_value_allocated_mask, sa->value, sa->header, shared_alloc);
+                        }
+                }
+        }
+
+        PUGI__FN void node_copy_tree(xml_node_struct* dn, xml_node_struct* sn)
+        {
+                xml_allocator& alloc = get_allocator(dn);
+                xml_allocator* shared_alloc = (&alloc == &get_allocator(sn)) ? &alloc : 0;
+
+                node_copy_contents(dn, sn, shared_alloc);
+
+                xml_node_struct* dit = dn;
+                xml_node_struct* sit = sn->first_child;
+
+                while (sit && sit != sn)
+                {
+                        if (sit != dn)
+                        {
+                                xml_node_struct* copy = append_new_node(dit, alloc, PUGI__NODETYPE(sit));
+
+                                if (copy)
+                                {
+                                        node_copy_contents(copy, sit, shared_alloc);
+
+                                        if (sit->first_child)
+                                        {
+                                                dit = copy;
+                                                sit = sit->first_child;
+                                                continue;
+                                        }
+                                }
+                        }
+
+                        // continue to the next node
+                        do
+                        {
+                                if (sit->next_sibling)
+                                {
+                                        sit = sit->next_sibling;
+                                        break;
+                                }
+
+                                sit = sit->parent;
+                                dit = dit->parent;
+                        }
+                        while (sit != sn);
+                }
+        }
+
+        PUGI__FN void node_copy_attribute(xml_attribute_struct* da, xml_attribute_struct* sa)
+        {
+                xml_allocator& alloc = get_allocator(da);
+                xml_allocator* shared_alloc = (&alloc == &get_allocator(sa)) ? &alloc : 0;
+
+                node_copy_string(da->name, da->header, xml_memory_page_name_allocated_mask, sa->name, sa->header, shared_alloc);
+                node_copy_string(da->value, da->header, xml_memory_page_value_allocated_mask, sa->value, sa->header, shared_alloc);
+        }
+
+        inline bool is_text_node(xml_node_struct* node)
+        {
+                xml_node_type type = PUGI__NODETYPE(node);
+
+                return type == node_pcdata || type == node_cdata;
+        }
+
+        // get value with conversion functions
+        template <typename U> U string_to_integer(const char_t* value, U minneg, U maxpos)
+        {
+                U result = 0;
+                const char_t* s = value;
+
+                while (PUGI__IS_CHARTYPE(*s, ct_space))
+                        s++;
+
+                bool negative = (*s == '-');
+
+                s += (*s == '+' || *s == '-');
+
+                bool overflow = false;
+
+                if (s[0] == '0' && (s[1] | ' ') == 'x')
+                {
+                        s += 2;
+
+                        // since overflow detection relies on length of the sequence skip leading zeros
+                        while (*s == '0')
+                                s++;
+
+                        const char_t* start = s;
+
+                        for (;;)
+                        {
+                                if (static_cast<unsigned>(*s - '0') < 10)
+                                        result = result * 16 + (*s - '0');
+                                else if (static_cast<unsigned>((*s | ' ') - 'a') < 6)
+                                        result = result * 16 + ((*s | ' ') - 'a' + 10);
+                                else
+                                        break;
+
+                                s++;
+                        }
+
+                        size_t digits = static_cast<size_t>(s - start);
+
+                        overflow = digits > sizeof(U) * 2;
+                }
+                else
+                {
+                        // since overflow detection relies on length of the sequence skip leading zeros
+                        while (*s == '0')
+                                s++;
+
+                        const char_t* start = s;
+
+                        for (;;)
+                        {
+                                if (static_cast<unsigned>(*s - '0') < 10)
+                                        result = result * 10 + (*s - '0');
+                                else
+                                        break;
+
+                                s++;
+                        }
+
+                        size_t digits = static_cast<size_t>(s - start);
+
+                        PUGI__STATIC_ASSERT(sizeof(U) == 8 || sizeof(U) == 4 || sizeof(U) == 2);
+
+                        const size_t max_digits10 = sizeof(U) == 8 ? 20 : sizeof(U) == 4 ? 10 : 5;
+                        const char_t max_lead = sizeof(U) == 8 ? '1' : sizeof(U) == 4 ? '4' : '6';
+                        const size_t high_bit = sizeof(U) * 8 - 1;
+
+                        overflow = digits >= max_digits10 && !(digits == max_digits10 && (*start < max_lead || (*start == max_lead && result >> high_bit)));
+                }
+
+                if (negative)
+                        return (overflow || result > minneg) ? 0 - minneg : 0 - result;
+                else
+                        return (overflow || result > maxpos) ? maxpos : result;
+        }
+
+        PUGI__FN int get_value_int(const char_t* value)
+        {
+                return string_to_integer<unsigned int>(value, 0 - static_cast<unsigned int>(INT_MIN), INT_MAX);
+        }
+
+        PUGI__FN unsigned int get_value_uint(const char_t* value)
+        {
+                return string_to_integer<unsigned int>(value, 0, UINT_MAX);
+        }
+
+        PUGI__FN double get_value_double(const char_t* value)
+        {
+        #ifdef PUGIXML_WCHAR_MODE
+                return wcstod(value, 0);
+        #else
+                return strtod(value, 0);
+        #endif
+        }
+
+        PUGI__FN float get_value_float(const char_t* value)
+        {
+        #ifdef PUGIXML_WCHAR_MODE
+                return static_cast<float>(wcstod(value, 0));
+        #else
+                return static_cast<float>(strtod(value, 0));
+        #endif
+        }
+
+        PUGI__FN bool get_value_bool(const char_t* value)
+        {
+                // only look at first char
+                char_t first = *value;
+
+                // 1*, t* (true), T* (True), y* (yes), Y* (YES)
+                return (first == '1' || first == 't' || first == 'T' || first == 'y' || first == 'Y');
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN long long get_value_llong(const char_t* value)
+        {
+                return string_to_integer<unsigned long long>(value, 0 - static_cast<unsigned long long>(LLONG_MIN), LLONG_MAX);
+        }
+
+        PUGI__FN unsigned long long get_value_ullong(const char_t* value)
+        {
+                return string_to_integer<unsigned long long>(value, 0, ULLONG_MAX);
+        }
+#endif
+
+        template <typename U> PUGI__FN char_t* integer_to_string(char_t* begin, char_t* end, U value, bool negative)
+        {
+                char_t* result = end - 1;
+                U rest = negative ? 0 - value : value;
+
+                do
+                {
+                        *result-- = static_cast<char_t>('0' + (rest % 10));
+                        rest /= 10;
+                }
+                while (rest);
+
+                assert(result >= begin);
+                (void)begin;
+
+                *result = '-';
+
+                return result + !negative;
+        }
+
+        // set value with conversion functions
+        template <typename String, typename Header>
+        PUGI__FN bool set_value_ascii(String& dest, Header& header, uintptr_t header_mask, char* buf)
+        {
+        #ifdef PUGIXML_WCHAR_MODE
+                char_t wbuf[128];
+                assert(strlen(buf) < sizeof(wbuf) / sizeof(wbuf[0]));
+
+                size_t offset = 0;
+                for (; buf[offset]; ++offset) wbuf[offset] = buf[offset];
+
+                return strcpy_insitu(dest, header, header_mask, wbuf, offset);
+        #else
+                return strcpy_insitu(dest, header, header_mask, buf, strlen(buf));
+        #endif
+        }
+
+        template <typename U, typename String, typename Header>
+        PUGI__FN bool set_value_integer(String& dest, Header& header, uintptr_t header_mask, U value, bool negative)
+        {
+                char_t buf[64];
+                char_t* end = buf + sizeof(buf) / sizeof(buf[0]);
+                char_t* begin = integer_to_string(buf, end, value, negative);
+
+                return strcpy_insitu(dest, header, header_mask, begin, end - begin);
+        }
+
+        template <typename String, typename Header>
+        PUGI__FN bool set_value_convert(String& dest, Header& header, uintptr_t header_mask, float value)
+        {
+                char buf[128];
+                sprintf(buf, "%.9g", value);
+
+                return set_value_ascii(dest, header, header_mask, buf);
+        }
+
+        template <typename String, typename Header>
+        PUGI__FN bool set_value_convert(String& dest, Header& header, uintptr_t header_mask, double value)
+        {
+                char buf[128];
+                sprintf(buf, "%.17g", value);
+
+                return set_value_ascii(dest, header, header_mask, buf);
+        }
+
+        template <typename String, typename Header>
+        PUGI__FN bool set_value_bool(String& dest, Header& header, uintptr_t header_mask, bool value)
+        {
+                return strcpy_insitu(dest, header, header_mask, value ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"), value ? 4 : 5);
+        }
+
+        PUGI__FN xml_parse_result load_buffer_impl(xml_document_struct* doc, xml_node_struct* root, void* contents, size_t size, unsigned int options, xml_encoding encoding, bool is_mutable, bool own, char_t** out_buffer)
+        {
+                // check input buffer
+                if (!contents && size) return make_parse_result(status_io_error);
+
+                // get actual encoding
+                xml_encoding buffer_encoding = impl::get_buffer_encoding(encoding, contents, size);
+
+                // get private buffer
+                char_t* buffer = 0;
+                size_t length = 0;
+
+                if (!impl::convert_buffer(buffer, length, buffer_encoding, contents, size, is_mutable)) return impl::make_parse_result(status_out_of_memory);
+
+                // delete original buffer if we performed a conversion
+                if (own && buffer != contents && contents) impl::xml_memory::deallocate(contents);
+
+                // grab onto buffer if it's our buffer, user is responsible for deallocating contents himself
+                if (own || buffer != contents) *out_buffer = buffer;
+
+                // store buffer for offset_debug
+                doc->buffer = buffer;
+
+                // parse
+                xml_parse_result res = impl::xml_parser::parse(buffer, length, doc, root, options);
+
+                // remember encoding
+                res.encoding = buffer_encoding;
+
+                return res;
         }
 
         // we need to get length of entire file to load it in memory; the only (relatively) sane way to do it is via seek/tell trick
-        xml_parse_status get_file_size(FILE* file, size_t& out_result)
-        {
-        #if defined(_MSC_VER) && _MSC_VER >= 1400
+        PUGI__FN xml_parse_status get_file_size(FILE* file, size_t& out_result)
+        {
+        #if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 && !defined(_WIN32_WCE)
                 // there are 64-bit versions of fseek/ftell, let's use them
                 typedef __int64 length_type;
@@ -3097 +4689 @@
                 length_type length = _ftelli64(file);
                 _fseeki64(file, 0, SEEK_SET);
-        #elif defined(__MINGW32__) && !defined(__NO_MINGW_LFS) && !defined(__STRICT_ANSI__)
+        #elif defined(__MINGW32__) && !defined(__NO_MINGW_LFS) && (!defined(__STRICT_ANSI__) || defined(__MINGW64_VERSION_MAJOR))
                 // there are 64-bit versions of fseek/ftell, let's use them
                 typedef off64_t length_type;
@@ -3115 +4707 @@
                 // check for I/O errors
                 if (length < 0) return status_io_error;
-                
+
                 // check for overflow
                 size_t result = static_cast<size_t>(length);
@@ -3127 +4719 @@
         }
 
-        xml_parse_result load_file_impl(xml_document& doc, FILE* file, unsigned int options, xml_encoding encoding)
+        // This function assumes that buffer has extra sizeof(char_t) writable bytes after size
+        PUGI__FN size_t zero_terminate_buffer(void* buffer, size_t size, xml_encoding encoding)
+        {
+                // We only need to zero-terminate if encoding conversion does not do it for us
+        #ifdef PUGIXML_WCHAR_MODE
+                xml_encoding wchar_encoding = get_wchar_encoding();
+
+                if (encoding == wchar_encoding || need_endian_swap_utf(encoding, wchar_encoding))
+                {
+                        size_t length = size / sizeof(char_t);
+
+                        static_cast<char_t*>(buffer)[length] = 0;
+                        return (length + 1) * sizeof(char_t);
+                }
+        #else
+                if (encoding == encoding_utf8)
+                {
+                        static_cast<char*>(buffer)[size] = 0;
+                        return size + 1;
+                }
+        #endif
+
+                return size;
+        }
+
+        PUGI__FN xml_parse_result load_file_impl(xml_document_struct* doc, FILE* file, unsigned int options, xml_encoding encoding, char_t** out_buffer)
         {
                 if (!file) return make_parse_result(status_file_not_found);
@@ -3134 +4751 @@
                 size_t size = 0;
                 xml_parse_status size_status = get_file_size(file, size);
-
-                if (size_status != status_ok)
-                {
-                        fclose(file);
-                        return make_parse_result(size_status);
-                }
-                
+                if (size_status != status_ok) return make_parse_result(size_status);
+
+                size_t max_suffix_size = sizeof(char_t);
+
                 // allocate buffer for the whole file
-                char* contents = static_cast<char*>(global_allocate(size > 0 ? size : 1));
-
-                if (!contents)
-                {
-                        fclose(file);
-                        return make_parse_result(status_out_of_memory);
-                }
+                char* contents = static_cast<char*>(xml_memory::allocate(size + max_suffix_size));
+                if (!contents) return make_parse_result(status_out_of_memory);
 
                 // read file in memory
                 size_t read_size = fread(contents, 1, size, file);
+
+                if (read_size != size)
+                {
+                        xml_memory::deallocate(contents);
+                        return make_parse_result(status_io_error);
+                }
+
+                xml_encoding real_encoding = get_buffer_encoding(encoding, contents, size);
+
+                return load_buffer_impl(doc, doc, contents, zero_terminate_buffer(contents, size, real_encoding), options, real_encoding, true, true, out_buffer);
+        }
+
+        PUGI__FN void close_file(FILE* file)
+        {
                 fclose(file);
-
-                if (read_size != size)
-                {
-                        global_deallocate(contents);
-                        return make_parse_result(status_io_error);
-                }
-                
-                return doc.load_buffer_inplace_own(contents, size, options, encoding);
         }
 
 #ifndef PUGIXML_NO_STL
-        template <typename T> xml_parse_result load_stream_impl(xml_document& doc, std::basic_istream<T>& stream, unsigned int options, xml_encoding encoding)
+        template <typename T> struct xml_stream_chunk
+        {
+                static xml_stream_chunk* create()
+                {
+                        void* memory = xml_memory::allocate(sizeof(xml_stream_chunk));
+                        if (!memory) return 0;
+
+                        return new (memory) xml_stream_chunk();
+                }
+
+                static void destroy(xml_stream_chunk* chunk)
+                {
+                        // free chunk chain
+                        while (chunk)
+                        {
+                                xml_stream_chunk* next_ = chunk->next;
+
+                                xml_memory::deallocate(chunk);
+
+                                chunk = next_;
+                        }
+                }
+
+                xml_stream_chunk(): next(0), size(0)
+                {
+                }
+
+                xml_stream_chunk* next;
+                size_t size;
+
+                T data[xml_memory_page_size / sizeof(T)];
+        };
+
+        template <typename T> PUGI__FN xml_parse_status load_stream_data_noseek(std::basic_istream<T>& stream, void** out_buffer, size_t* out_size)
+        {
+                auto_deleter<xml_stream_chunk<T> > chunks(0, xml_stream_chunk<T>::destroy);
+
+                // read file to a chunk list
+                size_t total = 0;
+                xml_stream_chunk<T>* last = 0;
+
+                while (!stream.eof())
+                {
+                        // allocate new chunk
+                        xml_stream_chunk<T>* chunk = xml_stream_chunk<T>::create();
+                        if (!chunk) return status_out_of_memory;
+
+                        // append chunk to list
+                        if (last) last = last->next = chunk;
+                        else chunks.data = last = chunk;
+
+                        // read data to chunk
+                        stream.read(chunk->data, static_cast<std::streamsize>(sizeof(chunk->data) / sizeof(T)));
+                        chunk->size = static_cast<size_t>(stream.gcount()) * sizeof(T);
+
+                        // read may set failbit | eofbit in case gcount() is less than read length, so check for other I/O errors
+                        if (stream.bad() || (!stream.eof() && stream.fail())) return status_io_error;
+
+                        // guard against huge files (chunk size is small enough to make this overflow check work)
+                        if (total + chunk->size < total) return status_out_of_memory;
+                        total += chunk->size;
+                }
+
+                size_t max_suffix_size = sizeof(char_t);
+
+                // copy chunk list to a contiguous buffer
+                char* buffer = static_cast<char*>(xml_memory::allocate(total + max_suffix_size));
+                if (!buffer) return status_out_of_memory;
+
+                char* write = buffer;
+
+                for (xml_stream_chunk<T>* chunk = chunks.data; chunk; chunk = chunk->next)
+                {
+                        assert(write + chunk->size <= buffer + total);
+                        memcpy(write, chunk->data, chunk->size);
+                        write += chunk->size;
+                }
+
+                assert(write == buffer + total);
+
+                // return buffer
+                *out_buffer = buffer;
+                *out_size = total;
+
+                return status_ok;
+        }
+
+        template <typename T> PUGI__FN xml_parse_status load_stream_data_seek(std::basic_istream<T>& stream, void** out_buffer, size_t* out_size)
         {
                 // get length of remaining data in stream
@@ -3172 +4874 @@
                 stream.seekg(pos);
 
-                if (stream.fail() || pos < 0) return make_parse_result(status_io_error);
+                if (stream.fail() || pos < 0) return status_io_error;
 
                 // guard against huge files
                 size_t read_length = static_cast<size_t>(length);
 
-                if (static_cast<std::streamsize>(read_length) != length || length < 0) return make_parse_result(status_out_of_memory);
+                if (static_cast<std::streamsize>(read_length) != length || length < 0) return status_out_of_memory;
+
+                size_t max_suffix_size = sizeof(char_t);
 
                 // read stream data into memory (guard against stream exceptions with buffer holder)
-                buffer_holder buffer(global_allocate((read_length > 0 ? read_length : 1) * sizeof(T)), global_deallocate);
-                if (!buffer.data) return make_parse_result(status_out_of_memory);
+                auto_deleter<void> buffer(xml_memory::allocate(read_length * sizeof(T) + max_suffix_size), xml_memory::deallocate);
+                if (!buffer.data) return status_out_of_memory;
 
                 stream.read(static_cast<T*>(buffer.data), static_cast<std::streamsize>(read_length));
 
                 // read may set failbit | eofbit in case gcount() is less than read_length (i.e. line ending conversion), so check for other I/O errors
-                if (stream.bad()) return make_parse_result(status_io_error);
-
-                // load data from buffer
+                if (stream.bad() || (!stream.eof() && stream.fail())) return status_io_error;
+
+                // return buffer
                 size_t actual_length = static_cast<size_t>(stream.gcount());
                 assert(actual_length <= read_length);
 
-                return doc.load_buffer_inplace_own(buffer.release(), actual_length * sizeof(T), options, encoding);
+                *out_buffer = buffer.release();
+                *out_size = actual_length * sizeof(T);
+
+                return status_ok;
+        }
+
+        template <typename T> PUGI__FN xml_parse_result load_stream_impl(xml_document_struct* doc, std::basic_istream<T>& stream, unsigned int options, xml_encoding encoding, char_t** out_buffer)
+        {
+                void* buffer = 0;
+                size_t size = 0;
+                xml_parse_status status = status_ok;
+
+                // if stream has an error bit set, bail out (otherwise tellg() can fail and we'll clear error bits)
+                if (stream.fail()) return make_parse_result(status_io_error);
+
+                // load stream to memory (using seek-based implementation if possible, since it's faster and takes less memory)
+                if (stream.tellg() < 0)
+                {
+                        stream.clear(); // clear error flags that could be set by a failing tellg
+                        status = load_stream_data_noseek(stream, &buffer, &size);
+                }
+                else
+                        status = load_stream_data_seek(stream, &buffer, &size);
+
+                if (status != status_ok) return make_parse_result(status);
+
+                xml_encoding real_encoding = get_buffer_encoding(encoding, buffer, size);
+
+                return load_buffer_impl(doc, doc, buffer, zero_terminate_buffer(buffer, size, real_encoding), options, real_encoding, true, true, out_buffer);
         }
 #endif
 
-#if defined(_MSC_VER) || defined(__BORLANDC__) || defined(__MINGW32__)
-        FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
+#if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__) || (defined(__MINGW32__) && (!defined(__STRICT_ANSI__) || defined(__MINGW64_VERSION_MAJOR)))
+        PUGI__FN FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
         {
                 return _wfopen(path, mode);
         }
 #else
-        char* convert_path_heap(const wchar_t* str)
+        PUGI__FN char* convert_path_heap(const wchar_t* str)
         {
                 assert(str);
 
                 // first pass: get length in utf8 characters
-                size_t length = wcslen(str);
-        size_t size = as_utf8_begin(str, length);
+                size_t length = strlength_wide(str);
+                size_t size = as_utf8_begin(str, length);
 
                 // allocate resulting string
-                char* result = static_cast<char*>(global_allocate(size + 1));
+                char* result = static_cast<char*>(xml_memory::allocate(size + 1));
                 if (!result) return 0;
 
                 // second pass: convert to utf8
-        as_utf8_end(result, size, str, length);
-
-                return result;
-        }
-
-        FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
+                as_utf8_end(result, size, str, length);
+
+                // zero-terminate
+                result[size] = 0;
+
+                return result;
+        }
+
+        PUGI__FN FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
         {
                 // there is no standard function to open wide paths, so our best bet is to try utf8 path
@@ -3234 +4969 @@
 
                 // free dummy buffer
-                global_deallocate(path_utf8);
+                xml_memory::deallocate(path_utf8);
 
                 return result;
         }
 #endif
-}
+
+        PUGI__FN bool save_file_impl(const xml_document& doc, FILE* file, const char_t* indent, unsigned int flags, xml_encoding encoding)
+        {
+                if (!file) return false;
+
+                xml_writer_file writer(file);
+                doc.save(writer, indent, flags, encoding);
+
+                return ferror(file) == 0;
+        }
+
+        struct name_null_sentry
+        {
+                xml_node_struct* node;
+                char_t* name;
+
+                name_null_sentry(xml_node_struct* node_): node(node_), name(node_->name)
+                {
+                        node->name = 0;
+                }
+
+                ~name_null_sentry()
+                {
+                        node->name = name;
+                }
+        };
+PUGI__NS_END
 
 namespace pugi
 {
-        xml_writer_file::xml_writer_file(void* file): file(file)
-        {
-        }
-
-        void xml_writer_file::write(const void* data, size_t size)
-        {
-                fwrite(data, size, 1, static_cast<FILE*>(file));
+        PUGI__FN xml_writer_file::xml_writer_file(void* file_): file(file_)
+        {
+        }
+
+        PUGI__FN void xml_writer_file::write(const void* data, size_t size)
+        {
+                size_t result = fwrite(data, 1, size, static_cast<FILE*>(file));
+                (void)!result; // unfortunately we can't do proper error handling here
         }
 
 #ifndef PUGIXML_NO_STL
-        xml_writer_stream::xml_writer_stream(std::basic_ostream<char, std::char_traits<char> >& stream): narrow_stream(&stream), wide_stream(0)
-        {
-        }
-
-        xml_writer_stream::xml_writer_stream(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream): narrow_stream(0), wide_stream(&stream)
-        {
-        }
-
-        void xml_writer_stream::write(const void* data, size_t size)
+        PUGI__FN xml_writer_stream::xml_writer_stream(std::basic_ostream<char, std::char_traits<char> >& stream): narrow_stream(&stream), wide_stream(0)
+        {
+        }
+
+        PUGI__FN xml_writer_stream::xml_writer_stream(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream): narrow_stream(0), wide_stream(&stream)
+        {
+        }
+
+        PUGI__FN void xml_writer_stream::write(const void* data, size_t size)
         {
                 if (narrow_stream)
@@ -3278 +5040 @@
 #endif
 
-        xml_tree_walker::xml_tree_walker(): _depth(0)
-        {
-        }
-        
-        xml_tree_walker::~xml_tree_walker()
-        {
-        }
-
-        int xml_tree_walker::depth() const
+        PUGI__FN xml_tree_walker::xml_tree_walker(): _depth(0)
+        {
+        }
+
+        PUGI__FN xml_tree_walker::~xml_tree_walker()
+        {
+        }
+
+        PUGI__FN int xml_tree_walker::depth() const
         {
                 return _depth;
         }
 
-        bool xml_tree_walker::begin(xml_node&)
+        PUGI__FN bool xml_tree_walker::begin(xml_node&)
         {
                 return true;
         }
 
-        bool xml_tree_walker::end(xml_node&)
+        PUGI__FN bool xml_tree_walker::end(xml_node&)
         {
                 return true;
         }
 
-        xml_attribute::xml_attribute(): _attr(0)
-        {
-        }
-
-        xml_attribute::xml_attribute(xml_attribute_struct* attr): _attr(attr)
-        {
-        }
-
-        xml_attribute::operator xml_attribute::unspecified_bool_type() const
-        {
-        return _attr ? &xml_attribute::_attr : 0;
-        }
-
-        bool xml_attribute::operator!() const
-        {
-                return !_attr;
-        }
-
-        bool xml_attribute::operator==(const xml_attribute& r) const
+        PUGI__FN xml_attribute::xml_attribute(): _attr(0)
+        {
+        }
+
+        PUGI__FN xml_attribute::xml_attribute(xml_attribute_struct* attr): _attr(attr)
+        {
+        }
+
+        PUGI__FN static void unspecified_bool_xml_attribute(xml_attribute***)
+        {
+        }
+
+        PUGI__FN xml_attribute::operator xml_attribute::unspecified_bool_type() const
+        {
+                return _attr ? unspecified_bool_xml_attribute : 0;
+        }
+
+        PUGI__FN bool xml_attribute::operator!() const
+        {
+                return !_attr;
+        }
+
+        PUGI__FN bool xml_attribute::operator==(const xml_attribute& r) const
         {
                 return (_attr == r._attr);
         }
-        
-        bool xml_attribute::operator!=(const xml_attribute& r) const
+
+        PUGI__FN bool xml_attribute::operator!=(const xml_attribute& r) const
         {
                 return (_attr != r._attr);
         }
 
-        bool xml_attribute::operator<(const xml_attribute& r) const
+        PUGI__FN bool xml_attribute::operator<(const xml_attribute& r) const
         {
                 return (_attr < r._attr);
         }
-        
-        bool xml_attribute::operator>(const xml_attribute& r) const
+
+        PUGI__FN bool xml_attribute::operator>(const xml_attribute& r) const
         {
                 return (_attr > r._attr);
         }
-        
-        bool xml_attribute::operator<=(const xml_attribute& r) const
+
+        PUGI__FN bool xml_attribute::operator<=(const xml_attribute& r) const
         {
                 return (_attr <= r._attr);
         }
-        
-        bool xml_attribute::operator>=(const xml_attribute& r) const
+
+        PUGI__FN bool xml_attribute::operator>=(const xml_attribute& r) const
         {
                 return (_attr >= r._attr);
         }
 
-        xml_attribute xml_attribute::next_attribute() const
-        {
-        return _attr ? xml_attribute(_attr->next_attribute) : xml_attribute();
-        }
-
-    xml_attribute xml_attribute::previous_attribute() const
-    {
-        return _attr && _attr->prev_attribute_c->next_attribute ? xml_attribute(_attr->prev_attribute_c) : xml_attribute();
-    }
-
-        int xml_attribute::as_int() const
-        {
-                if (!_attr || !_attr->value) return 0;
-
-        #ifdef PUGIXML_WCHAR_MODE
-                return (int)wcstol(_attr->value, 0, 10);
-        #else
-                return (int)strtol(_attr->value, 0, 10);
-        #endif
-        }
-
-        unsigned int xml_attribute::as_uint() const
-        {
-                if (!_attr || !_attr->value) return 0;
-
-        #ifdef PUGIXML_WCHAR_MODE
-                return (unsigned int)wcstoul(_attr->value, 0, 10);
-        #else
-                return (unsigned int)strtoul(_attr->value, 0, 10);
-        #endif
-        }
-
-        double xml_attribute::as_double() const
-        {
-                if (!_attr || !_attr->value) return 0;
-
-        #ifdef PUGIXML_WCHAR_MODE
-                return wcstod(_attr->value, 0);
-        #else
-                return strtod(_attr->value, 0);
-        #endif
-        }
-
-        float xml_attribute::as_float() const
-        {
-                if (!_attr || !_attr->value) return 0;
-
-        #ifdef PUGIXML_WCHAR_MODE
-                return (float)wcstod(_attr->value, 0);
-        #else
-                return (float)strtod(_attr->value, 0);
-        #endif
-        }
-
-        bool xml_attribute::as_bool() const
-        {
-                if (!_attr || !_attr->value) return false;
-
-                // only look at first char
-                char_t first = *_attr->value;
-
-                // 1*, t* (true), T* (True), y* (yes), Y* (YES)
-                return (first == '1' || first == 't' || first == 'T' || first == 'y' || first == 'Y');
-        }
-
-        bool xml_attribute::empty() const
+        PUGI__FN xml_attribute xml_attribute::next_attribute() const
+        {
+                return _attr ? xml_attribute(_attr->next_attribute) : xml_attribute();
+        }
+
+        PUGI__FN xml_attribute xml_attribute::previous_attribute() const
+        {
+                return _attr && _attr->prev_attribute_c->next_attribute ? xml_attribute(_attr->prev_attribute_c) : xml_attribute();
+        }
+
+        PUGI__FN const char_t* xml_attribute::as_string(const char_t* def) const
+        {
+                return (_attr && _attr->value) ? _attr->value + 0 : def;
+        }
+
+        PUGI__FN int xml_attribute::as_int(int def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_int(_attr->value) : def;
+        }
+
+        PUGI__FN unsigned int xml_attribute::as_uint(unsigned int def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_uint(_attr->value) : def;
+        }
+
+        PUGI__FN double xml_attribute::as_double(double def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_double(_attr->value) : def;
+        }
+
+        PUGI__FN float xml_attribute::as_float(float def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_float(_attr->value) : def;
+        }
+
+        PUGI__FN bool xml_attribute::as_bool(bool def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_bool(_attr->value) : def;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN long long xml_attribute::as_llong(long long def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_llong(_attr->value) : def;
+        }
+
+        PUGI__FN unsigned long long xml_attribute::as_ullong(unsigned long long def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_ullong(_attr->value) : def;
+        }
+#endif
+
+        PUGI__FN bool xml_attribute::empty() const
         {
                 return !_attr;
         }
 
-        const char_t* xml_attribute::name() const
-        {
-                return (_attr && _attr->name) ? _attr->name : PUGIXML_TEXT("");
-        }
-
-        const char_t* xml_attribute::value() const
-        {
-                return (_attr && _attr->value) ? _attr->value : PUGIXML_TEXT("");
-        }
-
-    size_t xml_attribute::hash_value() const
-    {
-        return static_cast<size_t>(reinterpret_cast<uintptr_t>(_attr) / sizeof(xml_attribute_struct));
-    }
-
-        xml_attribute_struct* xml_attribute::internal_object() const
-        {
-        return _attr;
-        }
-
-        xml_attribute& xml_attribute::operator=(const char_t* rhs)
+        PUGI__FN const char_t* xml_attribute::name() const
+        {
+                return (_attr && _attr->name) ? _attr->name + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN const char_t* xml_attribute::value() const
+        {
+                return (_attr && _attr->value) ? _attr->value + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN size_t xml_attribute::hash_value() const
+        {
+                return static_cast<size_t>(reinterpret_cast<uintptr_t>(_attr) / sizeof(xml_attribute_struct));
+        }
+
+        PUGI__FN xml_attribute_struct* xml_attribute::internal_object() const
+        {
+                return _attr;
+        }
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(const char_t* rhs)
         {
                 set_value(rhs);
                 return *this;
         }
-        
-        xml_attribute& xml_attribute::operator=(int rhs)
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(int rhs)
         {
                 set_value(rhs);
@@ -3451 +5204 @@
         }
 
-        xml_attribute& xml_attribute::operator=(unsigned int rhs)
+        PUGI__FN xml_attribute& xml_attribute::operator=(unsigned int rhs)
         {
                 set_value(rhs);
@@ -3457 +5210 @@
         }
 
-        xml_attribute& xml_attribute::operator=(double rhs)
+        PUGI__FN xml_attribute& xml_attribute::operator=(long rhs)
         {
                 set_value(rhs);
                 return *this;
         }
-        
-        xml_attribute& xml_attribute::operator=(bool rhs)
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(unsigned long rhs)
         {
                 set_value(rhs);
@@ -3469 +5222 @@
         }
 
-        bool xml_attribute::set_name(const char_t* rhs)
+        PUGI__FN xml_attribute& xml_attribute::operator=(double rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(float rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(bool rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN xml_attribute& xml_attribute::operator=(long long rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(unsigned long long rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+#endif
+
+        PUGI__FN bool xml_attribute::set_name(const char_t* rhs)
         {
                 if (!_attr) return false;
-                
-                return strcpy_insitu(_attr->name, _attr->header, xml_memory_page_name_allocated_mask, rhs);
-        }
-                
-        bool xml_attribute::set_value(const char_t* rhs)
+
+                return impl::strcpy_insitu(_attr->name, _attr->header, impl::xml_memory_page_name_allocated_mask, rhs, impl::strlength(rhs));
+        }
+
+        PUGI__FN bool xml_attribute::set_value(const char_t* rhs)
         {
                 if (!_attr) return false;
 
-                return strcpy_insitu(_attr->value, _attr->header, xml_memory_page_value_allocated_mask, rhs);
-        }
-
-        bool xml_attribute::set_value(int rhs)
-        {
-                char buf[128];
-                sprintf(buf, "%d", rhs);
-        
-        #ifdef PUGIXML_WCHAR_MODE
-                char_t wbuf[128];
-                widen_ascii(wbuf, buf);
-
-                return set_value(wbuf);
-        #else
-                return set_value(buf);
-        #endif
-        }
-
-        bool xml_attribute::set_value(unsigned int rhs)
-        {
-                char buf[128];
-                sprintf(buf, "%u", rhs);
-
-        #ifdef PUGIXML_WCHAR_MODE
-                char_t wbuf[128];
-                widen_ascii(wbuf, buf);
-
-                return set_value(wbuf);
-        #else
-                return set_value(buf);
-        #endif
-        }
-
-        bool xml_attribute::set_value(double rhs)
-        {
-                char buf[128];
-                sprintf(buf, "%g", rhs);
-
-        #ifdef PUGIXML_WCHAR_MODE
-                char_t wbuf[128];
-                widen_ascii(wbuf, buf);
-
-                return set_value(wbuf);
-        #else
-                return set_value(buf);
-        #endif
-        }
-        
-        bool xml_attribute::set_value(bool rhs)
-        {
-                return set_value(rhs ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));
-        }
+                return impl::strcpy_insitu(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, impl::strlength(rhs));
+        }
+
+        PUGI__FN bool xml_attribute::set_value(int rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned int>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(unsigned int rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned int>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, false);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(long rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned long>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(unsigned long rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned long>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, false);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(double rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(float rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(bool rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_bool(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN bool xml_attribute::set_value(long long rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned long long>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(unsigned long long rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned long long>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, false);
+        }
+#endif
 
 #ifdef __BORLANDC__
-        bool operator&&(const xml_attribute& lhs, bool rhs)
+        PUGI__FN bool operator&&(const xml_attribute& lhs, bool rhs)
         {
                 return (bool)lhs && rhs;
         }
 
-        bool operator||(const xml_attribute& lhs, bool rhs)
+        PUGI__FN bool operator||(const xml_attribute& lhs, bool rhs)
         {
                 return (bool)lhs || rhs;
@@ -3545 +5345 @@
 #endif
 
-        xml_node::xml_node(): _root(0)
-        {
-        }
-
-        xml_node::xml_node(xml_node_struct* p): _root(p)
-        {
-        }
-        
-        xml_node::operator xml_node::unspecified_bool_type() const
-        {
-        return _root ? &xml_node::_root : 0;
-        }
-
-        bool xml_node::operator!() const
-        {
-                return !_root;
-        }
-
-        xml_node::iterator xml_node::begin() const
-        {
-                return iterator(_root ? _root->first_child : 0, _root);
-        }
-
-        xml_node::iterator xml_node::end() const
+        PUGI__FN xml_node::xml_node(): _root(0)
+        {
+        }
+
+        PUGI__FN xml_node::xml_node(xml_node_struct* p): _root(p)
+        {
+        }
+
+        PUGI__FN static void unspecified_bool_xml_node(xml_node***)
+        {
+        }
+
+        PUGI__FN xml_node::operator xml_node::unspecified_bool_type() const
+        {
+                return _root ? unspecified_bool_xml_node : 0;
+        }
+
+        PUGI__FN bool xml_node::operator!() const
+        {
+                return !_root;
+        }
+
+        PUGI__FN xml_node::iterator xml_node::begin() const
+        {
+                return iterator(_root ? _root->first_child + 0 : 0, _root);
+        }
+
+        PUGI__FN xml_node::iterator xml_node::end() const
         {
                 return iterator(0, _root);
         }
-        
-        xml_node::attribute_iterator xml_node::attributes_begin() const
-        {
-                return attribute_iterator(_root ? _root->first_attribute : 0, _root);
-        }
-
-        xml_node::attribute_iterator xml_node::attributes_end() const
+
+        PUGI__FN xml_node::attribute_iterator xml_node::attributes_begin() const
+        {
+                return attribute_iterator(_root ? _root->first_attribute + 0 : 0, _root);
+        }
+
+        PUGI__FN xml_node::attribute_iterator xml_node::attributes_end() const
         {
                 return attribute_iterator(0, _root);
         }
 
-        bool xml_node::operator==(const xml_node& r) const
+        PUGI__FN xml_object_range<xml_node_iterator> xml_node::children() const
+        {
+                return xml_object_range<xml_node_iterator>(begin(), end());
+        }
+
+        PUGI__FN xml_object_range<xml_named_node_iterator> xml_node::children(const char_t* name_) const
+        {
+                return xml_object_range<xml_named_node_iterator>(xml_named_node_iterator(child(name_)._root, _root, name_), xml_named_node_iterator(0, _root, name_));
+        }
+
+        PUGI__FN xml_object_range<xml_attribute_iterator> xml_node::attributes() const
+        {
+                return xml_object_range<xml_attribute_iterator>(attributes_begin(), attributes_end());
+        }
+
+        PUGI__FN bool xml_node::operator==(const xml_node& r) const
         {
                 return (_root == r._root);
         }
 
-        bool xml_node::operator!=(const xml_node& r) const
+        PUGI__FN bool xml_node::operator!=(const xml_node& r) const
         {
                 return (_root != r._root);
         }
 
-        bool xml_node::operator<(const xml_node& r) const
+        PUGI__FN bool xml_node::operator<(const xml_node& r) const
         {
                 return (_root < r._root);
         }
-        
-        bool xml_node::operator>(const xml_node& r) const
+
+        PUGI__FN bool xml_node::operator>(const xml_node& r) const
         {
                 return (_root > r._root);
         }
-        
-        bool xml_node::operator<=(const xml_node& r) const
+
+        PUGI__FN bool xml_node::operator<=(const xml_node& r) const
         {
                 return (_root <= r._root);
         }
-        
-        bool xml_node::operator>=(const xml_node& r) const
+
+        PUGI__FN bool xml_node::operator>=(const xml_node& r) const
         {
                 return (_root >= r._root);
         }
 
-        bool xml_node::empty() const
+        PUGI__FN bool xml_node::empty() const
         {
                 return !_root;
         }
-        
-        const char_t* xml_node::name() const
-        {
-                return (_root && _root->name) ? _root->name : PUGIXML_TEXT("");
-        }
-
-        xml_node_type xml_node::type() const
-        {
-                return _root ? static_cast<xml_node_type>((_root->header & xml_memory_page_type_mask) + 1) : node_null;
-        }
-        
-        const char_t* xml_node::value() const
-        {
-                return (_root && _root->value) ? _root->value : PUGIXML_TEXT("");
-        }
-        
-        xml_node xml_node::child(const char_t* name) const
+
+        PUGI__FN const char_t* xml_node::name() const
+        {
+                return (_root && _root->name) ? _root->name + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN xml_node_type xml_node::type() const
+        {
+                return _root ? PUGI__NODETYPE(_root) : node_null;
+        }
+
+        PUGI__FN const char_t* xml_node::value() const
+        {
+                return (_root && _root->value) ? _root->value + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN xml_node xml_node::child(const char_t* name_) const
         {
                 if (!_root) return xml_node();
 
                 for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
-                        if (i->name && strequal(name, i->name)) return xml_node(i);
+                        if (i->name && impl::strequal(name_, i->name)) return xml_node(i);
 
                 return xml_node();
         }
 
-        xml_attribute xml_node::attribute(const char_t* name) const
+        PUGI__FN xml_attribute xml_node::attribute(const char_t* name_) const
         {
                 if (!_root) return xml_attribute();
 
                 for (xml_attribute_struct* i = _root->first_attribute; i; i = i->next_attribute)
-                        if (i->name && strequal(name, i->name))
+                        if (i->name && impl::strequal(name_, i->name))
                                 return xml_attribute(i);
-                
+
                 return xml_attribute();
         }
-        
-        xml_node xml_node::next_sibling(const char_t* name) const
+
+        PUGI__FN xml_node xml_node::next_sibling(const char_t* name_) const
         {
                 if (!_root) return xml_node();
-                
+
                 for (xml_node_struct* i = _root->next_sibling; i; i = i->next_sibling)
-                        if (i->name && strequal(name, i->name)) return xml_node(i);
+                        if (i->name && impl::strequal(name_, i->name)) return xml_node(i);
 
                 return xml_node();
         }
 
-        xml_node xml_node::next_sibling() const
+        PUGI__FN xml_node xml_node::next_sibling() const
+        {
+                return _root ? xml_node(_root->next_sibling) : xml_node();
+        }
+
+        PUGI__FN xml_node xml_node::previous_sibling(const char_t* name_) const
         {
                 if (!_root) return xml_node();
-                
-                if (_root->next_sibling) return xml_node(_root->next_sibling);
-                else return xml_node();
-        }
-
-        xml_node xml_node::previous_sibling(const char_t* name) const
+
+                for (xml_node_struct* i = _root->prev_sibling_c; i->next_sibling; i = i->prev_sibling_c)
+                        if (i->name && impl::strequal(name_, i->name)) return xml_node(i);
+
+                return xml_node();
+        }
+
+        PUGI__FN xml_attribute xml_node::attribute(const char_t* name_, xml_attribute& hint_) const
+        {
+                xml_attribute_struct* hint = hint_._attr;
+
+                // if hint is not an attribute of node, behavior is not defined
+                assert(!hint || (_root && impl::is_attribute_of(hint, _root)));
+
+                if (!_root) return xml_attribute();
+
+                // optimistically search from hint up until the end
+                for (xml_attribute_struct* i = hint; i; i = i->next_attribute)
+                        if (i->name && impl::strequal(name_, i->name))
+                        {
+                                // update hint to maximize efficiency of searching for consecutive attributes
+                                hint_._attr = i->next_attribute;
+
+                                return xml_attribute(i);
+                        }
+
+                // wrap around and search from the first attribute until the hint
+                // 'j' null pointer check is technically redundant, but it prevents a crash in case the assertion above fails
+                for (xml_attribute_struct* j = _root->first_attribute; j && j != hint; j = j->next_attribute)
+                        if (j->name && impl::strequal(name_, j->name))
+                        {
+                                // update hint to maximize efficiency of searching for consecutive attributes
+                                hint_._attr = j->next_attribute;
+
+                                return xml_attribute(j);
+                        }
+
+                return xml_attribute();
+        }
+
+        PUGI__FN xml_node xml_node::previous_sibling() const
         {
                 if (!_root) return xml_node();
-                
-                for (xml_node_struct* i = _root->prev_sibling_c; i->next_sibling; i = i->prev_sibling_c)
-                        if (i->name && strequal(name, i->name)) return xml_node(i);
-
-                return xml_node();
-        }
-
-        xml_node xml_node::previous_sibling() const
-        {
-                if (!_root) return xml_node();
-                
+
                 if (_root->prev_sibling_c->next_sibling) return xml_node(_root->prev_sibling_c);
                 else return xml_node();
         }
 
-        xml_node xml_node::parent() const
+        PUGI__FN xml_node xml_node::parent() const
         {
                 return _root ? xml_node(_root->parent) : xml_node();
         }
 
-        xml_node xml_node::root() const
+        PUGI__FN xml_node xml_node::root() const
+        {
+                return _root ? xml_node(&impl::get_document(_root)) : xml_node();
+        }
+
+        PUGI__FN xml_text xml_node::text() const
+        {
+                return xml_text(_root);
+        }
+
+        PUGI__FN const char_t* xml_node::child_value() const
+        {
+                if (!_root) return PUGIXML_TEXT("");
+
+                // element nodes can have value if parse_embed_pcdata was used
+                if (PUGI__NODETYPE(_root) == node_element && _root->value)
+                        return _root->value;
+
+                for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
+                        if (impl::is_text_node(i) && i->value)
+                                return i->value;
+
+                return PUGIXML_TEXT("");
+        }
+
+        PUGI__FN const char_t* xml_node::child_value(const char_t* name_) const
+        {
+                return child(name_).child_value();
+        }
+
+        PUGI__FN xml_attribute xml_node::first_attribute() const
+        {
+                return _root ? xml_attribute(_root->first_attribute) : xml_attribute();
+        }
+
+        PUGI__FN xml_attribute xml_node::last_attribute() const
+        {
+                return _root && _root->first_attribute ? xml_attribute(_root->first_attribute->prev_attribute_c) : xml_attribute();
+        }
+
+        PUGI__FN xml_node xml_node::first_child() const
+        {
+                return _root ? xml_node(_root->first_child) : xml_node();
+        }
+
+        PUGI__FN xml_node xml_node::last_child() const
+        {
+                return _root && _root->first_child ? xml_node(_root->first_child->prev_sibling_c) : xml_node();
+        }
+
+        PUGI__FN bool xml_node::set_name(const char_t* rhs)
+        {
+                xml_node_type type_ = _root ? PUGI__NODETYPE(_root) : node_null;
+
+                if (type_ != node_element && type_ != node_pi && type_ != node_declaration)
+                        return false;
+
+                return impl::strcpy_insitu(_root->name, _root->header, impl::xml_memory_page_name_allocated_mask, rhs, impl::strlength(rhs));
+        }
+
+        PUGI__FN bool xml_node::set_value(const char_t* rhs)
+        {
+                xml_node_type type_ = _root ? PUGI__NODETYPE(_root) : node_null;
+
+                if (type_ != node_pcdata && type_ != node_cdata && type_ != node_comment && type_ != node_pi && type_ != node_doctype)
+                        return false;
+
+                return impl::strcpy_insitu(_root->value, _root->header, impl::xml_memory_page_value_allocated_mask, rhs, impl::strlength(rhs));
+        }
+
+        PUGI__FN xml_attribute xml_node::append_attribute(const char_t* name_)
+        {
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::append_attribute(a._attr, _root);
+
+                a.set_name(name_);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::prepend_attribute(const char_t* name_)
+        {
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::prepend_attribute(a._attr, _root);
+
+                a.set_name(name_);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::insert_attribute_after(const char_t* name_, const xml_attribute& attr)
+        {
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+                if (!attr || !impl::is_attribute_of(attr._attr, _root)) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::insert_attribute_after(a._attr, attr._attr, _root);
+
+                a.set_name(name_);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::insert_attribute_before(const char_t* name_, const xml_attribute& attr)
+        {
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+                if (!attr || !impl::is_attribute_of(attr._attr, _root)) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::insert_attribute_before(a._attr, attr._attr, _root);
+
+                a.set_name(name_);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::append_copy(const xml_attribute& proto)
+        {
+                if (!proto) return xml_attribute();
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::append_attribute(a._attr, _root);
+                impl::node_copy_attribute(a._attr, proto._attr);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::prepend_copy(const xml_attribute& proto)
+        {
+                if (!proto) return xml_attribute();
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::prepend_attribute(a._attr, _root);
+                impl::node_copy_attribute(a._attr, proto._attr);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::insert_copy_after(const xml_attribute& proto, const xml_attribute& attr)
+        {
+                if (!proto) return xml_attribute();
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+                if (!attr || !impl::is_attribute_of(attr._attr, _root)) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::insert_attribute_after(a._attr, attr._attr, _root);
+                impl::node_copy_attribute(a._attr, proto._attr);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::insert_copy_before(const xml_attribute& proto, const xml_attribute& attr)
+        {
+                if (!proto) return xml_attribute();
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+                if (!attr || !impl::is_attribute_of(attr._attr, _root)) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::insert_attribute_before(a._attr, attr._attr, _root);
+                impl::node_copy_attribute(a._attr, proto._attr);
+
+                return a;
+        }
+
+        PUGI__FN xml_node xml_node::append_child(xml_node_type type_)
+        {
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::append_node(n._root, _root);
+
+                if (type_ == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::prepend_child(xml_node_type type_)
+        {
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::prepend_node(n._root, _root);
+
+                if (type_ == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::insert_child_before(xml_node_type type_, const xml_node& node)
+        {
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::insert_node_before(n._root, node._root);
+
+                if (type_ == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::insert_child_after(xml_node_type type_, const xml_node& node)
+        {
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::insert_node_after(n._root, node._root);
+
+                if (type_ == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::append_child(const char_t* name_)
+        {
+                xml_node result = append_child(node_element);
+
+                result.set_name(name_);
+
+                return result;
+        }
+
+        PUGI__FN xml_node xml_node::prepend_child(const char_t* name_)
+        {
+                xml_node result = prepend_child(node_element);
+
+                result.set_name(name_);
+
+                return result;
+        }
+
+        PUGI__FN xml_node xml_node::insert_child_after(const char_t* name_, const xml_node& node)
+        {
+                xml_node result = insert_child_after(node_element, node);
+
+                result.set_name(name_);
+
+                return result;
+        }
+
+        PUGI__FN xml_node xml_node::insert_child_before(const char_t* name_, const xml_node& node)
+        {
+                xml_node result = insert_child_before(node_element, node);
+
+                result.set_name(name_);
+
+                return result;
+        }
+
+        PUGI__FN xml_node xml_node::append_copy(const xml_node& proto)
+        {
+                xml_node_type type_ = proto.type();
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::append_node(n._root, _root);
+                impl::node_copy_tree(n._root, proto._root);
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::prepend_copy(const xml_node& proto)
+        {
+                xml_node_type type_ = proto.type();
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::prepend_node(n._root, _root);
+                impl::node_copy_tree(n._root, proto._root);
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::insert_copy_after(const xml_node& proto, const xml_node& node)
+        {
+                xml_node_type type_ = proto.type();
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::insert_node_after(n._root, node._root);
+                impl::node_copy_tree(n._root, proto._root);
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::insert_copy_before(const xml_node& proto, const xml_node& node)
+        {
+                xml_node_type type_ = proto.type();
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::insert_node_before(n._root, node._root);
+                impl::node_copy_tree(n._root, proto._root);
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::append_move(const xml_node& moved)
+        {
+                if (!impl::allow_move(*this, moved)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                // disable document_buffer_order optimization since moving nodes around changes document order without changing buffer pointers
+                impl::get_document(_root).header |= impl::xml_memory_page_contents_shared_mask;
+
+                impl::remove_node(moved._root);
+                impl::append_node(moved._root, _root);
+
+                return moved;
+        }
+
+        PUGI__FN xml_node xml_node::prepend_move(const xml_node& moved)
+        {
+                if (!impl::allow_move(*this, moved)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                // disable document_buffer_order optimization since moving nodes around changes document order without changing buffer pointers
+                impl::get_document(_root).header |= impl::xml_memory_page_contents_shared_mask;
+
+                impl::remove_node(moved._root);
+                impl::prepend_node(moved._root, _root);
+
+                return moved;
+        }
+
+        PUGI__FN xml_node xml_node::insert_move_after(const xml_node& moved, const xml_node& node)
+        {
+                if (!impl::allow_move(*this, moved)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+                if (moved._root == node._root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                // disable document_buffer_order optimization since moving nodes around changes document order without changing buffer pointers
+                impl::get_document(_root).header |= impl::xml_memory_page_contents_shared_mask;
+
+                impl::remove_node(moved._root);
+                impl::insert_node_after(moved._root, node._root);
+
+                return moved;
+        }
+
+        PUGI__FN xml_node xml_node::insert_move_before(const xml_node& moved, const xml_node& node)
+        {
+                if (!impl::allow_move(*this, moved)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+                if (moved._root == node._root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                // disable document_buffer_order optimization since moving nodes around changes document order without changing buffer pointers
+                impl::get_document(_root).header |= impl::xml_memory_page_contents_shared_mask;
+
+                impl::remove_node(moved._root);
+                impl::insert_node_before(moved._root, node._root);
+
+                return moved;
+        }
+
+        PUGI__FN bool xml_node::remove_attribute(const char_t* name_)
+        {
+                return remove_attribute(attribute(name_));
+        }
+
+        PUGI__FN bool xml_node::remove_attribute(const xml_attribute& a)
+        {
+                if (!_root || !a._attr) return false;
+                if (!impl::is_attribute_of(a._attr, _root)) return false;
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return false;
+
+                impl::remove_attribute(a._attr, _root);
+                impl::destroy_attribute(a._attr, alloc);
+
+                return true;
+        }
+
+        PUGI__FN bool xml_node::remove_child(const char_t* name_)
+        {
+                return remove_child(child(name_));
+        }
+
+        PUGI__FN bool xml_node::remove_child(const xml_node& n)
+        {
+                if (!_root || !n._root || n._root->parent != _root) return false;
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return false;
+
+                impl::remove_node(n._root);
+                impl::destroy_node(n._root, alloc);
+
+                return true;
+        }
+
+        PUGI__FN xml_parse_result xml_node::append_buffer(const void* contents, size_t size, unsigned int options, xml_encoding encoding)
+        {
+                // append_buffer is only valid for elements/documents
+                if (!impl::allow_insert_child(type(), node_element)) return impl::make_parse_result(status_append_invalid_root);
+
+                // get document node
+                impl::xml_document_struct* doc = &impl::get_document(_root);
+
+                // disable document_buffer_order optimization since in a document with multiple buffers comparing buffer pointers does not make sense
+                doc->header |= impl::xml_memory_page_contents_shared_mask;
+
+                // get extra buffer element (we'll store the document fragment buffer there so that we can deallocate it later)
+                impl::xml_memory_page* page = 0;
+                impl::xml_extra_buffer* extra = static_cast<impl::xml_extra_buffer*>(doc->allocate_memory(sizeof(impl::xml_extra_buffer), page));
+                (void)page;
+
+                if (!extra) return impl::make_parse_result(status_out_of_memory);
+
+                // add extra buffer to the list
+                extra->buffer = 0;
+                extra->next = doc->extra_buffers;
+                doc->extra_buffers = extra;
+
+                // name of the root has to be NULL before parsing - otherwise closing node mismatches will not be detected at the top level
+                impl::name_null_sentry sentry(_root);
+
+                return impl::load_buffer_impl(doc, _root, const_cast<void*>(contents), size, options, encoding, false, false, &extra->buffer);
+        }
+
+        PUGI__FN xml_node xml_node::find_child_by_attribute(const char_t* name_, const char_t* attr_name, const char_t* attr_value) const
         {
                 if (!_root) return xml_node();
 
-                xml_memory_page* page = reinterpret_cast<xml_memory_page*>(_root->header & xml_memory_page_pointer_mask);
-
-                return xml_node(static_cast<xml_document_struct*>(page->allocator));
-        }
-
-        const char_t* xml_node::child_value() const
-        {
-                if (!_root) return PUGIXML_TEXT("");
-                
                 for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
-                {
-                        xml_node_type type = static_cast<xml_node_type>((i->header & xml_memory_page_type_mask) + 1);
-
-                        if (i->value && (type == node_pcdata || type == node_cdata))
-                                return i->value;
-                }
-
-                return PUGIXML_TEXT("");
-        }
-
-        const char_t* xml_node::child_value(const char_t* name) const
-        {
-                return child(name).child_value();
-        }
-
-        xml_attribute xml_node::first_attribute() const
-        {
-                return _root ? xml_attribute(_root->first_attribute) : xml_attribute();
-        }
-
-        xml_attribute xml_node::last_attribute() const
-        {
-                return _root && _root->first_attribute ? xml_attribute(_root->first_attribute->prev_attribute_c) : xml_attribute();
-        }
-
-        xml_node xml_node::first_child() const
-        {
-                return _root ? xml_node(_root->first_child) : xml_node();
-        }
-
-        xml_node xml_node::last_child() const
-        {
-                return _root && _root->first_child ? xml_node(_root->first_child->prev_sibling_c) : xml_node();
-        }
-
-        bool xml_node::set_name(const char_t* rhs)
-        {
-                switch (type())
-                {
-                case node_pi:
-                case node_declaration:
-                case node_element:
-                        return strcpy_insitu(_root->name, _root->header, xml_memory_page_name_allocated_mask, rhs);
-
-                default:
-                        return false;
-                }
-        }
-                
-        bool xml_node::set_value(const char_t* rhs)
-        {
-                switch (type())
-                {
-                case node_pi:
-                case node_cdata:
-                case node_pcdata:
-                case node_comment:
-        case node_doctype:
-                        return strcpy_insitu(_root->value, _root->header, xml_memory_page_value_allocated_mask, rhs);
-
-                default:
-                        return false;
-                }
-        }
-
-        xml_attribute xml_node::append_attribute(const char_t* name)
-        {
-                if (type() != node_element && type() != node_declaration) return xml_attribute();
-                
-                xml_attribute a(append_attribute_ll(_root, get_allocator(_root)));
-                a.set_name(name);
-                
-                return a;
-        }
-
-        xml_attribute xml_node::prepend_attribute(const char_t* name)
-        {
-                if (type() != node_element && type() != node_declaration) return xml_attribute();
-                
-                xml_attribute a(allocate_attribute(get_allocator(_root)));
-                if (!a) return xml_attribute();
-
-                a.set_name(name);
-                
-        xml_attribute_struct* head = _root->first_attribute;
-
-                if (head)
-        {
-            a._attr->prev_attribute_c = head->prev_attribute_c;
-            head->prev_attribute_c = a._attr;
-        }
-        else
-            a._attr->prev_attribute_c = a._attr;
-                
-                a._attr->next_attribute = head;
-        _root->first_attribute = a._attr;
-                                
-                return a;
-        }
-
-        xml_attribute xml_node::insert_attribute_before(const char_t* name, const xml_attribute& attr)
-        {
-                if ((type() != node_element && type() != node_declaration) || attr.empty()) return xml_attribute();
-                
-                // check that attribute belongs to *this
-                xml_attribute_struct* cur = attr._attr;
-
-                while (cur->prev_attribute_c->next_attribute) cur = cur->prev_attribute_c;
-
-                if (cur != _root->first_attribute) return xml_attribute();
-
-                xml_attribute a(allocate_attribute(get_allocator(_root)));
-                if (!a) return xml_attribute();
-
-                a.set_name(name);
-
-                if (attr._attr->prev_attribute_c->next_attribute)
-                        attr._attr->prev_attribute_c->next_attribute = a._attr;
-                else
-                        _root->first_attribute = a._attr;
-                
-                a._attr->prev_attribute_c = attr._attr->prev_attribute_c;
-                a._attr->next_attribute = attr._attr;
-                attr._attr->prev_attribute_c = a._attr;
-                                
-                return a;
-        }
-
-        xml_attribute xml_node::insert_attribute_after(const char_t* name, const xml_attribute& attr)
-        {
-                if ((type() != node_element && type() != node_declaration) || attr.empty()) return xml_attribute();
-                
-                // check that attribute belongs to *this
-                xml_attribute_struct* cur = attr._attr;
-
-                while (cur->prev_attribute_c->next_attribute) cur = cur->prev_attribute_c;
-
-                if (cur != _root->first_attribute) return xml_attribute();
-
-                xml_attribute a(allocate_attribute(get_allocator(_root)));
-                if (!a) return xml_attribute();
-
-                a.set_name(name);
-
-                if (attr._attr->next_attribute)
-                        attr._attr->next_attribute->prev_attribute_c = a._attr;
-                else
-                        _root->first_attribute->prev_attribute_c = a._attr;
-                
-                a._attr->next_attribute = attr._attr->next_attribute;
-                a._attr->prev_attribute_c = attr._attr;
-                attr._attr->next_attribute = a._attr;
-
-                return a;
-        }
-
-        xml_attribute xml_node::append_copy(const xml_attribute& proto)
-        {
-                if (!proto) return xml_attribute();
-
-                xml_attribute result = append_attribute(proto.name());
-                result.set_value(proto.value());
-
-                return result;
-        }
-
-        xml_attribute xml_node::prepend_copy(const xml_attribute& proto)
-        {
-                if (!proto) return xml_attribute();
-
-                xml_attribute result = prepend_attribute(proto.name());
-                result.set_value(proto.value());
-
-                return result;
-        }
-
-        xml_attribute xml_node::insert_copy_after(const xml_attribute& proto, const xml_attribute& attr)
-        {
-                if (!proto) return xml_attribute();
-
-                xml_attribute result = insert_attribute_after(proto.name(), attr);
-                result.set_value(proto.value());
-
-                return result;
-        }
-
-        xml_attribute xml_node::insert_copy_before(const xml_attribute& proto, const xml_attribute& attr)
-        {
-                if (!proto) return xml_attribute();
-
-                xml_attribute result = insert_attribute_before(proto.name(), attr);
-                result.set_value(proto.value());
-
-                return result;
-        }
-
-        xml_node xml_node::append_child(xml_node_type type)
-        {
-                if (!allow_insert_child(this->type(), type)) return xml_node();
-                
-                xml_node n(append_node(_root, get_allocator(_root), type));
-
-                if (type == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
-
-                return n;
-        }
-
-        xml_node xml_node::prepend_child(xml_node_type type)
-        {
-                if (!allow_insert_child(this->type(), type)) return xml_node();
-                
-                xml_node n(allocate_node(get_allocator(_root), type));
-                if (!n) return xml_node();
-
-        n._root->parent = _root;
-
-        xml_node_struct* head = _root->first_child;
-
-                if (head)
-        {
-            n._root->prev_sibling_c = head->prev_sibling_c;
-            head->prev_sibling_c = n._root;
-        }
-        else
-            n._root->prev_sibling_c = n._root;
-                
-                n._root->next_sibling = head;
-        _root->first_child = n._root;
-                                
-                if (type == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
-
-                return n;
-        }
-
-        xml_node xml_node::insert_child_before(xml_node_type type, const xml_node& node)
-        {
-                if (!allow_insert_child(this->type(), type)) return xml_node();
-                if (!node._root || node._root->parent != _root) return xml_node();
-        
-                xml_node n(allocate_node(get_allocator(_root), type));
-                if (!n) return xml_node();
-
-                n._root->parent = _root;
-                
-                if (node._root->prev_sibling_c->next_sibling)
-                        node._root->prev_sibling_c->next_sibling = n._root;
-                else
-                        _root->first_child = n._root;
-                
-                n._root->prev_sibling_c = node._root->prev_sibling_c;
-                n._root->next_sibling = node._root;
-                node._root->prev_sibling_c = n._root;
-
-                if (type == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
-
-                return n;
-        }
-
-        xml_node xml_node::insert_child_after(xml_node_type type, const xml_node& node)
-        {
-                if (!allow_insert_child(this->type(), type)) return xml_node();
-                if (!node._root || node._root->parent != _root) return xml_node();
-        
-                xml_node n(allocate_node(get_allocator(_root), type));
-                if (!n) return xml_node();
-
-                n._root->parent = _root;
-        
-                if (node._root->next_sibling)
-                        node._root->next_sibling->prev_sibling_c = n._root;
-                else
-                        _root->first_child->prev_sibling_c = n._root;
-                
-                n._root->next_sibling = node._root->next_sibling;
-                n._root->prev_sibling_c = node._root;
-                node._root->next_sibling = n._root;
-
-                if (type == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
-
-                return n;
-        }
-
-    xml_node xml_node::append_child(const char_t* name)
-    {
-        xml_node result = append_child(node_element);
-
-        result.set_name(name);
-
-        return result;
-    }
-
-    xml_node xml_node::prepend_child(const char_t* name)
-    {
-        xml_node result = prepend_child(node_element);
-
-        result.set_name(name);
-
-        return result;
-    }
-
-    xml_node xml_node::insert_child_after(const char_t* name, const xml_node& node)
-    {
-        xml_node result = insert_child_after(node_element, node);
-
-        result.set_name(name);
-
-        return result;
-    }
-
-    xml_node xml_node::insert_child_before(const char_t* name, const xml_node& node)
-    {
-        xml_node result = insert_child_before(node_element, node);
-
-        result.set_name(name);
-
-        return result;
-    }
-
-        xml_node xml_node::append_copy(const xml_node& proto)
-        {
-                xml_node result = append_child(proto.type());
-
-                if (result) recursive_copy_skip(result, proto, result);
-
-                return result;
-        }
-
-        xml_node xml_node::prepend_copy(const xml_node& proto)
-        {
-                xml_node result = prepend_child(proto.type());
-
-                if (result) recursive_copy_skip(result, proto, result);
-
-                return result;
-        }
-
-        xml_node xml_node::insert_copy_after(const xml_node& proto, const xml_node& node)
-        {
-                xml_node result = insert_child_after(proto.type(), node);
-
-                if (result) recursive_copy_skip(result, proto, result);
-
-                return result;
-        }
-
-        xml_node xml_node::insert_copy_before(const xml_node& proto, const xml_node& node)
-        {
-                xml_node result = insert_child_before(proto.type(), node);
-
-                if (result) recursive_copy_skip(result, proto, result);
-
-                return result;
-        }
-
-        bool xml_node::remove_attribute(const char_t* name)
-        {
-                return remove_attribute(attribute(name));
-        }
-
-        bool xml_node::remove_attribute(const xml_attribute& a)
-        {
-                if (!_root || !a._attr) return false;
-
-                // check that attribute belongs to *this
-                xml_attribute_struct* attr = a._attr;
-
-                while (attr->prev_attribute_c->next_attribute) attr = attr->prev_attribute_c;
-
-                if (attr != _root->first_attribute) return false;
-
-                if (a._attr->next_attribute) a._attr->next_attribute->prev_attribute_c = a._attr->prev_attribute_c;
-                else if (_root->first_attribute) _root->first_attribute->prev_attribute_c = a._attr->prev_attribute_c;
-                
-                if (a._attr->prev_attribute_c->next_attribute) a._attr->prev_attribute_c->next_attribute = a._attr->next_attribute;
-                else _root->first_attribute = a._attr->next_attribute;
-
-                destroy_attribute(a._attr, get_allocator(_root));
-
-                return true;
-        }
-
-        bool xml_node::remove_child(const char_t* name)
-        {
-                return remove_child(child(name));
-        }
-
-        bool xml_node::remove_child(const xml_node& n)
-        {
-                if (!_root || !n._root || n._root->parent != _root) return false;
-
-                if (n._root->next_sibling) n._root->next_sibling->prev_sibling_c = n._root->prev_sibling_c;
-                else if (_root->first_child) _root->first_child->prev_sibling_c = n._root->prev_sibling_c;
-                
-                if (n._root->prev_sibling_c->next_sibling) n._root->prev_sibling_c->next_sibling = n._root->next_sibling;
-                else _root->first_child = n._root->next_sibling;
-        
-        destroy_node(n._root, get_allocator(_root));
-
-                return true;
-        }
-
-        xml_node xml_node::find_child_by_attribute(const char_t* name, const char_t* attr_name, const char_t* attr_value) const
+                        if (i->name && impl::strequal(name_, i->name))
+                        {
+                                for (xml_attribute_struct* a = i->first_attribute; a; a = a->next_attribute)
+                                        if (a->name && impl::strequal(attr_name, a->name) && impl::strequal(attr_value, a->value ? a->value + 0 : PUGIXML_TEXT("")))
+                                                return xml_node(i);
+                        }
+
+                return xml_node();
+        }
+
+        PUGI__FN xml_node xml_node::find_child_by_attribute(const char_t* attr_name, const char_t* attr_value) const
         {
                 if (!_root) return xml_node();
-                
-                for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
-                        if (i->name && strequal(name, i->name))
-                        {
-                                for (xml_attribute_struct* a = i->first_attribute; a; a = a->next_attribute)
-                                        if (strequal(attr_name, a->name) && strequal(attr_value, a->value))
-                                                return xml_node(i);
-                        }
-
-                return xml_node();
-        }
-
-        xml_node xml_node::find_child_by_attribute(const char_t* attr_name, const char_t* attr_value) const
-        {
-                if (!_root) return xml_node();
-                
+
                 for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
                         for (xml_attribute_struct* a = i->first_attribute; a; a = a->next_attribute)
-                                if (strequal(attr_name, a->name) && strequal(attr_value, a->value))
+                                if (a->name && impl::strequal(attr_name, a->name) && impl::strequal(attr_value, a->value ? a->value + 0 : PUGIXML_TEXT("")))
                                         return xml_node(i);
 
@@ -4138 +6092 @@
 
 #ifndef PUGIXML_NO_STL
-        string_t xml_node::path(char_t delimiter) const
-        {
-                string_t path;
-
-                xml_node cursor = *this; // Make a copy.
-                
-                path = cursor.name();
-
-                while (cursor.parent())
-                {
-                        cursor = cursor.parent();
-                        
-                        string_t temp = cursor.name();
-                        temp += delimiter;
-                        temp += path;
-                        path.swap(temp);
-                }
-
-                return path;
+        PUGI__FN string_t xml_node::path(char_t delimiter) const
+        {
+                if (!_root) return string_t();
+
+                size_t offset = 0;
+
+                for (xml_node_struct* i = _root; i; i = i->parent)
+                {
+                        offset += (i != _root);
+                        offset += i->name ? impl::strlength(i->name) : 0;
+                }
+
+                string_t result;
+                result.resize(offset);
+
+                for (xml_node_struct* j = _root; j; j = j->parent)
+                {
+                        if (j != _root)
+                                result[--offset] = delimiter;
+
+                        if (j->name && *j->name)
+                        {
+                                size_t length = impl::strlength(j->name);
+
+                                offset -= length;
+                                memcpy(&result[offset], j->name, length * sizeof(char_t));
+                        }
+                }
+
+                assert(offset == 0);
+
+                return result;
         }
 #endif
 
-        xml_node xml_node::first_element_by_path(const char_t* path, char_t delimiter) const
+        PUGI__FN xml_node xml_node::first_element_by_path(const char_t* path_, char_t delimiter) const
         {
                 xml_node found = *this; // Current search context.
 
-                if (!_root || !path || !path[0]) return found;
-
-                if (path[0] == delimiter)
+                if (!_root || !path_ || !path_[0]) return found;
+
+                if (path_[0] == delimiter)
                 {
                         // Absolute path; e.g. '/foo/bar'
                         found = found.root();
-                        ++path;
-                }
-
-                const char_t* path_segment = path;
+                        ++path_;
+                }
+
+                const char_t* path_segment = path_;
 
                 while (*path_segment == delimiter) ++path_segment;
@@ -4195 +6162 @@
                         for (xml_node_struct* j = found._root->first_child; j; j = j->next_sibling)
                         {
-                                if (j->name && strequalrange(j->name, path_segment, static_cast<size_t>(path_segment_end - path_segment)))
+                                if (j->name && impl::strequalrange(j->name, path_segment, static_cast<size_t>(path_segment_end - path_segment)))
                                 {
                                         xml_node subsearch = xml_node(j).first_element_by_path(next_segment, delimiter);
@@ -4207 +6174 @@
         }
 
-        bool xml_node::traverse(xml_tree_walker& walker)
+        PUGI__FN bool xml_node::traverse(xml_tree_walker& walker)
         {
                 walker._depth = -1;
-                
+
                 xml_node arg_begin = *this;
                 if (!walker.begin(arg_begin)) return false;
 
                 xml_node cur = first_child();
-                                
+
                 if (cur)
                 {
                         ++walker._depth;
 
-                        do 
+                        do
                         {
                                 xml_node arg_for_each = cur;
                                 if (!walker.for_each(arg_for_each))
                                         return false;
-                                                
+
                                 if (cur.first_child())
                                 {
@@ -4236 +6203 @@
                                 {
                                         // Borland C++ workaround
-                                        while (!cur.next_sibling() && cur != *this && (bool)cur.parent())
+                                        while (!cur.next_sibling() && cur != *this && !cur.parent().empty())
                                         {
                                                 --walker._depth;
                                                 cur = cur.parent();
                                         }
-                                                
+
                                         if (cur != *this)
                                                 cur = cur.next_sibling();
@@ -4255 +6222 @@
         }
 
-    size_t xml_node::hash_value() const
-    {
-        return static_cast<size_t>(reinterpret_cast<uintptr_t>(_root) / sizeof(xml_node_struct));
-    }
-
-        xml_node_struct* xml_node::internal_object() const
-        {
-        return _root;
-        }
-
-        void xml_node::print(xml_writer& writer, const char_t* indent, unsigned int flags, xml_encoding encoding, unsigned int depth) const
+        PUGI__FN size_t xml_node::hash_value() const
+        {
+                return static_cast<size_t>(reinterpret_cast<uintptr_t>(_root) / sizeof(xml_node_struct));
+        }
+
+        PUGI__FN xml_node_struct* xml_node::internal_object() const
+        {
+                return _root;
+        }
+
+        PUGI__FN void xml_node::print(xml_writer& writer, const char_t* indent, unsigned int flags, xml_encoding encoding, unsigned int depth) const
         {
                 if (!_root) return;
 
-                xml_buffered_writer buffered_writer(writer, encoding);
-
-                node_output(buffered_writer, *this, indent, flags, depth);
+                impl::xml_buffered_writer buffered_writer(writer, encoding);
+
+                impl::node_output(buffered_writer, _root, indent, flags, depth);
+
+                buffered_writer.flush();
         }
 
 #ifndef PUGIXML_NO_STL
-        void xml_node::print(std::basic_ostream<char, std::char_traits<char> >& stream, const char_t* indent, unsigned int flags, xml_encoding encoding, unsigned int depth) const
+        PUGI__FN void xml_node::print(std::basic_ostream<char, std::char_traits<char> >& stream, const char_t* indent, unsigned int flags, xml_encoding encoding, unsigned int depth) const
         {
                 xml_writer_stream writer(stream);
@@ -4282 +6251 @@
         }
 
-        void xml_node::print(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream, const char_t* indent, unsigned int flags, unsigned int depth) const
+        PUGI__FN void xml_node::print(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream, const char_t* indent, unsigned int flags, unsigned int depth) const
         {
                 xml_writer_stream writer(stream);
@@ -4290 +6259 @@
 #endif
 
-        ptrdiff_t xml_node::offset_debug() const
-        {
-                xml_node_struct* r = root()._root;
-
-                if (!r) return -1;
-
-                const char_t* buffer = static_cast<xml_document_struct*>(r)->buffer;
-
-                if (!buffer) return -1;
+        PUGI__FN ptrdiff_t xml_node::offset_debug() const
+        {
+                if (!_root) return -1;
+
+                impl::xml_document_struct& doc = impl::get_document(_root);
+
+                // we can determine the offset reliably only if there is exactly once parse buffer
+                if (!doc.buffer || doc.extra_buffers) return -1;
 
                 switch (type())
@@ -4308 +6276 @@
                 case node_declaration:
                 case node_pi:
-                        return (_root->header & xml_memory_page_name_allocated_mask) ? -1 : _root->name - buffer;
+                        return _root->name && (_root->header & impl::xml_memory_page_name_allocated_or_shared_mask) == 0 ? _root->name - doc.buffer : -1;
 
                 case node_pcdata:
@@ -4314 +6282 @@
                 case node_comment:
                 case node_doctype:
-                        return (_root->header & xml_memory_page_value_allocated_mask) ? -1 : _root->value - buffer;
+                        return _root->value && (_root->header & impl::xml_memory_page_value_allocated_or_shared_mask) == 0 ? _root->value - doc.buffer : -1;
 
                 default:
@@ -4322 +6290 @@
 
 #ifdef __BORLANDC__
-        bool operator&&(const xml_node& lhs, bool rhs)
+        PUGI__FN bool operator&&(const xml_node& lhs, bool rhs)
         {
                 return (bool)lhs && rhs;
         }
 
-        bool operator||(const xml_node& lhs, bool rhs)
+        PUGI__FN bool operator||(const xml_node& lhs, bool rhs)
         {
                 return (bool)lhs || rhs;
@@ -4333 +6301 @@
 #endif
 
-        xml_node_iterator::xml_node_iterator()
-        {
-        }
-
-        xml_node_iterator::xml_node_iterator(const xml_node& node): _wrap(node), _parent(node.parent())
-        {
-        }
-
-        xml_node_iterator::xml_node_iterator(xml_node_struct* ref, xml_node_struct* parent): _wrap(ref), _parent(parent)
-        {
-        }
-
-        bool xml_node_iterator::operator==(const xml_node_iterator& rhs) const
+        PUGI__FN xml_text::xml_text(xml_node_struct* root): _root(root)
+        {
+        }
+
+        PUGI__FN xml_node_struct* xml_text::_data() const
+        {
+                if (!_root || impl::is_text_node(_root)) return _root;
+
+                // element nodes can have value if parse_embed_pcdata was used
+                if (PUGI__NODETYPE(_root) == node_element && _root->value)
+                        return _root;
+
+                for (xml_node_struct* node = _root->first_child; node; node = node->next_sibling)
+                        if (impl::is_text_node(node))
+                                return node;
+
+                return 0;
+        }
+
+        PUGI__FN xml_node_struct* xml_text::_data_new()
+        {
+                xml_node_struct* d = _data();
+                if (d) return d;
+
+                return xml_node(_root).append_child(node_pcdata).internal_object();
+        }
+
+        PUGI__FN xml_text::xml_text(): _root(0)
+        {
+        }
+
+        PUGI__FN static void unspecified_bool_xml_text(xml_text***)
+        {
+        }
+
+        PUGI__FN xml_text::operator xml_text::unspecified_bool_type() const
+        {
+                return _data() ? unspecified_bool_xml_text : 0;
+        }
+
+        PUGI__FN bool xml_text::operator!() const
+        {
+                return !_data();
+        }
+
+        PUGI__FN bool xml_text::empty() const
+        {
+                return _data() == 0;
+        }
+
+        PUGI__FN const char_t* xml_text::get() const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? d->value + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN const char_t* xml_text::as_string(const char_t* def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? d->value + 0 : def;
+        }
+
+        PUGI__FN int xml_text::as_int(int def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_int(d->value) : def;
+        }
+
+        PUGI__FN unsigned int xml_text::as_uint(unsigned int def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_uint(d->value) : def;
+        }
+
+        PUGI__FN double xml_text::as_double(double def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_double(d->value) : def;
+        }
+
+        PUGI__FN float xml_text::as_float(float def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_float(d->value) : def;
+        }
+
+        PUGI__FN bool xml_text::as_bool(bool def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_bool(d->value) : def;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN long long xml_text::as_llong(long long def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_llong(d->value) : def;
+        }
+
+        PUGI__FN unsigned long long xml_text::as_ullong(unsigned long long def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_ullong(d->value) : def;
+        }
+#endif
+
+        PUGI__FN bool xml_text::set(const char_t* rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::strcpy_insitu(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, impl::strlength(rhs)) : false;
+        }
+
+        PUGI__FN bool xml_text::set(int rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned int>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0) : false;
+        }
+
+        PUGI__FN bool xml_text::set(unsigned int rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned int>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, false) : false;
+        }
+
+        PUGI__FN bool xml_text::set(long rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned long>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0) : false;
+        }
+
+        PUGI__FN bool xml_text::set(unsigned long rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned long>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, false) : false;
+        }
+
+        PUGI__FN bool xml_text::set(float rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
+        }
+
+        PUGI__FN bool xml_text::set(double rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
+        }
+
+        PUGI__FN bool xml_text::set(bool rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_bool(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN bool xml_text::set(long long rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned long long>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0) : false;
+        }
+
+        PUGI__FN bool xml_text::set(unsigned long long rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned long long>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, false) : false;
+        }
+#endif
+
+        PUGI__FN xml_text& xml_text::operator=(const char_t* rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(int rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(unsigned int rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(long rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(unsigned long rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(double rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(float rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(bool rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN xml_text& xml_text::operator=(long long rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(unsigned long long rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+#endif
+
+        PUGI__FN xml_node xml_text::data() const
+        {
+                return xml_node(_data());
+        }
+
+#ifdef __BORLANDC__
+        PUGI__FN bool operator&&(const xml_text& lhs, bool rhs)
+        {
+                return (bool)lhs && rhs;
+        }
+
+        PUGI__FN bool operator||(const xml_text& lhs, bool rhs)
+        {
+                return (bool)lhs || rhs;
+        }
+#endif
+
+        PUGI__FN xml_node_iterator::xml_node_iterator()
+        {
+        }
+
+        PUGI__FN xml_node_iterator::xml_node_iterator(const xml_node& node): _wrap(node), _parent(node.parent())
+        {
+        }
+
+        PUGI__FN xml_node_iterator::xml_node_iterator(xml_node_struct* ref, xml_node_struct* parent): _wrap(ref), _parent(parent)
+        {
+        }
+
+        PUGI__FN bool xml_node_iterator::operator==(const xml_node_iterator& rhs) const
         {
                 return _wrap._root == rhs._wrap._root && _parent._root == rhs._parent._root;
         }
-        
-        bool xml_node_iterator::operator!=(const xml_node_iterator& rhs) const
+
+        PUGI__FN bool xml_node_iterator::operator!=(const xml_node_iterator& rhs) const
         {
                 return _wrap._root != rhs._wrap._root || _parent._root != rhs._parent._root;
         }
 
-        xml_node& xml_node_iterator::operator*()
+        PUGI__FN xml_node& xml_node_iterator::operator*() const
         {
                 assert(_wrap._root);
@@ -4361 +6595 @@
         }
 
-        xml_node* xml_node_iterator::operator->()
+        PUGI__FN xml_node* xml_node_iterator::operator->() const
         {
                 assert(_wrap._root);
-                return &_wrap;
-        }
-
-        const xml_node_iterator& xml_node_iterator::operator++()
+                return const_cast<xml_node*>(&_wrap); // BCC5 workaround
+        }
+
+        PUGI__FN const xml_node_iterator& xml_node_iterator::operator++()
         {
                 assert(_wrap._root);
@@ -4374 +6608 @@
         }
 
-        xml_node_iterator xml_node_iterator::operator++(int)
+        PUGI__FN xml_node_iterator xml_node_iterator::operator++(int)
         {
                 xml_node_iterator temp = *this;
@@ -4381 +6615 @@
         }
 
-        const xml_node_iterator& xml_node_iterator::operator--()
+        PUGI__FN const xml_node_iterator& xml_node_iterator::operator--()
         {
                 _wrap = _wrap._root ? _wrap.previous_sibling() : _parent.last_child();
@@ -4387 +6621 @@
         }
 
-        xml_node_iterator xml_node_iterator::operator--(int)
+        PUGI__FN xml_node_iterator xml_node_iterator::operator--(int)
         {
                 xml_node_iterator temp = *this;
@@ -4394 +6628 @@
         }
 
-        xml_attribute_iterator::xml_attribute_iterator()
-        {
-        }
-
-        xml_attribute_iterator::xml_attribute_iterator(const xml_attribute& attr, const xml_node& parent): _wrap(attr), _parent(parent)
-        {
-        }
-
-        xml_attribute_iterator::xml_attribute_iterator(xml_attribute_struct* ref, xml_node_struct* parent): _wrap(ref), _parent(parent)
-        {
-        }
-
-        bool xml_attribute_iterator::operator==(const xml_attribute_iterator& rhs) const
+        PUGI__FN xml_attribute_iterator::xml_attribute_iterator()
+        {
+        }
+
+        PUGI__FN xml_attribute_iterator::xml_attribute_iterator(const xml_attribute& attr, const xml_node& parent): _wrap(attr), _parent(parent)
+        {
+        }
+
+        PUGI__FN xml_attribute_iterator::xml_attribute_iterator(xml_attribute_struct* ref, xml_node_struct* parent): _wrap(ref), _parent(parent)
+        {
+        }
+
+        PUGI__FN bool xml_attribute_iterator::operator==(const xml_attribute_iterator& rhs) const
         {
                 return _wrap._attr == rhs._wrap._attr && _parent._root == rhs._parent._root;
         }
-        
-        bool xml_attribute_iterator::operator!=(const xml_attribute_iterator& rhs) const
+
+        PUGI__FN bool xml_attribute_iterator::operator!=(const xml_attribute_iterator& rhs) const
         {
                 return _wrap._attr != rhs._wrap._attr || _parent._root != rhs._parent._root;
         }
 
-        xml_attribute& xml_attribute_iterator::operator*()
+        PUGI__FN xml_attribute& xml_attribute_iterator::operator*() const
         {
                 assert(_wrap._attr);
@@ -4422 +6656 @@
         }
 
-        xml_attribute* xml_attribute_iterator::operator->()
+        PUGI__FN xml_attribute* xml_attribute_iterator::operator->() const
         {
                 assert(_wrap._attr);
-                return &_wrap;
-        }
-
-        const xml_attribute_iterator& xml_attribute_iterator::operator++()
+                return const_cast<xml_attribute*>(&_wrap); // BCC5 workaround
+        }
+
+        PUGI__FN const xml_attribute_iterator& xml_attribute_iterator::operator++()
         {
                 assert(_wrap._attr);
@@ -4435 +6669 @@
         }
 
-        xml_attribute_iterator xml_attribute_iterator::operator++(int)
+        PUGI__FN xml_attribute_iterator xml_attribute_iterator::operator++(int)
         {
                 xml_attribute_iterator temp = *this;
@@ -4442 +6676 @@
         }
 
-        const xml_attribute_iterator& xml_attribute_iterator::operator--()
+        PUGI__FN const xml_attribute_iterator& xml_attribute_iterator::operator--()
         {
                 _wrap = _wrap._attr ? _wrap.previous_attribute() : _parent.last_attribute();
@@ -4448 +6682 @@
         }
 
-        xml_attribute_iterator xml_attribute_iterator::operator--(int)
+        PUGI__FN xml_attribute_iterator xml_attribute_iterator::operator--(int)
         {
                 xml_attribute_iterator temp = *this;
@@ -4455 +6689 @@
         }
 
-    xml_parse_result::xml_parse_result(): status(status_internal_error), offset(0), encoding(encoding_auto)
-    {
-    }
-
-    xml_parse_result::operator bool() const
-    {
-        return status == status_ok;
-    }
-
-        const char* xml_parse_result::description() const
+        PUGI__FN xml_named_node_iterator::xml_named_node_iterator(): _name(0)
+        {
+        }
+
+        PUGI__FN xml_named_node_iterator::xml_named_node_iterator(const xml_node& node, const char_t* name): _wrap(node), _parent(node.parent()), _name(name)
+        {
+        }
+
+        PUGI__FN xml_named_node_iterator::xml_named_node_iterator(xml_node_struct* ref, xml_node_struct* parent, const char_t* name): _wrap(ref), _parent(parent), _name(name)
+        {
+        }
+
+        PUGI__FN bool xml_named_node_iterator::operator==(const xml_named_node_iterator& rhs) const
+        {
+                return _wrap._root == rhs._wrap._root && _parent._root == rhs._parent._root;
+        }
+
+        PUGI__FN bool xml_named_node_iterator::operator!=(const xml_named_node_iterator& rhs) const
+        {
+                return _wrap._root != rhs._wrap._root || _parent._root != rhs._parent._root;
+        }
+
+        PUGI__FN xml_node& xml_named_node_iterator::operator*() const
+        {
+                assert(_wrap._root);
+                return _wrap;
+        }
+
+        PUGI__FN xml_node* xml_named_node_iterator::operator->() const
+        {
+                assert(_wrap._root);
+                return const_cast<xml_node*>(&_wrap); // BCC5 workaround
+        }
+
+        PUGI__FN const xml_named_node_iterator& xml_named_node_iterator::operator++()
+        {
+                assert(_wrap._root);
+                _wrap = _wrap.next_sibling(_name);
+                return *this;
+        }
+
+        PUGI__FN xml_named_node_iterator xml_named_node_iterator::operator++(int)
+        {
+                xml_named_node_iterator temp = *this;
+                ++*this;
+                return temp;
+        }
+
+        PUGI__FN const xml_named_node_iterator& xml_named_node_iterator::operator--()
+        {
+                if (_wrap._root)
+                        _wrap = _wrap.previous_sibling(_name);
+                else
+                {
+                        _wrap = _parent.last_child();
+
+                        if (!impl::strequal(_wrap.name(), _name))
+                                _wrap = _wrap.previous_sibling(_name);
+                }
+
+                return *this;
+        }
+
+        PUGI__FN xml_named_node_iterator xml_named_node_iterator::operator--(int)
+        {
+                xml_named_node_iterator temp = *this;
+                --*this;
+                return temp;
+        }
+
+        PUGI__FN xml_parse_result::xml_parse_result(): status(status_internal_error), offset(0), encoding(encoding_auto)
+        {
+        }
+
+        PUGI__FN xml_parse_result::operator bool() const
+        {
+                return status == status_ok;
+        }
+
+        PUGI__FN const char* xml_parse_result::description() const
         {
                 switch (status)
@@ -4487 +6791 @@
                 case status_end_element_mismatch: return "Start-end tags mismatch";
 
+                case status_append_invalid_root: return "Unable to append nodes: root is not an element or document";
+
+                case status_no_document_element: return "No document element found";
+
                 default: return "Unknown error";
                 }
         }
 
-        xml_document::xml_document(): _buffer(0)
-        {
-                create();
-        }
-
-        xml_document::~xml_document()
-        {
-                destroy();
-        }
-
-        void xml_document::reset()
-        {
-                destroy();
-                create();
-        }
-
-    void xml_document::reset(const xml_document& proto)
-    {
-        reset();
-
-        for (xml_node cur = proto.first_child(); cur; cur = cur.next_sibling())
-            append_copy(cur);
-    }
-
-        void xml_document::create()
-        {
+        PUGI__FN xml_document::xml_document(): _buffer(0)
+        {
+                _create();
+        }
+
+        PUGI__FN xml_document::~xml_document()
+        {
+                _destroy();
+        }
+
+        PUGI__FN void xml_document::reset()
+        {
+                _destroy();
+                _create();
+        }
+
+        PUGI__FN void xml_document::reset(const xml_document& proto)
+        {
+                reset();
+
+                for (xml_node cur = proto.first_child(); cur; cur = cur.next_sibling())
+                        append_copy(cur);
+        }
+
+        PUGI__FN void xml_document::_create()
+        {
+                assert(!_root);
+
+        #ifdef PUGIXML_COMPACT
+                const size_t page_offset = sizeof(uint32_t);
+        #else
+                const size_t page_offset = 0;
+        #endif
+
                 // initialize sentinel page
-                STATIC_ASSERT(offsetof(xml_memory_page, data) + sizeof(xml_document_struct) + xml_memory_page_alignment <= sizeof(_memory));
-
-                // align upwards to page boundary
-                void* page_memory = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(_memory) + (xml_memory_page_alignment - 1)) & ~(xml_memory_page_alignment - 1));
+                PUGI__STATIC_ASSERT(sizeof(impl::xml_memory_page) + sizeof(impl::xml_document_struct) + page_offset <= sizeof(_memory));
 
                 // prepare page structure
-                xml_memory_page* page = xml_memory_page::construct(page_memory);
-
-                page->busy_size = xml_memory_page_size;
+                impl::xml_memory_page* page = impl::xml_memory_page::construct(_memory);
+                assert(page);
+
+                page->busy_size = impl::xml_memory_page_size;
+
+                // setup first page marker
+        #ifdef PUGIXML_COMPACT
+                // round-trip through void* to avoid 'cast increases required alignment of target type' warning
+                page->compact_page_marker = reinterpret_cast<uint32_t*>(static_cast<void*>(reinterpret_cast<char*>(page) + sizeof(impl::xml_memory_page)));
+                *page->compact_page_marker = sizeof(impl::xml_memory_page);
+        #endif
 
                 // allocate new root
-                _root = new (page->data) xml_document_struct(page);
+                _root = new (reinterpret_cast<char*>(page) + sizeof(impl::xml_memory_page) + page_offset) impl::xml_document_struct(page);
                 _root->prev_sibling_c = _root;
 
                 // setup sentinel page
-                page->allocator = static_cast<xml_document_struct*>(_root);
-        }
-
-        void xml_document::destroy()
-        {
+                page->allocator = static_cast<impl::xml_document_struct*>(_root);
+
+                // setup hash table pointer in allocator
+        #ifdef PUGIXML_COMPACT
+                page->allocator->_hash = &static_cast<impl::xml_document_struct*>(_root)->hash;
+        #endif
+
+                // verify the document allocation
+                assert(reinterpret_cast<char*>(_root) + sizeof(impl::xml_document_struct) <= _memory + sizeof(_memory));
+        }
+
+        PUGI__FN void xml_document::_destroy()
+        {
+                assert(_root);
+
                 // destroy static storage
                 if (_buffer)
                 {
-                        global_deallocate(_buffer);
+                        impl::xml_memory::deallocate(_buffer);
                         _buffer = 0;
                 }
 
+                // destroy extra buffers (note: no need to destroy linked list nodes, they're allocated using document allocator)
+                for (impl::xml_extra_buffer* extra = static_cast<impl::xml_document_struct*>(_root)->extra_buffers; extra; extra = extra->next)
+                {
+                        if (extra->buffer) impl::xml_memory::deallocate(extra->buffer);
+                }
+
                 // destroy dynamic storage, leave sentinel page (it's in static memory)
-                if (_root)
-                {
-                        xml_memory_page* root_page = reinterpret_cast<xml_memory_page*>(_root->header & xml_memory_page_pointer_mask);
-                        assert(root_page && !root_page->prev && !root_page->memory);
-
-                        // destroy all pages
-                        for (xml_memory_page* page = root_page->next; page; )
-                        {
-                                xml_memory_page* next = page->next;
-
-                                xml_allocator::deallocate_page(page);
-
-                                page = next;
-                        }
-
-                        // cleanup root page
-                        root_page->allocator = 0;
-                        root_page->next = 0;
-                        root_page->busy_size = root_page->freed_size = 0;
-
-                        _root = 0;
-                }
+                impl::xml_memory_page* root_page = PUGI__GETPAGE(_root);
+                assert(root_page && !root_page->prev);
+                assert(reinterpret_cast<char*>(root_page) >= _memory && reinterpret_cast<char*>(root_page) < _memory + sizeof(_memory));
+
+                for (impl::xml_memory_page* page = root_page->next; page; )
+                {
+                        impl::xml_memory_page* next = page->next;
+
+                        impl::xml_allocator::deallocate_page(page);
+
+                        page = next;
+                }
+
+        #ifdef PUGIXML_COMPACT
+                // destroy hash table
+                static_cast<impl::xml_document_struct*>(_root)->hash.clear();
+        #endif
+
+                _root = 0;
         }
 
 #ifndef PUGIXML_NO_STL
-        xml_parse_result xml_document::load(std::basic_istream<char, std::char_traits<char> >& stream, unsigned int options, xml_encoding encoding)
+        PUGI__FN xml_parse_result xml_document::load(std::basic_istream<char, std::char_traits<char> >& stream, unsigned int options, xml_encoding encoding)
         {
                 reset();
 
-                return load_stream_impl(*this, stream, options, encoding);
-        }
-
-        xml_parse_result xml_document::load(std::basic_istream<wchar_t, std::char_traits<wchar_t> >& stream, unsigned int options)
+                return impl::load_stream_impl(static_cast<impl::xml_document_struct*>(_root), stream, options, encoding, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load(std::basic_istream<wchar_t, std::char_traits<wchar_t> >& stream, unsigned int options)
         {
                 reset();
 
-                return load_stream_impl(*this, stream, options, encoding_wchar);
+                return impl::load_stream_impl(static_cast<impl::xml_document_struct*>(_root), stream, options, encoding_wchar, &_buffer);
         }
 #endif
 
-        xml_parse_result xml_document::load(const char_t* contents, unsigned int options)
+        PUGI__FN xml_parse_result xml_document::load_string(const char_t* contents, unsigned int options)
         {
                 // Force native encoding (skip autodetection)
@@ -4595 +6929 @@
         #endif
 
-                return load_buffer(contents, strlength(contents) * sizeof(char_t), options, encoding);
-        }
-
-        xml_parse_result xml_document::load_file(const char* path, unsigned int options, xml_encoding encoding)
+                return load_buffer(contents, impl::strlength(contents) * sizeof(char_t), options, encoding);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load(const char_t* contents, unsigned int options)
+        {
+                return load_string(contents, options);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_file(const char* path_, unsigned int options, xml_encoding encoding)
         {
                 reset();
 
-                FILE* file = fopen(path, "rb");
-
-                return load_file_impl(*this, file, options, encoding);
-        }
-
-        xml_parse_result xml_document::load_file(const wchar_t* path, unsigned int options, xml_encoding encoding)
+                using impl::auto_deleter; // MSVC7 workaround
+                auto_deleter<FILE> file(fopen(path_, "rb"), impl::close_file);
+
+                return impl::load_file_impl(static_cast<impl::xml_document_struct*>(_root), file.data, options, encoding, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_file(const wchar_t* path_, unsigned int options, xml_encoding encoding)
         {
                 reset();
 
-                FILE* file = open_file_wide(path, L"rb");
-
-                return load_file_impl(*this, file, options, encoding);
-        }
-
-        xml_parse_result xml_document::load_buffer_impl(void* contents, size_t size, unsigned int options, xml_encoding encoding, bool is_mutable, bool own)
+                using impl::auto_deleter; // MSVC7 workaround
+                auto_deleter<FILE> file(impl::open_file_wide(path_, L"rb"), impl::close_file);
+
+                return impl::load_file_impl(static_cast<impl::xml_document_struct*>(_root), file.data, options, encoding, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_buffer(const void* contents, size_t size, unsigned int options, xml_encoding encoding)
         {
                 reset();
 
-                // check input buffer
-                assert(contents || size == 0);
-
-                // get actual encoding
-                xml_encoding buffer_encoding = get_buffer_encoding(encoding, contents, size);
-
-                // get private buffer
-                char_t* buffer = 0;
-                size_t length = 0;
-
-                if (!convert_buffer(buffer, length, buffer_encoding, contents, size, is_mutable)) return make_parse_result(status_out_of_memory);
-                
-                // delete original buffer if we performed a conversion
-                if (own && buffer != contents && contents) global_deallocate(contents);
-
-                // parse
-                xml_parse_result res = xml_parser::parse(buffer, length, _root, options);
-
-                // remember encoding
-                res.encoding = buffer_encoding;
-
-                // grab onto buffer if it's our buffer, user is responsible for deallocating contens himself
-                if (own || buffer != contents) _buffer = buffer;
-
-                return res;
-        }
-
-        xml_parse_result xml_document::load_buffer(const void* contents, size_t size, unsigned int options, xml_encoding encoding)
-        {
-                return load_buffer_impl(const_cast<void*>(contents), size, options, encoding, false, false);
-        }
-
-        xml_parse_result xml_document::load_buffer_inplace(void* contents, size_t size, unsigned int options, xml_encoding encoding)
-        {
-                return load_buffer_impl(contents, size, options, encoding, true, false);
-        }
-                
-        xml_parse_result xml_document::load_buffer_inplace_own(void* contents, size_t size, unsigned int options, xml_encoding encoding)
-        {
-                return load_buffer_impl(contents, size, options, encoding, true, true);
-        }
-
-        void xml_document::save(xml_writer& writer, const char_t* indent, unsigned int flags, xml_encoding encoding) const
-        {
-                if (flags & format_write_bom) write_bom(writer, get_write_encoding(encoding));
-
-                xml_buffered_writer buffered_writer(writer, encoding);
-
-                if (!(flags & format_no_declaration) && !has_declaration(*this))
-                {
-                        buffered_writer.write(PUGIXML_TEXT("<?xml version=\"1.0\"?>"));
+                return impl::load_buffer_impl(static_cast<impl::xml_document_struct*>(_root), _root, const_cast<void*>(contents), size, options, encoding, false, false, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_buffer_inplace(void* contents, size_t size, unsigned int options, xml_encoding encoding)
+        {
+                reset();
+
+                return impl::load_buffer_impl(static_cast<impl::xml_document_struct*>(_root), _root, contents, size, options, encoding, true, false, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_buffer_inplace_own(void* contents, size_t size, unsigned int options, xml_encoding encoding)
+        {
+                reset();
+
+                return impl::load_buffer_impl(static_cast<impl::xml_document_struct*>(_root), _root, contents, size, options, encoding, true, true, &_buffer);
+        }
+
+        PUGI__FN void xml_document::save(xml_writer& writer, const char_t* indent, unsigned int flags, xml_encoding encoding) const
+        {
+                impl::xml_buffered_writer buffered_writer(writer, encoding);
+
+                if ((flags & format_write_bom) && encoding != encoding_latin1)
+                {
+                        // BOM always represents the codepoint U+FEFF, so just write it in native encoding
+                #ifdef PUGIXML_WCHAR_MODE
+                        unsigned int bom = 0xfeff;
+                        buffered_writer.write(static_cast<wchar_t>(bom));
+                #else
+                        buffered_writer.write('\xef', '\xbb', '\xbf');
+                #endif
+                }
+
+                if (!(flags & format_no_declaration) && !impl::has_declaration(_root))
+                {
+                        buffered_writer.write_string(PUGIXML_TEXT("<?xml version=\"1.0\""));
+                        if (encoding == encoding_latin1) buffered_writer.write_string(PUGIXML_TEXT(" encoding=\"ISO-8859-1\""));
+                        buffered_writer.write('?', '>');
                         if (!(flags & format_raw)) buffered_writer.write('\n');
                 }
 
-                node_output(buffered_writer, *this, indent, flags, 0);
+                impl::node_output(buffered_writer, _root, indent, flags, 0);
+
+                buffered_writer.flush();
         }
 
 #ifndef PUGIXML_NO_STL
-        void xml_document::save(std::basic_ostream<char, std::char_traits<char> >& stream, const char_t* indent, unsigned int flags, xml_encoding encoding) const
+        PUGI__FN void xml_document::save(std::basic_ostream<char, std::char_traits<char> >& stream, const char_t* indent, unsigned int flags, xml_encoding encoding) const
         {
                 xml_writer_stream writer(stream);
@@ -4685 +7014 @@
         }
 
-        void xml_document::save(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream, const char_t* indent, unsigned int flags) const
+        PUGI__FN void xml_document::save(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream, const char_t* indent, unsigned int flags) const
         {
                 xml_writer_stream writer(stream);
@@ -4693 +7022 @@
 #endif
 
-        bool xml_document::save_file(const char* path, const char_t* indent, unsigned int flags, xml_encoding encoding) const
-        {
-                FILE* file = fopen(path, "wb");
-                if (!file) return false;
-
-                xml_writer_file writer(file);
-                save(writer, indent, flags, encoding);
-
-                fclose(file);
-
-                return true;
-        }
-
-        bool xml_document::save_file(const wchar_t* path, const char_t* indent, unsigned int flags, xml_encoding encoding) const
-        {
-                FILE* file = open_file_wide(path, L"wb");
-                if (!file) return false;
-
-                xml_writer_file writer(file);
-                save(writer, indent, flags, encoding);
-
-                fclose(file);
-
-                return true;
-        }
-
-    xml_node xml_document::document_element() const
-    {
+        PUGI__FN bool xml_document::save_file(const char* path_, const char_t* indent, unsigned int flags, xml_encoding encoding) const
+        {
+                using impl::auto_deleter; // MSVC7 workaround
+                auto_deleter<FILE> file(fopen(path_, (flags & format_save_file_text) ? "w" : "wb"), impl::close_file);
+
+                return impl::save_file_impl(*this, file.data, indent, flags, encoding);
+        }
+
+        PUGI__FN bool xml_document::save_file(const wchar_t* path_, const char_t* indent, unsigned int flags, xml_encoding encoding) const
+        {
+                using impl::auto_deleter; // MSVC7 workaround
+                auto_deleter<FILE> file(impl::open_file_wide(path_, (flags & format_save_file_text) ? L"w" : L"wb"), impl::close_file);
+
+                return impl::save_file_impl(*this, file.data, indent, flags, encoding);
+        }
+
+        PUGI__FN xml_node xml_document::document_element() const
+        {
+                assert(_root);
+
                 for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
-                        if ((i->header & xml_memory_page_type_mask) + 1 == node_element)
-                return xml_node(i);
-
-        return xml_node();
-    }
+                        if (PUGI__NODETYPE(i) == node_element)
+                                return xml_node(i);
+
+                return xml_node();
+        }
 
 #ifndef PUGIXML_NO_STL
-        std::string PUGIXML_FUNCTION as_utf8(const wchar_t* str)
+        PUGI__FN std::string PUGIXML_FUNCTION as_utf8(const wchar_t* str)
         {
                 assert(str);
 
-        return as_utf8_impl(str, wcslen(str));
-        }
-
-        std::string PUGIXML_FUNCTION as_utf8(const std::wstring& str)
-        {
-        return as_utf8_impl(str.c_str(), str.size());
-        }
-        
-        std::wstring PUGIXML_FUNCTION as_wide(const char* str)
+                return impl::as_utf8_impl(str, impl::strlength_wide(str));
+        }
+
+        PUGI__FN std::string PUGIXML_FUNCTION as_utf8(const std::basic_string<wchar_t>& str)
+        {
+                return impl::as_utf8_impl(str.c_str(), str.size());
+        }
+
+        PUGI__FN std::basic_string<wchar_t> PUGIXML_FUNCTION as_wide(const char* str)
         {
                 assert(str);
 
-        return as_wide_impl(str, strlen(str));
-        }
-        
-        std::wstring PUGIXML_FUNCTION as_wide(const std::string& str)
-        {
-        return as_wide_impl(str.c_str(), str.size());
+                return impl::as_wide_impl(str, strlen(str));
+        }
+
+        PUGI__FN std::basic_string<wchar_t> PUGIXML_FUNCTION as_wide(const std::string& str)
+        {
+                return impl::as_wide_impl(str.c_str(), str.size());
         }
 #endif
 
-    void PUGIXML_FUNCTION set_memory_management_functions(allocation_function allocate, deallocation_function deallocate)
-    {
-        global_allocate = allocate;
-        global_deallocate = deallocate;
-    }
-
-    allocation_function PUGIXML_FUNCTION get_memory_allocation_function()
-    {
-        return global_allocate;
-    }
-
-    deallocation_function PUGIXML_FUNCTION get_memory_deallocation_function()
-    {
-        return global_deallocate;
-    }
+        PUGI__FN void PUGIXML_FUNCTION set_memory_management_functions(allocation_function allocate, deallocation_function deallocate)
+        {
+                impl::xml_memory::allocate = allocate;
+                impl::xml_memory::deallocate = deallocate;
+        }
+
+        PUGI__FN allocation_function PUGIXML_FUNCTION get_memory_allocation_function()
+        {
+                return impl::xml_memory::allocate;
+        }
+
+        PUGI__FN deallocation_function PUGIXML_FUNCTION get_memory_deallocation_function()
+        {
+                return impl::xml_memory::deallocate;
+        }
 }
 
@@ -4775 +7096 @@
 {
         // Workarounds for (non-standard) iterator category detection for older versions (MSVC7/IC8 and earlier)
-        std::bidirectional_iterator_tag _Iter_cat(const xml_node_iterator&)
+        PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_node_iterator&)
         {
                 return std::bidirectional_iterator_tag();
         }
 
-        std::bidirectional_iterator_tag _Iter_cat(const xml_attribute_iterator&)
+        PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_attribute_iterator&)
+        {
+                return std::bidirectional_iterator_tag();
+        }
+
+        PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_named_node_iterator&)
         {
                 return std::bidirectional_iterator_tag();
@@ -4791 +7117 @@
 {
         // Workarounds for (non-standard) iterator category detection
-        std::bidirectional_iterator_tag __iterator_category(const xml_node_iterator&)
+        PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_node_iterator&)
         {
                 return std::bidirectional_iterator_tag();
         }
 
-        std::bidirectional_iterator_tag __iterator_category(const xml_attribute_iterator&)
+        PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_attribute_iterator&)
+        {
+                return std::bidirectional_iterator_tag();
+        }
+
+        PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_named_node_iterator&)
         {
                 return std::bidirectional_iterator_tag();
@@ -4804 +7135 @@
 
 #ifndef PUGIXML_NO_XPATH
-
 // STL replacements
-namespace
-{
+PUGI__NS_BEGIN
         struct equal_to
         {
@@ -4860 +7189 @@
         template <typename I> void reverse(I begin, I end)
         {
-                while (begin + 1 < end) swap(*begin++, *--end);
+                while (end - begin > 1) swap(*begin++, *--end);
         }
 
@@ -4866 +7195 @@
         {
                 // fast skip head
-                while (begin + 1 < end && *begin != *(begin + 1)) begin++;
+                while (end - begin > 1 && *begin != *(begin + 1)) begin++;
 
                 if (begin == end) return begin;
 
                 // last written element
-                I write = begin++; 
+                I write = begin++;
 
                 // merge unique elements
@@ -5031 +7360 @@
                 if (begin != end) insertion_sort(begin, end, pred, &*begin);
         }
-}
+PUGI__NS_END
 
 // Allocator used for AST and evaluation stacks
-namespace
-{
+PUGI__NS_BEGIN
+        static const size_t xpath_memory_page_size =
+        #ifdef PUGIXML_MEMORY_XPATH_PAGE_SIZE
+                PUGIXML_MEMORY_XPATH_PAGE_SIZE
+        #else
+                4096
+        #endif
+                ;
+
+        static const uintptr_t xpath_memory_block_alignment = sizeof(double) > sizeof(void*) ? sizeof(double) : sizeof(void*);
+
         struct xpath_memory_block
-        {       
+        {
                 xpath_memory_block* next;
-
-                char data[4096];
+                size_t capacity;
+
+                union
+                {
+                        char data[xpath_memory_page_size];
+                        double alignment;
+                };
         };
-                
+
         class xpath_allocator
         {
@@ -5059 +7402 @@
                 #endif
                 }
-                
+
                 void* allocate_nothrow(size_t size)
                 {
-                        const size_t block_capacity = sizeof(_root->data);
-
-                        // align size so that we're able to store pointers in subsequent blocks
-                        size = (size + sizeof(void*) - 1) & ~(sizeof(void*) - 1);
-
-                        if (_root_size + size <= block_capacity)
-                        {
-                                void* buf = _root->data + _root_size;
+                        // round size up to block alignment boundary
+                        size = (size + xpath_memory_block_alignment - 1) & ~(xpath_memory_block_alignment - 1);
+
+                        if (_root_size + size <= _root->capacity)
+                        {
+                                void* buf = &_root->data[0] + _root_size;
                                 _root_size += size;
                                 return buf;
@@ -5075 +7416 @@
                         else
                         {
-                                size_t block_data_size = (size > block_capacity) ? size : block_capacity;
-                                size_t block_size = block_data_size + offsetof(xpath_memory_block, data);
-
-                                xpath_memory_block* block = static_cast<xpath_memory_block*>(global_allocate(block_size));
+                                // make sure we have at least 1/4th of the page free after allocation to satisfy subsequent allocation requests
+                                size_t block_capacity_base = sizeof(_root->data);
+                                size_t block_capacity_req = size + block_capacity_base / 4;
+                                size_t block_capacity = (block_capacity_base > block_capacity_req) ? block_capacity_base : block_capacity_req;
+
+                                size_t block_size = block_capacity + offsetof(xpath_memory_block, data);
+
+                                xpath_memory_block* block = static_cast<xpath_memory_block*>(xml_memory::allocate(block_size));
                                 if (!block) return 0;
-                                
+
                                 block->next = _root;
-                                
+                                block->capacity = block_capacity;
+
                                 _root = block;
                                 _root_size = size;
-                                
+
                                 return block->data;
                         }
@@ -5109 +7455 @@
                 void* reallocate(void* ptr, size_t old_size, size_t new_size)
                 {
-                        // align size so that we're able to store pointers in subsequent blocks
-                        old_size = (old_size + sizeof(void*) - 1) & ~(sizeof(void*) - 1);
-                        new_size = (new_size + sizeof(void*) - 1) & ~(sizeof(void*) - 1);
+                        // round size up to block alignment boundary
+                        old_size = (old_size + xpath_memory_block_alignment - 1) & ~(xpath_memory_block_alignment - 1);
+                        new_size = (new_size + xpath_memory_block_alignment - 1) & ~(xpath_memory_block_alignment - 1);
 
                         // we can only reallocate the last object
-                        assert(ptr == 0 || static_cast<char*>(ptr) + old_size == _root->data + _root_size);
+                        assert(ptr == 0 || static_cast<char*>(ptr) + old_size == &_root->data[0] + _root_size);
 
                         // adjust root size so that we have not allocated the object at all
@@ -5129 +7475 @@
                         {
                                 // copy old data
-                                assert(new_size > old_size);
+                                assert(new_size >= old_size);
                                 memcpy(result, ptr, old_size);
 
@@ -5143 +7489 @@
                                         {
                                                 // deallocate the whole page, unless it was the first one
-                                                global_deallocate(_root->next);
+                                                xml_memory::deallocate(_root->next);
                                                 _root->next = next;
                                         }
@@ -5161 +7507 @@
                                 xpath_memory_block* next = cur->next;
 
-                                global_deallocate(cur);
+                                xml_memory::deallocate(cur);
 
                                 cur = next;
@@ -5180 +7526 @@
                                 xpath_memory_block* next = cur->next;
 
-                                global_deallocate(cur);
+                                xml_memory::deallocate(cur);
 
                                 cur = next;
@@ -5222 +7568 @@
                 {
                         blocks[0].next = blocks[1].next = 0;
+                        blocks[0].capacity = blocks[1].capacity = sizeof(blocks[0].data);
 
                         stack.result = &result;
@@ -5237 +7584 @@
                 }
         };
-}
+PUGI__NS_END
 
 // String class
-namespace
-{
+PUGI__NS_BEGIN
         class xpath_string
         {
                 const char_t* _buffer;
                 bool _uses_heap;
+                size_t _length_heap;
 
                 static char_t* duplicate_string(const char_t* string, size_t length, xpath_allocator* alloc)
@@ -5258 +7605 @@
                 }
 
-                static char_t* duplicate_string(const char_t* string, xpath_allocator* alloc)
-                {
-                        return duplicate_string(string, strlength(string), alloc);
+                xpath_string(const char_t* buffer, bool uses_heap_, size_t length_heap): _buffer(buffer), _uses_heap(uses_heap_), _length_heap(length_heap)
+                {
                 }
 
         public:
-                xpath_string(): _buffer(PUGIXML_TEXT("")), _uses_heap(false)
-                {
-                }
-
-                explicit xpath_string(const char_t* str, xpath_allocator* alloc)
-                {
-                        bool empty = (*str == 0);
-
-                        _buffer = empty ? PUGIXML_TEXT("") : duplicate_string(str, alloc);
-                        _uses_heap = !empty;
-                }
-
-                explicit xpath_string(const char_t* str, bool use_heap): _buffer(str), _uses_heap(use_heap)
-                {
-                }
-
-                xpath_string(const char_t* begin, const char_t* end, xpath_allocator* alloc)
+                static xpath_string from_const(const char_t* str)
+                {
+                        return xpath_string(str, false, 0);
+                }
+
+                static xpath_string from_heap_preallocated(const char_t* begin, const char_t* end)
+                {
+                        assert(begin <= end && *end == 0);
+
+                        return xpath_string(begin, true, static_cast<size_t>(end - begin));
+                }
+
+                static xpath_string from_heap(const char_t* begin, const char_t* end, xpath_allocator* alloc)
                 {
                         assert(begin <= end);
 
-                        bool empty = (begin == end);
-
-                        _buffer = empty ? PUGIXML_TEXT("") : duplicate_string(begin, static_cast<size_t>(end - begin), alloc);
-                        _uses_heap = !empty;
+                        size_t length = static_cast<size_t>(end - begin);
+
+                        return length == 0 ? xpath_string() : xpath_string(duplicate_string(begin, length, alloc), true, length);
+                }
+
+                xpath_string(): _buffer(PUGIXML_TEXT("")), _uses_heap(false), _length_heap(0)
+                {
                 }
 
@@ -5303 +7648 @@
                         {
                                 // need to make heap copy
-                                size_t target_length = strlength(_buffer);
-                                size_t source_length = strlength(o._buffer);
-                                size_t length = target_length + source_length;
+                                size_t target_length = length();
+                                size_t source_length = o.length();
+                                size_t result_length = target_length + source_length;
 
                                 // allocate new buffer
-                                char_t* result = static_cast<char_t*>(alloc->reallocate(_uses_heap ? const_cast<char_t*>(_buffer) : 0, (target_length + 1) * sizeof(char_t), (length + 1) * sizeof(char_t)));
+                                char_t* result = static_cast<char_t*>(alloc->reallocate(_uses_heap ? const_cast<char_t*>(_buffer) : 0, (target_length + 1) * sizeof(char_t), (result_length + 1) * sizeof(char_t)));
                                 assert(result);
 
@@ -5316 +7661 @@
                                 // append second string to the new buffer
                                 memcpy(result + target_length, o._buffer, source_length * sizeof(char_t));
-                                result[length] = 0;
+                                result[result_length] = 0;
 
                                 // finalize
                                 _buffer = result;
                                 _uses_heap = true;
+                                _length_heap = result_length;
                         }
                 }
@@ -5331 +7677 @@
                 size_t length() const
                 {
-                        return strlength(_buffer);
-                }
-                
+                        return _uses_heap ? _length_heap : strlength(_buffer);
+                }
+
                 char_t* data(xpath_allocator* alloc)
                 {
@@ -5339 +7685 @@
                         if (!_uses_heap)
                         {
-                                _buffer = duplicate_string(_buffer, alloc);
+                                size_t length_ = strlength(_buffer);
+
+                                _buffer = duplicate_string(_buffer, length_, alloc);
                                 _uses_heap = true;
+                                _length_heap = length_;
                         }
 
@@ -5366 +7715 @@
                 }
         };
-
-        xpath_string xpath_string_const(const char_t* str)
-        {
-                return xpath_string(str, false);
-        }
-}
-
-namespace
-{
-        bool starts_with(const char_t* string, const char_t* pattern)
+PUGI__NS_END
+
+PUGI__NS_BEGIN
+        PUGI__FN bool starts_with(const char_t* string, const char_t* pattern)
         {
                 while (*pattern && *string == *pattern)
@@ -5386 +7729 @@
         }
 
-        const char_t* find_char(const char_t* s, char_t c)
+        PUGI__FN const char_t* find_char(const char_t* s, char_t c)
         {
         #ifdef PUGIXML_WCHAR_MODE
@@ -5395 +7738 @@
         }
 
-        const char_t* find_substring(const char_t* s, const char_t* p)
+        PUGI__FN const char_t* find_substring(const char_t* s, const char_t* p)
         {
         #ifdef PUGIXML_WCHAR_MODE
@@ -5406 +7749 @@
 
         // Converts symbol to lower case, if it is an ASCII one
-        char_t tolower_ascii(char_t ch)
+        PUGI__FN char_t tolower_ascii(char_t ch)
         {
                 return static_cast<unsigned int>(ch - 'A') < 26 ? static_cast<char_t>(ch | ' ') : ch;
         }
 
-        xpath_string string_value(const xpath_node& na, xpath_allocator* alloc)
+        PUGI__FN xpath_string string_value(const xpath_node& na, xpath_allocator* alloc)
         {
                 if (na.attribute())
-                        return xpath_string_const(na.attribute().value());
+                        return xpath_string::from_const(na.attribute().value());
                 else
                 {
-                        const xml_node& n = na.node();
+                        xml_node n = na.node();
 
                         switch (n.type())
@@ -5425 +7768 @@
                         case node_comment:
                         case node_pi:
-                                return xpath_string_const(n.value());
-                        
+                                return xpath_string::from_const(n.value());
+
                         case node_document:
                         case node_element:
@@ -5432 +7775 @@
                                 xpath_string result;
 
+                                // element nodes can have value if parse_embed_pcdata was used
+                                if (n.value()[0])
+                                        result.append(xpath_string::from_const(n.value()), alloc);
+
                                 xml_node cur = n.first_child();
-                                
+
                                 while (cur && cur != n)
                                 {
                                         if (cur.type() == node_pcdata || cur.type() == node_cdata)
-                                                result.append(xpath_string_const(cur.value()), alloc);
+                                                result.append(xpath_string::from_const(cur.value()), alloc);
 
                                         if (cur.first_child())
@@ -5451 +7798 @@
                                         }
                                 }
-                                
+
                                 return result;
                         }
-                        
+
                         default:
                                 return xpath_string();
@@ -5460 +7807 @@
                 }
         }
-        
-        unsigned int node_height(xml_node n)
-        {
-            unsigned int result = 0;
-            
-            while (n)
-            {
-                ++result;
-                n = n.parent();
-            }
-            
-            return result;
-        }
-        
-        bool node_is_before(xml_node ln, unsigned int lh, xml_node rn, unsigned int rh)
-        {
-                // normalize heights
-                for (unsigned int i = rh; i < lh; i++) ln = ln.parent();
-                for (unsigned int j = lh; j < rh; j++) rn = rn.parent();
-            
+
+        PUGI__FN bool node_is_before_sibling(xml_node_struct* ln, xml_node_struct* rn)
+        {
+                assert(ln->parent == rn->parent);
+
+                // there is no common ancestor (the shared parent is null), nodes are from different documents
+                if (!ln->parent) return ln < rn;
+
+                // determine sibling order
+                xml_node_struct* ls = ln;
+                xml_node_struct* rs = rn;
+
+                while (ls && rs)
+                {
+                        if (ls == rn) return true;
+                        if (rs == ln) return false;
+
+                        ls = ls->next_sibling;
+                        rs = rs->next_sibling;
+                }
+
+                // if rn sibling chain ended ln must be before rn
+                return !rs;
+        }
+
+        PUGI__FN bool node_is_before(xml_node_struct* ln, xml_node_struct* rn)
+        {
+                // find common ancestor at the same depth, if any
+                xml_node_struct* lp = ln;
+                xml_node_struct* rp = rn;
+
+                while (lp && rp && lp->parent != rp->parent)
+                {
+                        lp = lp->parent;
+                        rp = rp->parent;
+                }
+
+                // parents are the same!
+                if (lp && rp) return node_is_before_sibling(lp, rp);
+
+                // nodes are at different depths, need to normalize heights
+                bool left_higher = !lp;
+
+                while (lp)
+                {
+                        lp = lp->parent;
+                        ln = ln->parent;
+                }
+
+                while (rp)
+                {
+                        rp = rp->parent;
+                        rn = rn->parent;
+                }
+
                 // one node is the ancestor of the other
-            if (ln == rn) return lh < rh;
-            
-                // find common ancestor
-            while (ln.parent() != rn.parent())
-            {
-                ln = ln.parent();
-                rn = rn.parent();
-            }
-
-                // there is no common ancestor (the shared parent is null), nodes are from different documents
-                if (!ln.parent()) return ln < rn;
-
-                // determine sibling order
-        for (; ln; ln = ln.next_sibling())
-            if (ln == rn)
-                return true;
-                
-        return false;
-    }
-
-    bool node_is_ancestor(xml_node parent, xml_node node)
-    {
-        while (node && node != parent) node = node.parent();
-
-        return parent && node == parent;
-    }
-
-    const void* document_order(const xpath_node& xnode)
-    {
-        xml_node_struct* node = xnode.node().internal_object();
-
-        if (node)
-        {
-            if (node->name && (node->header & xml_memory_page_name_allocated_mask) == 0) return node->name;
-            if (node->value && (node->header & xml_memory_page_value_allocated_mask) == 0) return node->value;
-            return 0;
-        }
-
-        xml_attribute_struct* attr = xnode.attribute().internal_object();
-
-        if (attr)
-        {
-            if ((attr->header & xml_memory_page_name_allocated_mask) == 0) return attr->name;
-            if ((attr->header & xml_memory_page_value_allocated_mask) == 0) return attr->value;
-            return 0;
-        }
+                if (ln == rn) return left_higher;
+
+                // find common ancestor... again
+                while (ln->parent != rn->parent)
+                {
+                        ln = ln->parent;
+                        rn = rn->parent;
+                }
+
+                return node_is_before_sibling(ln, rn);
+        }
+
+        PUGI__FN bool node_is_ancestor(xml_node_struct* parent, xml_node_struct* node)
+        {
+                while (node && node != parent) node = node->parent;
+
+                return parent && node == parent;
+        }
+
+        PUGI__FN const void* document_buffer_order(const xpath_node& xnode)
+        {
+                xml_node_struct* node = xnode.node().internal_object();
+
+                if (node)
+                {
+                        if ((get_document(node).header & xml_memory_page_contents_shared_mask) == 0)
+                        {
+                                if (node->name && (node->header & impl::xml_memory_page_name_allocated_or_shared_mask) == 0) return node->name;
+                                if (node->value && (node->header & impl::xml_memory_page_value_allocated_or_shared_mask) == 0) return node->value;
+                        }
+
+                        return 0;
+                }
+
+                xml_attribute_struct* attr = xnode.attribute().internal_object();
+
+                if (attr)
+                {
+                        if ((get_document(attr).header & xml_memory_page_contents_shared_mask) == 0)
+                        {
+                                if ((attr->header & impl::xml_memory_page_name_allocated_or_shared_mask) == 0) return attr->name;
+                                if ((attr->header & impl::xml_memory_page_value_allocated_or_shared_mask) == 0) return attr->value;
+                        }
+
+                        return 0;
+                }
 
                 return 0;
-    }
-    
+        }
+
         struct document_order_comparator
         {
@@ -5536 +7918 @@
                 {
                         // optimized document order based check
-                        const void* lo = document_order(lhs);
-                        const void* ro = document_order(rhs);
+                        const void* lo = document_buffer_order(lhs);
+                        const void* ro = document_buffer_order(rhs);
 
                         if (lo && ro) return lo < ro;
 
-            // slow comparison
+                        // slow comparison
                         xml_node ln = lhs.node(), rn = rhs.node();
 
@@ -5551 +7933 @@
                                 {
                                         // determine sibling order
-                                    for (xml_attribute a = lhs.attribute(); a; a = a.next_attribute())
-                                        if (a == rhs.attribute())
-                                            return true;
-                                    
-                                    return false;
-                                }
-                                
+                                        for (xml_attribute a = lhs.attribute(); a; a = a.next_attribute())
+                                                if (a == rhs.attribute())
+                                                        return true;
+
+                                        return false;
+                                }
+
                                 // compare attribute parents
                                 ln = lhs.parent();
@@ -5566 +7948 @@
                                 // attributes go after the parent element
                                 if (lhs.parent() == rhs.node()) return false;
-                                
+
                                 ln = lhs.parent();
                         }
@@ -5573 +7955 @@
                                 // attributes go after the parent element
                                 if (rhs.parent() == lhs.node()) return true;
-                                
+
                                 rn = rhs.parent();
                         }
 
                         if (ln == rn) return false;
-                        
-                        unsigned int lh = node_height(ln);
-                        unsigned int rh = node_height(rn);
-                        
-                        return node_is_before(ln, lh, rn, rh);
+
+                        if (!ln || !rn) return ln < rn;
+
+                        return node_is_before(ln.internal_object(), rn.internal_object());
                 }
         };
@@ -5594 +7975 @@
                 }
         };
-        
-        double gen_nan()
+
+        PUGI__FN double gen_nan()
         {
         #if defined(__STDC_IEC_559__) || ((FLT_RADIX - 0 == 2) && (FLT_MAX_EXP - 0 == 128) && (FLT_MANT_DIG - 0 == 24))
-                union { float f; int32_t i; } u[sizeof(float) == sizeof(int32_t) ? 1 : -1];
-                u[0].i = 0x7fc00000;
-                return u[0].f;
+                PUGI__STATIC_ASSERT(sizeof(float) == sizeof(uint32_t));
+                typedef uint32_t UI; // BCC5 workaround
+                union { float f; UI i; } u;
+                u.i = 0x7fc00000;
+                return u.f;
         #else
                 // fallback
@@ -5607 +7990 @@
         #endif
         }
-        
-        bool is_nan(double value)
-        {
-        #if defined(_MSC_VER) || defined(__BORLANDC__)
+
+        PUGI__FN bool is_nan(double value)
+        {
+        #if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__)
                 return !!_isnan(value);
         #elif defined(fpclassify) && defined(FP_NAN)
@@ -5620 +8003 @@
         #endif
         }
-        
-        const char_t* convert_number_to_string_special(double value)
-        {
-        #if defined(_MSC_VER) || defined(__BORLANDC__)
+
+        PUGI__FN const char_t* convert_number_to_string_special(double value)
+        {
+        #if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__)
                 if (_finite(value)) return (value == 0) ? PUGIXML_TEXT("0") : 0;
                 if (_isnan(value)) return PUGIXML_TEXT("NaN");
-                return PUGIXML_TEXT("-Infinity") + (value > 0);
+                return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
         #elif defined(fpclassify) && defined(FP_NAN) && defined(FP_INFINITE) && defined(FP_ZERO)
                 switch (fpclassify(value))
@@ -5634 +8017 @@
 
                 case FP_INFINITE:
-                        return PUGIXML_TEXT("-Infinity") + (value > 0);
+                        return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
 
                 case FP_ZERO:
@@ -5648 +8031 @@
                 if (v == 0) return PUGIXML_TEXT("0");
                 if (v != v) return PUGIXML_TEXT("NaN");
-                if (v * 2 == v) return PUGIXML_TEXT("-Infinity") + (value > 0);
+                if (v * 2 == v) return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
                 return 0;
         #endif
         }
-        
-        bool convert_number_to_boolean(double value)
+
+        PUGI__FN bool convert_number_to_boolean(double value)
         {
                 return (value != 0 && !is_nan(value));
         }
-        
-        void truncate_zeros(char* begin, char* end)
+
+        PUGI__FN void truncate_zeros(char* begin, char* end)
         {
                 while (begin != end && end[-1] == '0') end--;
@@ -5666 +8049 @@
 
         // gets mantissa digits in the form of 0.xxxxx with 0. implied and the exponent
-#if defined(_MSC_VER) && _MSC_VER >= 1400
-        void convert_number_to_mantissa_exponent(double value, char* buffer, size_t buffer_size, char** out_mantissa, int* out_exponent)
+#if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 && !defined(_WIN32_WCE)
+        PUGI__FN void convert_number_to_mantissa_exponent(double value, char* buffer, size_t buffer_size, char** out_mantissa, int* out_exponent)
         {
                 // get base values
@@ -5681 +8064 @@
         }
 #else
-        void convert_number_to_mantissa_exponent(double value, char* buffer, size_t buffer_size, char** out_mantissa, int* out_exponent)
+        PUGI__FN void convert_number_to_mantissa_exponent(double value, char* buffer, size_t buffer_size, char** out_mantissa, int* out_exponent)
         {
                 // get a scientific notation value with IEEE DBL_DIG decimals
@@ -5712 +8095 @@
 #endif
 
-        xpath_string convert_number_to_string(double value, xpath_allocator* alloc)
+        PUGI__FN xpath_string convert_number_to_string(double value, xpath_allocator* alloc)
         {
                 // try special number conversion
                 const char_t* special = convert_number_to_string_special(value);
-                if (special) return xpath_string_const(special);
+                if (special) return xpath_string::from_const(special);
 
                 // get mantissa + exponent form
-                char mantissa_buffer[64];
+                char mantissa_buffer[32];
 
                 char* mantissa;
@@ -5725 +8108 @@
                 convert_number_to_mantissa_exponent(value, mantissa_buffer, sizeof(mantissa_buffer), &mantissa, &exponent);
 
+                // allocate a buffer of suitable length for the number
+                size_t result_size = strlen(mantissa_buffer) + (exponent > 0 ? exponent : -exponent) + 4;
+                char_t* result = static_cast<char_t*>(alloc->allocate(sizeof(char_t) * result_size));
+                assert(result);
+
                 // make the number!
-                char_t result[512];
                 char_t* s = result;
 
@@ -5741 +8128 @@
                         while (exponent > 0)
                         {
-                                assert(*mantissa == 0 || (unsigned)(*mantissa - '0') <= 9);
+                                assert(*mantissa == 0 || static_cast<unsigned int>(static_cast<unsigned int>(*mantissa) - '0') <= 9);
                                 *s++ = *mantissa ? *mantissa++ : '0';
                                 exponent--;
@@ -5763 +8150 @@
                         while (*mantissa)
                         {
-                                assert((unsigned)(*mantissa - '0') <= 9);
+                                assert(static_cast<unsigned int>(*mantissa - '0') <= 9);
                                 *s++ = *mantissa++;
                         }
@@ -5769 +8156 @@
 
                 // zero-terminate
-                assert(s < result + sizeof(result) / sizeof(result[0]));
+                assert(s < result + result_size);
                 *s = 0;
 
-                return xpath_string(result, alloc);
-        }
-        
-        bool check_string_to_number_format(const char_t* string)
+                return xpath_string::from_heap_preallocated(result, s);
+        }
+
+        PUGI__FN bool check_string_to_number_format(const char_t* string)
         {
                 // parse leading whitespace
-                while (IS_CHARTYPE(*string, ct_space)) ++string;
+                while (PUGI__IS_CHARTYPE(*string, ct_space)) ++string;
 
                 // parse sign
@@ -5786 +8173 @@
 
                 // if there is no integer part, there should be a decimal part with at least one digit
-                if (!IS_CHARTYPEX(string[0], ctx_digit) && (string[0] != '.' || !IS_CHARTYPEX(string[1], ctx_digit))) return false;
+                if (!PUGI__IS_CHARTYPEX(string[0], ctx_digit) && (string[0] != '.' || !PUGI__IS_CHARTYPEX(string[1], ctx_digit))) return false;
 
                 // parse integer part
-                while (IS_CHARTYPEX(*string, ctx_digit)) ++string;
+                while (PUGI__IS_CHARTYPEX(*string, ctx_digit)) ++string;
 
                 // parse decimal part
@@ -5796 +8183 @@
                         ++string;
 
-                        while (IS_CHARTYPEX(*string, ctx_digit)) ++string;
+                        while (PUGI__IS_CHARTYPEX(*string, ctx_digit)) ++string;
                 }
 
                 // parse trailing whitespace
-                while (IS_CHARTYPE(*string, ct_space)) ++string;
+                while (PUGI__IS_CHARTYPE(*string, ct_space)) ++string;
 
                 return *string == 0;
         }
 
-        double convert_string_to_number(const char_t* string)
+        PUGI__FN double convert_string_to_number(const char_t* string)
         {
                 // check string format
@@ -5814 +8201 @@
                 return wcstod(string, 0);
         #else
-                return atof(string);
+                return strtod(string, 0);
         #endif
         }
 
-        bool convert_string_to_number(const char_t* begin, const char_t* end, double* out_result)
-        {
-                char_t buffer[32];
-
+        PUGI__FN bool convert_string_to_number_scratch(char_t (&buffer)[32], const char_t* begin, const char_t* end, double* out_result)
+        {
                 size_t length = static_cast<size_t>(end - begin);
                 char_t* scratch = buffer;
@@ -5828 +8213 @@
                 {
                         // need to make dummy on-heap copy
-                        scratch = static_cast<char_t*>(global_allocate((length + 1) * sizeof(char_t)));
+                        scratch = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
                         if (!scratch) return false;
                 }
@@ -5839 +8224 @@
 
                 // free dummy buffer
-                if (scratch != buffer) global_deallocate(scratch);
+                if (scratch != buffer) xml_memory::deallocate(scratch);
 
                 return true;
         }
-        
-        double round_nearest(double value)
+
+        PUGI__FN double round_nearest(double value)
         {
                 return floor(value + 0.5);
         }
 
-        double round_nearest_nzero(double value)
+        PUGI__FN double round_nearest_nzero(double value)
         {
                 // same as round_nearest, but returns -0 for [-0.5, -0]
@@ -5855 +8240 @@
                 return (value >= -0.5 && value <= 0) ? ceil(value) : floor(value + 0.5);
         }
-        
-        const char_t* qualified_name(const xpath_node& node)
+
+        PUGI__FN const char_t* qualified_name(const xpath_node& node)
         {
                 return node.attribute() ? node.attribute().name() : node.node().name();
         }
-        
-        const char_t* local_name(const xpath_node& node)
+
+        PUGI__FN const char_t* local_name(const xpath_node& node)
         {
                 const char_t* name = qualified_name(node);
                 const char_t* p = find_char(name, ':');
-                
+
                 return p ? p + 1 : name;
         }
@@ -5882 +8267 @@
                 }
 
-                bool operator()(const xml_attribute& a) const
+                bool operator()(xml_attribute a) const
                 {
                         const char_t* name = a.name();
@@ -5892 +8277 @@
         };
 
-        const char_t* namespace_uri(const xml_node& node)
+        PUGI__FN const char_t* namespace_uri(xml_node node)
         {
                 namespace_uri_predicate pred = node.name();
-                
+
                 xml_node p = node;
-                
+
                 while (p)
                 {
                         xml_attribute a = p.find_attribute(pred);
-                        
+
                         if (a) return a.value();
-                        
+
                         p = p.parent();
                 }
-                
+
                 return PUGIXML_TEXT("");
         }
 
-        const char_t* namespace_uri(const xml_attribute& attr, const xml_node& parent)
+        PUGI__FN const char_t* namespace_uri(xml_attribute attr, xml_node parent)
         {
                 namespace_uri_predicate pred = attr.name();
-                
+
                 // Default namespace does not apply to attributes
                 if (!pred.prefix) return PUGIXML_TEXT("");
-                
+
                 xml_node p = parent;
-                
+
                 while (p)
                 {
                         xml_attribute a = p.find_attribute(pred);
-                        
+
                         if (a) return a.value();
-                        
+
                         p = p.parent();
                 }
-                
+
                 return PUGIXML_TEXT("");
         }
 
-        const char_t* namespace_uri(const xpath_node& node)
+        PUGI__FN const char_t* namespace_uri(const xpath_node& node)
         {
                 return node.attribute() ? namespace_uri(node.attribute(), node.parent()) : namespace_uri(node.node());
         }
 
-        void normalize_space(char_t* buffer)
+        PUGI__FN char_t* normalize_space(char_t* buffer)
         {
                 char_t* write = buffer;
@@ -5944 +8329 @@
                         char_t ch = *it++;
 
-                        if (IS_CHARTYPE(ch, ct_space))
+                        if (PUGI__IS_CHARTYPE(ch, ct_space))
                         {
                                 // replace whitespace sequence with single space
-                                while (IS_CHARTYPE(*it, ct_space)) it++;
+                                while (PUGI__IS_CHARTYPE(*it, ct_space)) it++;
 
                                 // avoid leading spaces
@@ -5956 +8341 @@
 
                 // remove trailing space
-                if (write != buffer && IS_CHARTYPE(write[-1], ct_space)) write--;
+                if (write != buffer && PUGI__IS_CHARTYPE(write[-1], ct_space)) write--;
 
                 // zero-terminate
                 *write = 0;
-        }
-
-        void translate(char_t* buffer, const char_t* from, const char_t* to)
-        {
-                size_t to_length = strlength(to);
-
+
+                return write;
+        }
+
+        PUGI__FN char_t* translate(char_t* buffer, const char_t* from, const char_t* to, size_t to_length)
+        {
                 char_t* write = buffer;
 
                 while (*buffer)
                 {
-                        DMC_VOLATILE char_t ch = *buffer++;
+                        PUGI__DMC_VOLATILE char_t ch = *buffer++;
 
                         const char_t* pos = find_char(from, ch);
@@ -5982 +8367 @@
                 // zero-terminate
                 *write = 0;
+
+                return write;
+        }
+
+        PUGI__FN unsigned char* translate_table_generate(xpath_allocator* alloc, const char_t* from, const char_t* to)
+        {
+                unsigned char table[128] = {0};
+
+                while (*from)
+                {
+                        unsigned int fc = static_cast<unsigned int>(*from);
+                        unsigned int tc = static_cast<unsigned int>(*to);
+
+                        if (fc >= 128 || tc >= 128)
+                                return 0;
+
+                        // code=128 means "skip character"
+                        if (!table[fc])
+                                table[fc] = static_cast<unsigned char>(tc ? tc : 128);
+
+                        from++;
+                        if (tc) to++;
+                }
+
+                for (int i = 0; i < 128; ++i)
+                        if (!table[i])
+                                table[i] = static_cast<unsigned char>(i);
+
+                void* result = alloc->allocate_nothrow(sizeof(table));
+
+                if (result)
+                {
+                        memcpy(result, table, sizeof(table));
+                }
+
+                return static_cast<unsigned char*>(result);
+        }
+
+        PUGI__FN char_t* translate_table(char_t* buffer, const unsigned char* table)
+        {
+                char_t* write = buffer;
+
+                while (*buffer)
+                {
+                        char_t ch = *buffer++;
+                        unsigned int index = static_cast<unsigned int>(ch);
+
+                        if (index < 128)
+                        {
+                                unsigned char code = table[index];
+
+                                // code=128 means "skip character" (table size is 128 so 128 can be a special value)
+                                // this code skips these characters without extra branches
+                                *write = static_cast<char_t>(code);
+                                write += 1 - (code >> 7);
+                        }
+                        else
+                        {
+                                *write++ = ch;
+                        }
+                }
+
+                // zero-terminate
+                *write = 0;
+
+                return write;
+        }
+
+        inline bool is_xpath_attribute(const char_t* name)
+        {
+                return !(starts_with(name, PUGIXML_TEXT("xmlns")) && (name[5] == 0 || name[5] == ':'));
         }
 
         struct xpath_variable_boolean: xpath_variable
         {
-                xpath_variable_boolean(): value(false)
+                xpath_variable_boolean(): xpath_variable(xpath_type_boolean), value(false)
                 {
                 }
@@ -5996 +8452 @@
         struct xpath_variable_number: xpath_variable
         {
-                xpath_variable_number(): value(0)
+                xpath_variable_number(): xpath_variable(xpath_type_number), value(0)
                 {
                 }
@@ -6006 +8462 @@
         struct xpath_variable_string: xpath_variable
         {
-                xpath_variable_string(): value(0)
+                xpath_variable_string(): xpath_variable(xpath_type_string), value(0)
                 {
                 }
@@ -6012 +8468 @@
                 ~xpath_variable_string()
                 {
-                        if (value) global_deallocate(value);
+                        if (value) xml_memory::deallocate(value);
                 }
 
@@ -6021 +8477 @@
         struct xpath_variable_node_set: xpath_variable
         {
+                xpath_variable_node_set(): xpath_variable(xpath_type_node_set)
+                {
+                }
+
                 xpath_node_set value;
                 char_t name[1];
         };
 
-        const xpath_node_set dummy_node_set;
-
-        unsigned int hash_string(const char_t* str)
+        static const xpath_node_set dummy_node_set;
+
+        PUGI__FN unsigned int hash_string(const char_t* str)
         {
                 // Jenkins one-at-a-time hash (http://en.wikipedia.org/wiki/Jenkins_hash_function#one-at-a-time)
@@ -6038 +8498 @@
                         result ^= result >> 6;
                 }
-        
+
                 result += result << 3;
                 result ^= result >> 11;
                 result += result << 15;
-        
+
                 return result;
         }
 
-        template <typename T> T* new_xpath_variable(const char_t* name)
+        template <typename T> PUGI__FN T* new_xpath_variable(const char_t* name)
         {
                 size_t length = strlength(name);
@@ -6052 +8512 @@
 
                 // $$ we can't use offsetof(T, name) because T is non-POD, so we just allocate additional length characters
-                void* memory = global_allocate(sizeof(T) + length * sizeof(char_t));
+                void* memory = xml_memory::allocate(sizeof(T) + length * sizeof(char_t));
                 if (!memory) return 0;
 
@@ -6062 +8522 @@
         }
 
-        xpath_variable* new_xpath_variable(xpath_value_type type, const char_t* name)
+        PUGI__FN xpath_variable* new_xpath_variable(xpath_value_type type, const char_t* name)
         {
                 switch (type)
@@ -6083 +8543 @@
         }
 
-        template <typename T> void delete_xpath_variable(T* var)
+        template <typename T> PUGI__FN void delete_xpath_variable(T* var)
         {
                 var->~T();
-                global_deallocate(var);
-        }
-
-        void delete_xpath_variable(xpath_value_type type, xpath_variable* var)
+                xml_memory::deallocate(var);
+        }
+
+        PUGI__FN void delete_xpath_variable(xpath_value_type type, xpath_variable* var)
         {
                 switch (type)
@@ -6110 +8570 @@
 
                 default:
-                        assert(!"Invalid variable type");
-                }
-        }
-
-        xpath_variable* get_variable(xpath_variable_set* set, const char_t* begin, const char_t* end)
-        {
-                char_t buffer[32];
-
+                        assert(false && "Invalid variable type");
+                }
+        }
+
+        PUGI__FN bool copy_xpath_variable(xpath_variable* lhs, const xpath_variable* rhs)
+        {
+                switch (rhs->type())
+                {
+                case xpath_type_node_set:
+                        return lhs->set(static_cast<const xpath_variable_node_set*>(rhs)->value);
+
+                case xpath_type_number:
+                        return lhs->set(static_cast<const xpath_variable_number*>(rhs)->value);
+
+                case xpath_type_string:
+                        return lhs->set(static_cast<const xpath_variable_string*>(rhs)->value);
+
+                case xpath_type_boolean:
+                        return lhs->set(static_cast<const xpath_variable_boolean*>(rhs)->value);
+
+                default:
+                        assert(false && "Invalid variable type");
+                        return false;
+                }
+        }
+
+        PUGI__FN bool get_variable_scratch(char_t (&buffer)[32], xpath_variable_set* set, const char_t* begin, const char_t* end, xpath_variable** out_result)
+        {
                 size_t length = static_cast<size_t>(end - begin);
                 char_t* scratch = buffer;
@@ -6124 +8604 @@
                 {
                         // need to make dummy on-heap copy
-                        scratch = static_cast<char_t*>(global_allocate((length + 1) * sizeof(char_t)));
-                        if (!scratch) return 0;
+                        scratch = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                        if (!scratch) return false;
                 }
 
@@ -6132 +8612 @@
                 scratch[length] = 0;
 
-                xpath_variable* result = set->get(scratch);
+                *out_result = set->get(scratch);
 
                 // free dummy buffer
-                if (scratch != buffer) global_deallocate(scratch);
-
-                return result;
-        }
-}
+                if (scratch != buffer) xml_memory::deallocate(scratch);
+
+                return true;
+        }
+PUGI__NS_END
 
 // Internal node set class
-namespace
-{
-        xpath_node_set::type_t xpath_sort(xpath_node* begin, xpath_node* end, xpath_node_set::type_t type, bool rev)
+PUGI__NS_BEGIN
+        PUGI__FN xpath_node_set::type_t xpath_get_order(const xpath_node* begin, const xpath_node* end)
+        {
+                if (end - begin < 2)
+                        return xpath_node_set::type_sorted;
+
+                document_order_comparator cmp;
+
+                bool first = cmp(begin[0], begin[1]);
+
+                for (const xpath_node* it = begin + 1; it + 1 < end; ++it)
+                        if (cmp(it[0], it[1]) != first)
+                                return xpath_node_set::type_unsorted;
+
+                return first ? xpath_node_set::type_sorted : xpath_node_set::type_sorted_reverse;
+        }
+
+        PUGI__FN xpath_node_set::type_t xpath_sort(xpath_node* begin, xpath_node* end, xpath_node_set::type_t type, bool rev)
         {
                 xpath_node_set::type_t order = rev ? xpath_node_set::type_sorted_reverse : xpath_node_set::type_sorted;
@@ -6150 +8645 @@
                 if (type == xpath_node_set::type_unsorted)
                 {
-                        sort(begin, end, document_order_comparator());
-
-                        type = xpath_node_set::type_sorted;
-                }
-                
+                        xpath_node_set::type_t sorted = xpath_get_order(begin, end);
+
+                        if (sorted == xpath_node_set::type_unsorted)
+                        {
+                                sort(begin, end, document_order_comparator());
+
+                                type = xpath_node_set::type_sorted;
+                        }
+                        else
+                                type = sorted;
+                }
+
                 if (type != order) reverse(begin, end);
-                        
+
                 return order;
         }
 
-        xpath_node xpath_first(const xpath_node* begin, const xpath_node* end, xpath_node_set::type_t type)
+        PUGI__FN xpath_node xpath_first(const xpath_node* begin, const xpath_node* end, xpath_node_set::type_t type)
         {
                 if (begin == end) return xpath_node();
@@ -6176 +8678 @@
 
                 default:
-                        assert(!"Invalid node set type");
+                        assert(false && "Invalid node set type");
                         return xpath_node();
                 }
         }
+
         class xpath_node_set_raw
         {
@@ -6218 +8721 @@
                 }
 
+                void push_back_grow(const xpath_node& node, xpath_allocator* alloc);
+
                 void push_back(const xpath_node& node, xpath_allocator* alloc)
                 {
-                        if (_end == _eos)
-                        {
-                                size_t capacity = static_cast<size_t>(_eos - _begin);
-
-                                // get new capacity (1.5x rule)
-                                size_t new_capacity = capacity + capacity / 2 + 1;
-
+                        if (_end != _eos)
+                                *_end++ = node;
+                        else
+                                push_back_grow(node, alloc);
+                }
+
+                void append(const xpath_node* begin_, const xpath_node* end_, xpath_allocator* alloc)
+                {
+                        if (begin_ == end_) return;
+
+                        size_t size_ = static_cast<size_t>(_end - _begin);
+                        size_t capacity = static_cast<size_t>(_eos - _begin);
+                        size_t count = static_cast<size_t>(end_ - begin_);
+
+                        if (size_ + count > capacity)
+                        {
                                 // reallocate the old array or allocate a new one
-                                xpath_node* data = static_cast<xpath_node*>(alloc->reallocate(_begin, capacity * sizeof(xpath_node), new_capacity * sizeof(xpath_node)));
+                                xpath_node* data = static_cast<xpath_node*>(alloc->reallocate(_begin, capacity * sizeof(xpath_node), (size_ + count) * sizeof(xpath_node)));
                                 assert(data);
 
                                 // finalize
                                 _begin = data;
-                                _end = data + capacity;
-                                _eos = data + new_capacity;
-                        }
-
-                        *_end++ = node;
-                }
-
-                void append(const xpath_node* begin, const xpath_node* end, xpath_allocator* alloc)
-                {
-                        size_t size = static_cast<size_t>(_end - _begin);
-                        size_t capacity = static_cast<size_t>(_eos - _begin);
-                        size_t count = static_cast<size_t>(end - begin);
-
-                        if (size + count > capacity)
-                        {
-                                // reallocate the old array or allocate a new one
-                                xpath_node* data = static_cast<xpath_node*>(alloc->reallocate(_begin, capacity * sizeof(xpath_node), (size + count) * sizeof(xpath_node)));
-                                assert(data);
-
-                                // finalize
-                                _begin = data;
-                                _end = data + size;
-                                _eos = data + size + count;
-                        }
-
-                        memcpy(_end, begin, count * sizeof(xpath_node));
+                                _end = data + size_;
+                                _eos = data + size_ + count;
+                        }
+
+                        memcpy(_end, begin_, count * sizeof(xpath_node));
                         _end += count;
                 }
@@ -6278 +8771 @@
                         if (_type == xpath_node_set::type_unsorted)
                                 sort(_begin, _end, duplicate_comparator());
-                
+
                         _end = unique(_begin, _end);
                 }
@@ -6287 +8780 @@
                 }
 
-                void set_type(xpath_node_set::type_t type)
-                {
-                        _type = type;
+                void set_type(xpath_node_set::type_t value)
+                {
+                        _type = value;
                 }
         };
-}
-
-namespace
-{
+
+        PUGI__FN_NO_INLINE void xpath_node_set_raw::push_back_grow(const xpath_node& node, xpath_allocator* alloc)
+        {
+                size_t capacity = static_cast<size_t>(_eos - _begin);
+
+                // get new capacity (1.5x rule)
+                size_t new_capacity = capacity + capacity / 2 + 1;
+
+                // reallocate the old array or allocate a new one
+                xpath_node* data = static_cast<xpath_node*>(alloc->reallocate(_begin, capacity * sizeof(xpath_node), new_capacity * sizeof(xpath_node)));
+                assert(data);
+
+                // finalize
+                _begin = data;
+                _end = data + capacity;
+                _eos = data + new_capacity;
+
+                // push
+                *_end++ = node;
+        }
+PUGI__NS_END
+
+PUGI__NS_BEGIN
         struct xpath_context
         {
@@ -6301 +8813 @@
                 size_t position, size;
 
-                xpath_context(const xpath_node& n, size_t position, size_t size): n(n), position(position), size(size)
+                xpath_context(const xpath_node& n_, size_t position_, size_t size_): n(n_), position(position_), size(size_)
                 {
                 }
@@ -6367 +8879 @@
                         next();
                 }
-                
+
                 const char_t* state() const
                 {
                         return _cur;
                 }
-                
+
                 void next()
                 {
                         const char_t* cur = _cur;
 
-                        while (IS_CHARTYPE(*cur, ct_space)) ++cur;
+                        while (PUGI__IS_CHARTYPE(*cur, ct_space)) ++cur;
 
                         // save lexeme position for error reporting
@@ -6387 +8899 @@
                                 _cur_lexeme = lex_eof;
                                 break;
-                        
+
                         case '>':
                                 if (*(cur+1) == '=')
@@ -6431 +8943 @@
 
                                 break;
-                        
+
                         case '+':
                                 cur += 1;
@@ -6455 +8967 @@
 
                                 break;
-                        
+
                         case '$':
                                 cur += 1;
 
-                                if (IS_CHARTYPEX(*cur, ctx_start_symbol))
+                                if (PUGI__IS_CHARTYPEX(*cur, ctx_start_symbol))
                                 {
                                         _cur_lexeme_contents.begin = cur;
 
-                                        while (IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
-
-                                        if (cur[0] == ':' && IS_CHARTYPEX(cur[1], ctx_symbol)) // qname
+                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
+
+                                        if (cur[0] == ':' && PUGI__IS_CHARTYPEX(cur[1], ctx_symbol)) // qname
                                         {
                                                 cur++; // :
 
-                                                while (IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
+                                                while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
                                         }
 
                                         _cur_lexeme_contents.end = cur;
-                                
+
                                         _cur_lexeme = lex_var_ref;
                                 }
@@ -6494 +9006 @@
 
                                 break;
-                        
+
                         case '[':
                                 cur += 1;
@@ -6525 +9037 @@
                                 }
                                 break;
-                
+
                         case '.':
                                 if (*(cur+1) == '.')
@@ -6532 +9044 @@
                                         _cur_lexeme = lex_double_dot;
                                 }
-                                else if (IS_CHARTYPEX(*(cur+1), ctx_digit))
+                                else if (PUGI__IS_CHARTYPEX(*(cur+1), ctx_digit))
                                 {
                                         _cur_lexeme_contents.begin = cur; // .
@@ -6538 +9050 @@
                                         ++cur;
 
-                                        while (IS_CHARTYPEX(*cur, ctx_digit)) cur++;
+                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_digit)) cur++;
 
                                         _cur_lexeme_contents.end = cur;
-                                        
+
                                         _cur_lexeme = lex_number;
                                 }
@@ -6567 +9079 @@
                                 while (*cur && *cur != terminator) cur++;
                                 _cur_lexeme_contents.end = cur;
-                                
+
                                 if (!*cur)
                                         _cur_lexeme = lex_none;
@@ -6592 +9104 @@
 
                         default:
-                                if (IS_CHARTYPEX(*cur, ctx_digit))
+                                if (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
                                 {
                                         _cur_lexeme_contents.begin = cur;
 
-                                        while (IS_CHARTYPEX(*cur, ctx_digit)) cur++;
-                                
+                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_digit)) cur++;
+
                                         if (*cur == '.')
                                         {
                                                 cur++;
 
-                                                while (IS_CHARTYPEX(*cur, ctx_digit)) cur++;
+                                                while (PUGI__IS_CHARTYPEX(*cur, ctx_digit)) cur++;
                                         }
 
@@ -6609 +9121 @@
                                         _cur_lexeme = lex_number;
                                 }
-                                else if (IS_CHARTYPEX(*cur, ctx_start_symbol))
+                                else if (PUGI__IS_CHARTYPEX(*cur, ctx_start_symbol))
                                 {
                                         _cur_lexeme_contents.begin = cur;
 
-                                        while (IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
+                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
 
                                         if (cur[0] == ':')
@@ -6621 +9133 @@
                                                         cur += 2; // :*
                                                 }
-                                                else if (IS_CHARTYPEX(cur[1], ctx_symbol)) // namespace test qname
+                                                else if (PUGI__IS_CHARTYPEX(cur[1], ctx_symbol)) // namespace test qname
                                                 {
                                                         cur++; // :
 
-                                                        while (IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
+                                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
                                                 }
                                         }
 
                                         _cur_lexeme_contents.end = cur;
-                                
+
                                         _cur_lexeme = lex_string;
                                 }
@@ -6662 +9174 @@
         enum ast_type_t
         {
+                ast_unknown,
                 ast_op_or,                                              // left or right
                 ast_op_and,                                             // left and right
                 ast_op_equal,                                   // left = right
-                ast_op_not_equal,                               // left != right
+                ast_op_not_equal,                               // left != right
                 ast_op_less,                                    // left < right
                 ast_op_greater,                                 // left > right
@@ -6679 +9192 @@
                 ast_predicate,                                  // apply predicate to set; next points to next predicate
                 ast_filter,                                             // select * from left where right
-                ast_filter_posinv,                              // select * from left where right; proximity position invariant
                 ast_string_constant,                    // string constant
                 ast_number_constant,                    // number constant
@@ -6719 +9231 @@
                 ast_func_round,                                 // round(left)
                 ast_step,                                               // process set left with step
-                ast_step_root                                   // select root node
+                ast_step_root,                                  // select root node
+
+                ast_opt_translate_table,                // translate(left, right, third) where right/third are constants
+                ast_opt_compare_attribute               // @name = 'string'
         };
 
@@ -6738 +9253 @@
                 axis_self
         };
-        
+
         enum nodetest_t
         {
@@ -6752 +9267 @@
         };
 
+        enum predicate_t
+        {
+                predicate_default,
+                predicate_posinv,
+                predicate_constant,
+                predicate_constant_one
+        };
+
+        enum nodeset_eval_t
+        {
+                nodeset_eval_all,
+                nodeset_eval_any,
+                nodeset_eval_first
+        };
+
         template <axis_t N> struct axis_to_type
         {
@@ -6758 +9288 @@
 
         template <axis_t N> const axis_t axis_to_type<N>::axis = N;
-                
+
         class xpath_ast_node
         {
@@ -6766 +9296 @@
                 char _rettype;
 
-                // for ast_step / ast_predicate
+                // for ast_step
                 char _axis;
+
+                // for ast_step/ast_predicate/ast_filter
                 char _test;
 
@@ -6785 +9317 @@
                         // node test for ast_step (node name/namespace/node type/pi target)
                         const char_t* nodetest;
+                        // table for ast_opt_translate_table
+                        const unsigned char* table;
                 } _data;
 
@@ -6814 +9348 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
-                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack, nodeset_eval_all);
+                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                 for (const xpath_node* li = ls.begin(); li != ls.end(); ++li)
@@ -6843 +9377 @@
 
                                         double l = lhs->eval_number(c, stack);
-                                        xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                        xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                         for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
@@ -6860 +9394 @@
 
                                         xpath_string l = lhs->eval_string(c, stack);
-                                        xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                        xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                         for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
@@ -6874 +9408 @@
                         }
 
-                        assert(!"Wrong types");
+                        assert(false && "Wrong types");
                         return false;
+                }
+
+                static bool eval_once(xpath_node_set::type_t type, nodeset_eval_t eval)
+                {
+                        return type == xpath_node_set::type_sorted ? eval != nodeset_eval_all : eval == nodeset_eval_any;
                 }
 
@@ -6888 +9427 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
-                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack, nodeset_eval_all);
+                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                 for (const xpath_node* li = ls.begin(); li != ls.end(); ++li)
@@ -6913 +9452 @@
 
                                 double l = lhs->eval_number(c, stack);
-                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                 for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
@@ -6929 +9468 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
+                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack, nodeset_eval_all);
                                 double r = rhs->eval_number(c, stack);
 
@@ -6944 +9483 @@
                         else
                         {
-                                assert(!"Wrong types");
+                                assert(false && "Wrong types");
                                 return false;
                         }
                 }
 
-                void apply_predicate(xpath_node_set_raw& ns, size_t first, xpath_ast_node* expr, const xpath_stack& stack)
+                static void apply_predicate_boolean(xpath_node_set_raw& ns, size_t first, xpath_ast_node* expr, const xpath_stack& stack, bool once)
                 {
                         assert(ns.size() >= first);
+                        assert(expr->rettype() != xpath_type_number);
 
                         size_t i = 1;
                         size_t size = ns.size() - first;
-                                
+
                         xpath_node* last = ns.begin() + first;
-                                
+
                         // remove_if... or well, sort of
                         for (xpath_node* it = last; it != ns.end(); ++it, ++i)
                         {
                                 xpath_context c(*it, i, size);
-                        
-                                if (expr->rettype() == xpath_type_number)
-                                {
-                                        if (expr->eval_number(c, stack) == i)
-                                                *last++ = *it;
-                                }
-                                else if (expr->eval_boolean(c, stack))
+
+                                if (expr->eval_boolean(c, stack))
+                                {
                                         *last++ = *it;
-                        }
-                        
+
+                                        if (once) break;
+                                }
+                        }
+
                         ns.truncate(last);
                 }
 
-                void apply_predicates(xpath_node_set_raw& ns, size_t first, const xpath_stack& stack)
+                static void apply_predicate_number(xpath_node_set_raw& ns, size_t first, xpath_ast_node* expr, const xpath_stack& stack, bool once)
+                {
+                        assert(ns.size() >= first);
+                        assert(expr->rettype() == xpath_type_number);
+
+                        size_t i = 1;
+                        size_t size = ns.size() - first;
+
+                        xpath_node* last = ns.begin() + first;
+
+                        // remove_if... or well, sort of
+                        for (xpath_node* it = last; it != ns.end(); ++it, ++i)
+                        {
+                                xpath_context c(*it, i, size);
+
+                                if (expr->eval_number(c, stack) == i)
+                                {
+                                        *last++ = *it;
+
+                                        if (once) break;
+                                }
+                        }
+
+                        ns.truncate(last);
+                }
+
+                static void apply_predicate_number_const(xpath_node_set_raw& ns, size_t first, xpath_ast_node* expr, const xpath_stack& stack)
+                {
+                        assert(ns.size() >= first);
+                        assert(expr->rettype() == xpath_type_number);
+
+                        size_t size = ns.size() - first;
+
+                        xpath_node* last = ns.begin() + first;
+
+                        xpath_context c(xpath_node(), 1, size);
+
+                        double er = expr->eval_number(c, stack);
+
+                        if (er >= 1.0 && er <= size)
+                        {
+                                size_t eri = static_cast<size_t>(er);
+
+                                if (er == eri)
+                                {
+                                        xpath_node r = last[eri - 1];
+
+                                        *last++ = r;
+                                }
+                        }
+
+                        ns.truncate(last);
+                }
+
+                void apply_predicate(xpath_node_set_raw& ns, size_t first, const xpath_stack& stack, bool once)
                 {
                         if (ns.size() == first) return;
-                        
+
+                        assert(_type == ast_filter || _type == ast_predicate);
+
+                        if (_test == predicate_constant || _test == predicate_constant_one)
+                                apply_predicate_number_const(ns, first, _right, stack);
+                        else if (_right->rettype() == xpath_type_number)
+                                apply_predicate_number(ns, first, _right, stack, once);
+                        else
+                                apply_predicate_boolean(ns, first, _right, stack, once);
+                }
+
+                void apply_predicates(xpath_node_set_raw& ns, size_t first, const xpath_stack& stack, nodeset_eval_t eval)
+                {
+                        if (ns.size() == first) return;
+
+                        bool last_once = eval_once(ns.type(), eval);
+
                         for (xpath_ast_node* pred = _right; pred; pred = pred->_next)
-                        {
-                                apply_predicate(ns, first, pred->_left, stack);
-                        }
-                }
-
-                void step_push(xpath_node_set_raw& ns, const xml_attribute& a, const xml_node& parent, xpath_allocator* alloc)
-                {
-                        if (!a) return;
-
-                        const char_t* name = a.name();
-
-                        // There are no attribute nodes corresponding to attributes that declare namespaces
-                        // That is, "xmlns:..." or "xmlns"
-                        if (starts_with(name, PUGIXML_TEXT("xmlns")) && (name[5] == 0 || name[5] == ':')) return;
-                        
+                                pred->apply_predicate(ns, first, stack, !pred->_next && last_once);
+                }
+
+                bool step_push(xpath_node_set_raw& ns, xml_attribute_struct* a, xml_node_struct* parent, xpath_allocator* alloc)
+                {
+                        assert(a);
+
+                        const char_t* name = a->name ? a->name + 0 : PUGIXML_TEXT("");
+
                         switch (_test)
                         {
                         case nodetest_name:
-                                if (strequal(name, _data.nodetest)) ns.push_back(xpath_node(a, parent), alloc);
+                                if (strequal(name, _data.nodetest) && is_xpath_attribute(name))
+                                {
+                                        ns.push_back(xpath_node(xml_attribute(a), xml_node(parent)), alloc);
+                                        return true;
+                                }
                                 break;
-                                
+
                         case nodetest_type_node:
                         case nodetest_all:
-                                ns.push_back(xpath_node(a, parent), alloc);
+                                if (is_xpath_attribute(name))
+                                {
+                                        ns.push_back(xpath_node(xml_attribute(a), xml_node(parent)), alloc);
+                                        return true;
+                                }
                                 break;
-                                
+
                         case nodetest_all_in_namespace:
-                                if (starts_with(name, _data.nodetest))
-                                        ns.push_back(xpath_node(a, parent), alloc);
+                                if (starts_with(name, _data.nodetest) && is_xpath_attribute(name))
+                                {
+                                        ns.push_back(xpath_node(xml_attribute(a), xml_node(parent)), alloc);
+                                        return true;
+                                }
                                 break;
-                        
+
                         default:
                                 ;
                         }
-                }
-                
-                void step_push(xpath_node_set_raw& ns, const xml_node& n, xpath_allocator* alloc)
-                {
-                        if (!n) return;
+
+                        return false;
+                }
+
+                bool step_push(xpath_node_set_raw& ns, xml_node_struct* n, xpath_allocator* alloc)
+                {
+                        assert(n);
+
+                        xml_node_type type = PUGI__NODETYPE(n);
 
                         switch (_test)
                         {
                         case nodetest_name:
-                                if (n.type() == node_element && strequal(n.name(), _data.nodetest)) ns.push_back(n, alloc);
+                                if (type == node_element && n->name && strequal(n->name, _data.nodetest))
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
+
                         case nodetest_type_node:
-                                ns.push_back(n, alloc);
+                                ns.push_back(xml_node(n), alloc);
+                                return true;
+
+                        case nodetest_type_comment:
+                                if (type == node_comment)
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_type_comment:
-                                if (n.type() == node_comment)
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_type_text:
+                                if (type == node_pcdata || type == node_cdata)
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_type_text:
-                                if (n.type() == node_pcdata || n.type() == node_cdata)
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_type_pi:
+                                if (type == node_pi)
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_type_pi:
-                                if (n.type() == node_pi)
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_pi:
+                                if (type == node_pi && n->name && strequal(n->name, _data.nodetest))
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                                                        
-                        case nodetest_pi:
-                                if (n.type() == node_pi && strequal(n.name(), _data.nodetest))
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_all:
+                                if (type == node_element)
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_all:
-                                if (n.type() == node_element)
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_all_in_namespace:
+                                if (type == node_element && n->name && starts_with(n->name, _data.nodetest))
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_all_in_namespace:
-                                if (n.type() == node_element && starts_with(n.name(), _data.nodetest))
-                                        ns.push_back(n, alloc);
+
+                        default:
+                                assert(false && "Unknown axis");
+                        }
+
+                        return false;
+                }
+
+                template <class T> void step_fill(xpath_node_set_raw& ns, xml_node_struct* n, xpath_allocator* alloc, bool once, T)
+                {
+                        const axis_t axis = T::axis;
+
+                        switch (axis)
+                        {
+                        case axis_attribute:
+                        {
+                                for (xml_attribute_struct* a = n->first_attribute; a; a = a->next_attribute)
+                                        if (step_push(ns, a, n, alloc) & once)
+                                                return;
+
                                 break;
-
-                        default:
-                                assert(!"Unknown axis");
-                        } 
-                }
-
-                template <class T> void step_fill(xpath_node_set_raw& ns, const xml_node& n, xpath_allocator* alloc, T)
-                {
-                        const axis_t axis = T::axis;
-
-                        switch (axis)
-                        {
-                        case axis_attribute:
-                        {
-                                for (xml_attribute a = n.first_attribute(); a; a = a.next_attribute())
-                                        step_push(ns, a, n, alloc);
-                                
+                        }
+
+                        case axis_child:
+                        {
+                                for (xml_node_struct* c = n->first_child; c; c = c->next_sibling)
+                                        if (step_push(ns, c, alloc) & once)
+                                                return;
+
                                 break;
                         }
-                        
-                        case axis_child:
-                        {
-                                for (xml_node c = n.first_child(); c; c = c.next_sibling())
-                                        step_push(ns, c, alloc);
-                                        
-                                break;
-                        }
-                        
+
                         case axis_descendant:
                         case axis_descendant_or_self:
                         {
                                 if (axis == axis_descendant_or_self)
-                                        step_push(ns, n, alloc);
-                                        
-                                xml_node cur = n.first_child();
-                                
-                                while (cur && cur != n)
-                                {
-                                        step_push(ns, cur, alloc);
-                                        
-                                        if (cur.first_child())
-                                                cur = cur.first_child();
-                                        else if (cur.next_sibling())
-                                                cur = cur.next_sibling();
+                                        if (step_push(ns, n, alloc) & once)
+                                                return;
+
+                                xml_node_struct* cur = n->first_child;
+
+                                while (cur)
+                                {
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
+
+                                        if (cur->first_child)
+                                                cur = cur->first_child;
                                         else
                                         {
-                                                while (!cur.next_sibling() && cur != n)
-                                                        cur = cur.parent();
-                                        
-                                                if (cur != n) cur = cur.next_sibling();
+                                                while (!cur->next_sibling)
+                                                {
+                                                        cur = cur->parent;
+
+                                                        if (cur == n) return;
+                                                }
+
+                                                cur = cur->next_sibling;
                                         }
                                 }
-                                
+
                                 break;
                         }
-                        
+
                         case axis_following_sibling:
                         {
-                                for (xml_node c = n.next_sibling(); c; c = c.next_sibling())
-                                        step_push(ns, c, alloc);
-                                
+                                for (xml_node_struct* c = n->next_sibling; c; c = c->next_sibling)
+                                        if (step_push(ns, c, alloc) & once)
+                                                return;
+
                                 break;
                         }
-                        
+
                         case axis_preceding_sibling:
                         {
-                                for (xml_node c = n.previous_sibling(); c; c = c.previous_sibling())
-                                        step_push(ns, c, alloc);
-                                
+                                for (xml_node_struct* c = n->prev_sibling_c; c->next_sibling; c = c->prev_sibling_c)
+                                        if (step_push(ns, c, alloc) & once)
+                                                return;
+
                                 break;
                         }
-                        
+
                         case axis_following:
                         {
-                                xml_node cur = n;
+                                xml_node_struct* cur = n;
 
                                 // exit from this node so that we don't include descendants
-                                while (cur && !cur.next_sibling()) cur = cur.parent();
-                                cur = cur.next_sibling();
-
-                                for (;;)
-                                {
-                                        step_push(ns, cur, alloc);
-
-                                        if (cur.first_child())
-                                                cur = cur.first_child();
-                                        else if (cur.next_sibling())
-                                                cur = cur.next_sibling();
+                                while (!cur->next_sibling)
+                                {
+                                        cur = cur->parent;
+
+                                        if (!cur) return;
+                                }
+
+                                cur = cur->next_sibling;
+
+                                while (cur)
+                                {
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
+
+                                        if (cur->first_child)
+                                                cur = cur->first_child;
                                         else
                                         {
-                                                while (cur && !cur.next_sibling()) cur = cur.parent();
-                                                cur = cur.next_sibling();
-
-                                                if (!cur) break;
+                                                while (!cur->next_sibling)
+                                                {
+                                                        cur = cur->parent;
+
+                                                        if (!cur) return;
+                                                }
+
+                                                cur = cur->next_sibling;
                                         }
                                 }
@@ -7161 +9820 @@
                         case axis_preceding:
                         {
-                                xml_node cur = n;
-
-                                while (cur && !cur.previous_sibling()) cur = cur.parent();
-                                cur = cur.previous_sibling();
-
-                                for (;;)
-                                {
-                                        if (cur.last_child())
-                                                cur = cur.last_child();
+                                xml_node_struct* cur = n;
+
+                                // exit from this node so that we don't include descendants
+                                while (!cur->prev_sibling_c->next_sibling)
+                                {
+                                        cur = cur->parent;
+
+                                        if (!cur) return;
+                                }
+
+                                cur = cur->prev_sibling_c;
+
+                                while (cur)
+                                {
+                                        if (cur->first_child)
+                                                cur = cur->first_child->prev_sibling_c;
                                         else
                                         {
                                                 // leaf node, can't be ancestor
-                                                step_push(ns, cur, alloc);
-
-                                                if (cur.previous_sibling())
-                                                        cur = cur.previous_sibling();
-                                                else
+                                                if (step_push(ns, cur, alloc) & once)
+                                                        return;
+
+                                                while (!cur->prev_sibling_c->next_sibling)
                                                 {
-                                                        do 
-                                                        {
-                                                                cur = cur.parent();
-                                                                if (!cur) break;
-
-                                                                if (!node_is_ancestor(cur, n)) step_push(ns, cur, alloc);
-                                                        }
-                                                        while (!cur.previous_sibling());
-
-                                                        cur = cur.previous_sibling();
-
-                                                        if (!cur) break;
+                                                        cur = cur->parent;
+
+                                                        if (!cur) return;
+
+                                                        if (!node_is_ancestor(cur, n))
+                                                                if (step_push(ns, cur, alloc) & once)
+                                                                        return;
                                                 }
+
+                                                cur = cur->prev_sibling_c;
                                         }
                                 }
@@ -7197 +9859 @@
                                 break;
                         }
-                        
+
                         case axis_ancestor:
                         case axis_ancestor_or_self:
                         {
                                 if (axis == axis_ancestor_or_self)
-                                        step_push(ns, n, alloc);
-
-                                xml_node cur = n.parent();
-                                
+                                        if (step_push(ns, n, alloc) & once)
+                                                return;
+
+                                xml_node_struct* cur = n->parent;
+
                                 while (cur)
                                 {
-                                        step_push(ns, cur, alloc);
-                                        
-                                        cur = cur.parent();
-                                }
-                                
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
+
+                                        cur = cur->parent;
+                                }
+
                                 break;
                         }
@@ -7225 +9889 @@
                         case axis_parent:
                         {
-                                if (n.parent()) step_push(ns, n.parent(), alloc);
+                                if (n->parent)
+                                        step_push(ns, n->parent, alloc);
 
                                 break;
                         }
-                                
+
                         default:
-                                assert(!"Unimplemented axis");
-                        }
-                }
-                
-                template <class T> void step_fill(xpath_node_set_raw& ns, const xml_attribute& a, const xml_node& p, xpath_allocator* alloc, T v)
+                                assert(false && "Unimplemented axis");
+                        }
+                }
+
+                template <class T> void step_fill(xpath_node_set_raw& ns, xml_attribute_struct* a, xml_node_struct* p, xpath_allocator* alloc, bool once, T v)
                 {
                         const axis_t axis = T::axis;
@@ -7245 +9910 @@
                         {
                                 if (axis == axis_ancestor_or_self && _test == nodetest_type_node) // reject attributes based on principal node type test
-                                        step_push(ns, a, p, alloc);
-
-                                xml_node cur = p;
-                                
+                                        if (step_push(ns, a, p, alloc) & once)
+                                                return;
+
+                                xml_node_struct* cur = p;
+
                                 while (cur)
                                 {
-                                        step_push(ns, cur, alloc);
-                                        
-                                        cur = cur.parent();
-                                }
-                                
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
+
+                                        cur = cur->parent;
+                                }
+
                                 break;
                         }
@@ -7270 +9937 @@
                         case axis_following:
                         {
-                                xml_node cur = p;
-                                
-                                for (;;)
-                                {
-                                        if (cur.first_child())
-                                                cur = cur.first_child();
-                                        else if (cur.next_sibling())
-                                                cur = cur.next_sibling();
+                                xml_node_struct* cur = p;
+
+                                while (cur)
+                                {
+                                        if (cur->first_child)
+                                                cur = cur->first_child;
                                         else
                                         {
-                                                while (cur && !cur.next_sibling()) cur = cur.parent();
-                                                cur = cur.next_sibling();
-                                                
-                                                if (!cur) break;
+                                                while (!cur->next_sibling)
+                                                {
+                                                        cur = cur->parent;
+
+                                                        if (!cur) return;
+                                                }
+
+                                                cur = cur->next_sibling;
                                         }
 
-                                        step_push(ns, cur, alloc);
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
                                 }
 
@@ -7302 +9972 @@
                         {
                                 // preceding:: axis does not include attribute nodes and attribute ancestors (they are the same as parent's ancestors), so we can reuse node preceding
-                                step_fill(ns, p, alloc, v);
+                                step_fill(ns, p, alloc, once, v);
                                 break;
                         }
-                        
+
                         default:
-                                assert(!"Unimplemented axis");
-                        }
-                }
-                
-                template <class T> xpath_node_set_raw step_do(const xpath_context& c, const xpath_stack& stack, T v)
+                                assert(false && "Unimplemented axis");
+                        }
+                }
+
+                template <class T> void step_fill(xpath_node_set_raw& ns, const xpath_node& xn, xpath_allocator* alloc, bool once, T v)
                 {
                         const axis_t axis = T::axis;
-                        bool attributes = (axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_descendant_or_self || axis == axis_following || axis == axis_parent || axis == axis_preceding || axis == axis_self);
+                        const bool axis_has_attributes = (axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_descendant_or_self || axis == axis_following || axis == axis_parent || axis == axis_preceding || axis == axis_self);
+
+                        if (xn.node())
+                                step_fill(ns, xn.node().internal_object(), alloc, once, v);
+                        else if (axis_has_attributes && xn.attribute() && xn.parent())
+                                step_fill(ns, xn.attribute().internal_object(), xn.parent().internal_object(), alloc, once, v);
+                }
+
+                template <class T> xpath_node_set_raw step_do(const xpath_context& c, const xpath_stack& stack, nodeset_eval_t eval, T v)
+                {
+                        const axis_t axis = T::axis;
+                        const bool axis_reverse = (axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_preceding || axis == axis_preceding_sibling);
+                        const xpath_node_set::type_t axis_type = axis_reverse ? xpath_node_set::type_sorted_reverse : xpath_node_set::type_sorted;
+
+                        bool once =
+                                (axis == axis_attribute && _test == nodetest_name) ||
+                                (!_right && eval_once(axis_type, eval)) ||
+                                (_right && !_right->_next && _right->_test == predicate_constant_one);
 
                         xpath_node_set_raw ns;
-                        ns.set_type((axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_preceding || axis == axis_preceding_sibling) ? xpath_node_set::type_sorted_reverse : xpath_node_set::type_sorted);
+                        ns.set_type(axis_type);
 
                         if (_left)
                         {
-                                xpath_node_set_raw s = _left->eval_node_set(c, stack);
+                                xpath_node_set_raw s = _left->eval_node_set(c, stack, nodeset_eval_all);
 
                                 // self axis preserves the original order
@@ -7332 +10019 @@
                                         // in general, all axes generate elements in a particular order, but there is no order guarantee if axis is applied to two nodes
                                         if (axis != axis_self && size != 0) ns.set_type(xpath_node_set::type_unsorted);
-                                        
-                                        if (it->node())
-                                                step_fill(ns, it->node(), stack.result, v);
-                                        else if (attributes)
-                                                step_fill(ns, it->attribute(), it->parent(), stack.result, v);
-                                                
-                                        apply_predicates(ns, size, stack);
+
+                                        step_fill(ns, *it, stack.result, once, v);
+                                        if (_right) apply_predicates(ns, size, stack, eval);
                                 }
                         }
                         else
                         {
-                                if (c.n.node())
-                                        step_fill(ns, c.n.node(), stack.result, v);
-                                else if (attributes)
-                                        step_fill(ns, c.n.attribute(), c.n.parent(), stack.result, v);
-                                
-                                apply_predicates(ns, 0, stack);
+                                step_fill(ns, c.n, stack.result, once, v);
+                                if (_right) apply_predicates(ns, 0, stack, eval);
                         }
 
@@ -7358 +10037 @@
                         return ns;
                 }
-                
+
         public:
-                xpath_ast_node(ast_type_t type, xpath_value_type rettype, const char_t* value):
-                        _type((char)type), _rettype((char)rettype), _axis(0), _test(0), _left(0), _right(0), _next(0)
+                xpath_ast_node(ast_type_t type, xpath_value_type rettype_, const char_t* value):
+                        _type(static_cast<char>(type)), _rettype(static_cast<char>(rettype_)), _axis(0), _test(0), _left(0), _right(0), _next(0)
                 {
                         assert(type == ast_string_constant);
@@ -7367 +10046 @@
                 }
 
-                xpath_ast_node(ast_type_t type, xpath_value_type rettype, double value):
-                        _type((char)type), _rettype((char)rettype), _axis(0), _test(0), _left(0), _right(0), _next(0)
+                xpath_ast_node(ast_type_t type, xpath_value_type rettype_, double value):
+                        _type(static_cast<char>(type)), _rettype(static_cast<char>(rettype_)), _axis(0), _test(0), _left(0), _right(0), _next(0)
                 {
                         assert(type == ast_number_constant);
                         _data.number = value;
                 }
-                
-                xpath_ast_node(ast_type_t type, xpath_value_type rettype, xpath_variable* value):
-                        _type((char)type), _rettype((char)rettype), _axis(0), _test(0), _left(0), _right(0), _next(0)
+
+                xpath_ast_node(ast_type_t type, xpath_value_type rettype_, xpath_variable* value):
+                        _type(static_cast<char>(type)), _rettype(static_cast<char>(rettype_)), _axis(0), _test(0), _left(0), _right(0), _next(0)
                 {
                         assert(type == ast_variable);
                         _data.variable = value;
                 }
-                
-                xpath_ast_node(ast_type_t type, xpath_value_type rettype, xpath_ast_node* left = 0, xpath_ast_node* right = 0):
-                        _type((char)type), _rettype((char)rettype), _axis(0), _test(0), _left(left), _right(right), _next(0)
+
+                xpath_ast_node(ast_type_t type, xpath_value_type rettype_, xpath_ast_node* left = 0, xpath_ast_node* right = 0):
+                        _type(static_cast<char>(type)), _rettype(static_cast<char>(rettype_)), _axis(0), _test(0), _left(left), _right(right), _next(0)
                 {
                 }
 
                 xpath_ast_node(ast_type_t type, xpath_ast_node* left, axis_t axis, nodetest_t test, const char_t* contents):
-                        _type((char)type), _rettype(xpath_type_node_set), _axis((char)axis), _test((char)test), _left(left), _right(0), _next(0)
-                {
+                        _type(static_cast<char>(type)), _rettype(xpath_type_node_set), _axis(static_cast<char>(axis)), _test(static_cast<char>(test)), _left(left), _right(0), _next(0)
+                {
+                        assert(type == ast_step);
                         _data.nodetest = contents;
+                }
+
+                xpath_ast_node(ast_type_t type, xpath_ast_node* left, xpath_ast_node* right, predicate_t test):
+                        _type(static_cast<char>(type)), _rettype(xpath_type_node_set), _axis(0), _test(static_cast<char>(test)), _left(left), _right(right), _next(0)
+                {
+                        assert(type == ast_filter || type == ast_predicate);
                 }
 
@@ -7408 +10094 @@
                         case ast_op_or:
                                 return _left->eval_boolean(c, stack) || _right->eval_boolean(c, stack);
-                                
+
                         case ast_op_and:
                                 return _left->eval_boolean(c, stack) && _right->eval_boolean(c, stack);
-                                
+
                         case ast_op_equal:
                                 return compare_eq(_left, _right, c, stack, equal_to());
@@ -7417 +10103 @@
                         case ast_op_not_equal:
                                 return compare_eq(_left, _right, c, stack, not_equal_to());
-        
+
                         case ast_op_less:
                                 return compare_rel(_left, _right, c, stack, less());
-                        
+
                         case ast_op_greater:
                                 return compare_rel(_right, _left, c, stack, less());
@@ -7426 +10112 @@
                         case ast_op_less_or_equal:
                                 return compare_rel(_left, _right, c, stack, less_equal());
-                        
+
                         case ast_op_greater_or_equal:
                                 return compare_rel(_right, _left, c, stack, less_equal());
@@ -7452 +10138 @@
                         case ast_func_boolean:
                                 return _left->eval_boolean(c, stack);
-                                
+
                         case ast_func_not:
                                 return !_left->eval_boolean(c, stack);
-                                
+
                         case ast_func_true:
                                 return true;
-                                
+
                         case ast_func_false:
                                 return false;
@@ -7465 +10151 @@
                         {
                                 if (c.n.attribute()) return false;
-                                
+
                                 xpath_allocator_capture cr(stack.result);
 
                                 xpath_string lang = _left->eval_string(c, stack);
-                                
+
                                 for (xml_node n = c.n.node(); n; n = n.parent())
                                 {
                                         xml_attribute a = n.attribute(PUGIXML_TEXT("xml:lang"));
-                                        
+
                                         if (a)
                                         {
                                                 const char_t* value = a.value();
-                                                
+
                                                 // strnicmp / strncasecmp is not portable
                                                 for (const char_t* lit = lang.c_str(); *lit; ++lit)
@@ -7484 +10170 @@
                                                         ++value;
                                                 }
-                                                
+
                                                 return *value == 0 || *value == '-';
                                         }
                                 }
-                                
+
                                 return false;
+                        }
+
+                        case ast_opt_compare_attribute:
+                        {
+                                const char_t* value = (_right->_type == ast_string_constant) ? _right->_data.string : _right->_data.variable->get_string();
+
+                                xml_attribute attr = c.n.node().attribute(_left->_data.nodetest);
+
+                                return attr && strequal(attr.value(), value) && is_xpath_attribute(attr.name());
                         }
 
@@ -7508 +10203 @@
                                 case xpath_type_number:
                                         return convert_number_to_boolean(eval_number(c, stack));
-                                        
+
                                 case xpath_type_string:
                                 {
@@ -7515 +10210 @@
                                         return !eval_string(c, stack).empty();
                                 }
-                                        
-                                case xpath_type_node_set:                               
+
+                                case xpath_type_node_set:
                                 {
                                         xpath_allocator_capture cr(stack.result);
 
-                                        return !eval_node_set(c, stack).empty();
+                                        return !eval_node_set(c, stack, nodeset_eval_any).empty();
                                 }
 
                                 default:
-                                        assert(!"Wrong expression for return type boolean");
+                                        assert(false && "Wrong expression for return type boolean");
                                         return false;
                                 }
@@ -7537 +10232 @@
                         case ast_op_add:
                                 return _left->eval_number(c, stack) + _right->eval_number(c, stack);
-                                
+
                         case ast_op_subtract:
                                 return _left->eval_number(c, stack) - _right->eval_number(c, stack);
@@ -7557 +10252 @@
 
                         case ast_func_last:
-                                return (double)c.size;
-                        
+                                return static_cast<double>(c.size);
+
                         case ast_func_position:
-                                return (double)c.position;
+                                return static_cast<double>(c.position);
 
                         case ast_func_count:
@@ -7566 +10261 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                return (double)_left->eval_node_set(c, stack).size();
-                        }
-                        
+                                return static_cast<double>(_left->eval_node_set(c, stack, nodeset_eval_all).size());
+                        }
+
                         case ast_func_string_length_0:
                         {
                                 xpath_allocator_capture cr(stack.result);
 
-                                return (double)string_value(c.n, stack.result).length();
-                        }
-                        
+                                return static_cast<double>(string_value(c.n, stack.result).length());
+                        }
+
                         case ast_func_string_length_1:
                         {
                                 xpath_allocator_capture cr(stack.result);
 
-                                return (double)_left->eval_string(c, stack).length();
-                        }
-                        
+                                return static_cast<double>(_left->eval_string(c, stack).length());
+                        }
+
                         case ast_func_number_0:
                         {
@@ -7589 +10284 @@
                                 return convert_string_to_number(string_value(c.n, stack.result).c_str());
                         }
-                        
+
                         case ast_func_number_1:
                                 return _left->eval_number(c, stack);
@@ -7598 +10293 @@
 
                                 double r = 0;
-                                
-                                xpath_node_set_raw ns = _left->eval_node_set(c, stack);
-                                
+
+                                xpath_node_set_raw ns = _left->eval_node_set(c, stack, nodeset_eval_all);
+
                                 for (const xpath_node* it = ns.begin(); it != ns.end(); ++it)
                                 {
@@ -7607 +10302 @@
                                         r += convert_string_to_number(string_value(*it, stack.result).c_str());
                                 }
-                        
+
                                 return r;
                         }
@@ -7614 +10309 @@
                         {
                                 double r = _left->eval_number(c, stack);
-                                
+
                                 return r == r ? floor(r) : r;
                         }
@@ -7621 +10316 @@
                         {
                                 double r = _left->eval_number(c, stack);
-                                
+
                                 return r == r ? ceil(r) : r;
                         }
@@ -7627 +10322 @@
                         case ast_func_round:
                                 return round_nearest_nzero(_left->eval_number(c, stack));
-                        
+
                         case ast_variable:
                         {
@@ -7644 +10339 @@
                                 case xpath_type_boolean:
                                         return eval_boolean(c, stack) ? 1 : 0;
-                                        
+
                                 case xpath_type_string:
                                 {
@@ -7651 +10346 @@
                                         return convert_string_to_number(eval_string(c, stack).c_str());
                                 }
-                                        
+
                                 case xpath_type_node_set:
                                 {
@@ -7658 +10353 @@
                                         return convert_string_to_number(eval_string(c, stack).c_str());
                                 }
-                                        
+
                                 default:
-                                        assert(!"Wrong expression for return type number");
+                                        assert(false && "Wrong expression for return type number");
                                         return 0;
                                 }
-                                
-                        }
-                        }
-                }
-                
+
+                        }
+                        }
+                }
+
                 xpath_string eval_string_concat(const xpath_context& c, const xpath_stack& stack)
                 {
@@ -7714 +10409 @@
                         *ri = 0;
 
-                        return xpath_string(result, true);
+                        return xpath_string::from_heap_preallocated(result, ri);
                 }
 
@@ -7722 +10417 @@
                         {
                         case ast_string_constant:
-                                return xpath_string_const(_data.string);
-                        
+                                return xpath_string::from_const(_data.string);
+
                         case ast_func_local_name_0:
                         {
                                 xpath_node na = c.n;
-                                
-                                return xpath_string_const(local_name(na));
+
+                                return xpath_string::from_const(local_name(na));
                         }
 
@@ -7735 +10430 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ns = _left->eval_node_set(c, stack);
+                                xpath_node_set_raw ns = _left->eval_node_set(c, stack, nodeset_eval_first);
                                 xpath_node na = ns.first();
-                                
-                                return xpath_string_const(local_name(na));
+
+                                return xpath_string::from_const(local_name(na));
                         }
 
@@ -7744 +10439 @@
                         {
                                 xpath_node na = c.n;
-                                
-                                return xpath_string_const(qualified_name(na));
+
+                                return xpath_string::from_const(qualified_name(na));
                         }
 
@@ -7752 +10447 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ns = _left->eval_node_set(c, stack);
+                                xpath_node_set_raw ns = _left->eval_node_set(c, stack, nodeset_eval_first);
                                 xpath_node na = ns.first();
-                                
-                                return xpath_string_const(qualified_name(na));
+
+                                return xpath_string::from_const(qualified_name(na));
                         }
 
@@ -7761 +10456 @@
                         {
                                 xpath_node na = c.n;
-                                
-                                return xpath_string_const(namespace_uri(na));
+
+                                return xpath_string::from_const(namespace_uri(na));
                         }
 
@@ -7769 +10464 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ns = _left->eval_node_set(c, stack);
+                                xpath_node_set_raw ns = _left->eval_node_set(c, stack, nodeset_eval_first);
                                 xpath_node na = ns.first();
-                                
-                                return xpath_string_const(namespace_uri(na));
+
+                                return xpath_string::from_const(namespace_uri(na));
                         }
 
@@ -7794 +10489 @@
 
                                 const char_t* pos = find_substring(s.c_str(), p.c_str());
-                                
-                                return pos ? xpath_string(s.c_str(), pos, stack.result) : xpath_string();
-                        }
-                        
+
+                                return pos ? xpath_string::from_heap(s.c_str(), pos, stack.result) : xpath_string();
+                        }
+
                         case ast_func_substring_after:
                         {
@@ -7806 +10501 @@
                                 xpath_string s = _left->eval_string(c, swapped_stack);
                                 xpath_string p = _right->eval_string(c, swapped_stack);
-                                
+
                                 const char_t* pos = find_substring(s.c_str(), p.c_str());
                                 if (!pos) return xpath_string();
 
-                                const char_t* result = pos + p.length();
-
-                                return s.uses_heap() ? xpath_string(result, stack.result) : xpath_string_const(result);
+                                const char_t* rbegin = pos + p.length();
+                                const char_t* rend = s.c_str() + s.length();
+
+                                return s.uses_heap() ? xpath_string::from_heap(rbegin, rend, stack.result) : xpath_string::from_const(rbegin);
                         }
 
@@ -7825 +10521 @@
 
                                 double first = round_nearest(_right->eval_number(c, stack));
-                                
+
                                 if (is_nan(first)) return xpath_string(); // NaN
                                 else if (first >= s_length + 1) return xpath_string();
-                                
-                                size_t pos = first < 1 ? 1 : (size_t)first;
+
+                                size_t pos = first < 1 ? 1 : static_cast<size_t>(first);
                                 assert(1 <= pos && pos <= s_length + 1);
 
                                 const char_t* rbegin = s.c_str() + (pos - 1);
-                                
-                                return s.uses_heap() ? xpath_string(rbegin, stack.result) : xpath_string_const(rbegin);
-                        }
-                        
+                                const char_t* rend = s.c_str() + s.length();
+
+                                return s.uses_heap() ? xpath_string::from_heap(rbegin, rend, stack.result) : xpath_string::from_const(rbegin);
+                        }
+
                         case ast_func_substring_3:
                         {
@@ -7848 +10545 @@
                                 double first = round_nearest(_right->eval_number(c, stack));
                                 double last = first + round_nearest(_right->_next->eval_number(c, stack));
-                                
+
                                 if (is_nan(first) || is_nan(last)) return xpath_string();
                                 else if (first >= s_length + 1) return xpath_string();
                                 else if (first >= last) return xpath_string();
                                 else if (last < 1) return xpath_string();
-                                
-                                size_t pos = first < 1 ? 1 : (size_t)first;
-                                size_t end = last >= s_length + 1 ? s_length + 1 : (size_t)last;
+
+                                size_t pos = first < 1 ? 1 : static_cast<size_t>(first);
+                                size_t end = last >= s_length + 1 ? s_length + 1 : static_cast<size_t>(last);
 
                                 assert(1 <= pos && pos <= end && end <= s_length + 1);
@@ -7861 +10558 @@
                                 const char_t* rend = s.c_str() + (end - 1);
 
-                                return (end == s_length + 1 && !s.uses_heap()) ? xpath_string_const(rbegin) : xpath_string(rbegin, rend, stack.result);
+                                return (end == s_length + 1 && !s.uses_heap()) ? xpath_string::from_const(rbegin) : xpath_string::from_heap(rbegin, rend, stack.result);
                         }
 
@@ -7868 +10565 @@
                                 xpath_string s = string_value(c.n, stack.result);
 
-                                normalize_space(s.data(stack.result));
-
-                                return s;
+                                char_t* begin = s.data(stack.result);
+                                char_t* end = normalize_space(begin);
+
+                                return xpath_string::from_heap_preallocated(begin, end);
                         }
 
@@ -7877 +10575 @@
                                 xpath_string s = _left->eval_string(c, stack);
 
-                                normalize_space(s.data(stack.result));
-                        
-                                return s;
+                                char_t* begin = s.data(stack.result);
+                                char_t* end = normalize_space(begin);
+
+                                return xpath_string::from_heap_preallocated(begin, end);
                         }
 
@@ -7892 +10591 @@
                                 xpath_string to = _right->_next->eval_string(c, swapped_stack);
 
-                                translate(s.data(stack.result), from.c_str(), to.c_str());
-
-                                return s;
+                                char_t* begin = s.data(stack.result);
+                                char_t* end = translate(begin, from.c_str(), to.c_str(), to.length());
+
+                                return xpath_string::from_heap_preallocated(begin, end);
+                        }
+
+                        case ast_opt_translate_table:
+                        {
+                                xpath_string s = _left->eval_string(c, stack);
+
+                                char_t* begin = s.data(stack.result);
+                                char_t* end = translate_table(begin, _data.table);
+
+                                return xpath_string::from_heap_preallocated(begin, end);
                         }
 
@@ -7902 +10612 @@
 
                                 if (_rettype == xpath_type_string)
-                                        return xpath_string_const(_data.variable->get_string());
+                                        return xpath_string::from_const(_data.variable->get_string());
 
                                 // fallthrough to type conversion
@@ -7912 +10622 @@
                                 {
                                 case xpath_type_boolean:
-                                        return xpath_string_const(eval_boolean(c, stack) ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));
-                                        
+                                        return xpath_string::from_const(eval_boolean(c, stack) ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));
+
                                 case xpath_type_number:
                                         return convert_number_to_string(eval_number(c, stack), stack.result);
-                                        
+
                                 case xpath_type_node_set:
                                 {
@@ -7923 +10633 @@
                                         xpath_stack swapped_stack = {stack.temp, stack.result};
 
-                                        xpath_node_set_raw ns = eval_node_set(c, swapped_stack);
+                                        xpath_node_set_raw ns = eval_node_set(c, swapped_stack, nodeset_eval_first);
                                         return ns.empty() ? xpath_string() : string_value(ns.first(), stack.result);
                                 }
-                                
+
                                 default:
-                                        assert(!"Wrong expression for return type string");
+                                        assert(false && "Wrong expression for return type string");
                                         return xpath_string();
                                 }
@@ -7935 +10645 @@
                 }
 
-                xpath_node_set_raw eval_node_set(const xpath_context& c, const xpath_stack& stack)
+                xpath_node_set_raw eval_node_set(const xpath_context& c, const xpath_stack& stack, nodeset_eval_t eval)
                 {
                         switch (_type)
@@ -7945 +10655 @@
                                 xpath_stack swapped_stack = {stack.temp, stack.result};
 
-                                xpath_node_set_raw ls = _left->eval_node_set(c, swapped_stack);
-                                xpath_node_set_raw rs = _right->eval_node_set(c, stack);
-                                
+                                xpath_node_set_raw ls = _left->eval_node_set(c, swapped_stack, eval);
+                                xpath_node_set_raw rs = _right->eval_node_set(c, stack, eval);
+
                                 // we can optimize merging two sorted sets, but this is a very rare operation, so don't bother
-                        rs.set_type(xpath_node_set::type_unsorted);
+                                rs.set_type(xpath_node_set::type_unsorted);
 
                                 rs.append(ls.begin(), ls.end(), stack.result);
                                 rs.remove_duplicates();
-                                
+
                                 return rs;
                         }
 
                         case ast_filter:
-                        case ast_filter_posinv:
-                        {
-                                xpath_node_set_raw set = _left->eval_node_set(c, stack);
+                        {
+                                xpath_node_set_raw set = _left->eval_node_set(c, stack, _test == predicate_constant_one ? nodeset_eval_first : nodeset_eval_all);
 
                                 // either expression is a number or it contains position() call; sort by document order
-                                if (_type == ast_filter) set.sort_do();
-
-                                apply_predicate(set, 0, _right, stack);
-                        
+                                if (_test != predicate_posinv) set.sort_do();
+
+                                bool once = eval_once(set.type(), eval);
+
+                                apply_predicate(set, 0, stack, once);
+
                                 return set;
                         }
-                        
+
                         case ast_func_id:
                                 return xpath_node_set_raw();
-                        
+
                         case ast_step:
                         {
@@ -7978 +10689 @@
                                 {
                                 case axis_ancestor:
-                                        return step_do(c, stack, axis_to_type<axis_ancestor>());
-                                        
+                                        return step_do(c, stack, eval, axis_to_type<axis_ancestor>());
+
                                 case axis_ancestor_or_self:
-                                        return step_do(c, stack, axis_to_type<axis_ancestor_or_self>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_ancestor_or_self>());
 
                                 case axis_attribute:
-                                        return step_do(c, stack, axis_to_type<axis_attribute>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_attribute>());
 
                                 case axis_child:
-                                        return step_do(c, stack, axis_to_type<axis_child>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_child>());
+
                                 case axis_descendant:
-                                        return step_do(c, stack, axis_to_type<axis_descendant>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_descendant>());
 
                                 case axis_descendant_or_self:
-                                        return step_do(c, stack, axis_to_type<axis_descendant_or_self>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_descendant_or_self>());
 
                                 case axis_following:
-                                        return step_do(c, stack, axis_to_type<axis_following>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_following>());
+
                                 case axis_following_sibling:
-                                        return step_do(c, stack, axis_to_type<axis_following_sibling>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_following_sibling>());
+
                                 case axis_namespace:
                                         // namespaced axis is not supported
                                         return xpath_node_set_raw();
-                                
+
                                 case axis_parent:
-                                        return step_do(c, stack, axis_to_type<axis_parent>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_parent>());
+
                                 case axis_preceding:
-                                        return step_do(c, stack, axis_to_type<axis_preceding>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_preceding>());
 
                                 case axis_preceding_sibling:
-                                        return step_do(c, stack, axis_to_type<axis_preceding_sibling>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_preceding_sibling>());
+
                                 case axis_self:
-                                        return step_do(c, stack, axis_to_type<axis_self>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_self>());
+
+                                default:
+                                        assert(false && "Unknown axis");
+                                        return xpath_node_set_raw();
                                 }
                         }
@@ -8053 +10768 @@
 
                         default:
-                                assert(!"Wrong expression for return type node set");
+                                assert(false && "Wrong expression for return type node set");
                                 return xpath_node_set_raw();
                         }
                 }
-                
-                bool is_posinv()
+
+                void optimize(xpath_allocator* alloc)
+                {
+                        if (_left)
+                                _left->optimize(alloc);
+
+                        if (_right)
+                                _right->optimize(alloc);
+
+                        if (_next)
+                                _next->optimize(alloc);
+
+                        optimize_self(alloc);
+                }
+
+                void optimize_self(xpath_allocator* alloc)
+                {
+                        // Rewrite [position()=expr] with [expr]
+                        // Note that this step has to go before classification to recognize [position()=1]
+                        if ((_type == ast_filter || _type == ast_predicate) &&
+                                _right->_type == ast_op_equal && _right->_left->_type == ast_func_position && _right->_right->_rettype == xpath_type_number)
+                        {
+                                _right = _right->_right;
+                        }
+
+                        // Classify filter/predicate ops to perform various optimizations during evaluation
+                        if (_type == ast_filter || _type == ast_predicate)
+                        {
+                                assert(_test == predicate_default);
+
+                                if (_right->_type == ast_number_constant && _right->_data.number == 1.0)
+                                        _test = predicate_constant_one;
+                                else if (_right->_rettype == xpath_type_number && (_right->_type == ast_number_constant || _right->_type == ast_variable || _right->_type == ast_func_last))
+                                        _test = predicate_constant;
+                                else if (_right->_rettype != xpath_type_number && _right->is_posinv_expr())
+                                        _test = predicate_posinv;
+                        }
+
+                        // Rewrite descendant-or-self::node()/child::foo with descendant::foo
+                        // The former is a full form of //foo, the latter is much faster since it executes the node test immediately
+                        // Do a similar kind of rewrite for self/descendant/descendant-or-self axes
+                        // Note that we only rewrite positionally invariant steps (//foo[1] != /descendant::foo[1])
+                        if (_type == ast_step && (_axis == axis_child || _axis == axis_self || _axis == axis_descendant || _axis == axis_descendant_or_self) && _left &&
+                                _left->_type == ast_step && _left->_axis == axis_descendant_or_self && _left->_test == nodetest_type_node && !_left->_right &&
+                                is_posinv_step())
+                        {
+                                if (_axis == axis_child || _axis == axis_descendant)
+                                        _axis = axis_descendant;
+                                else
+                                        _axis = axis_descendant_or_self;
+
+                                _left = _left->_left;
+                        }
+
+                        // Use optimized lookup table implementation for translate() with constant arguments
+                        if (_type == ast_func_translate && _right->_type == ast_string_constant && _right->_next->_type == ast_string_constant)
+                        {
+                                unsigned char* table = translate_table_generate(alloc, _right->_data.string, _right->_next->_data.string);
+
+                                if (table)
+                                {
+                                        _type = ast_opt_translate_table;
+                                        _data.table = table;
+                                }
+                        }
+
+                        // Use optimized path for @attr = 'value' or @attr = $value
+                        if (_type == ast_op_equal &&
+                                _left->_type == ast_step && _left->_axis == axis_attribute && _left->_test == nodetest_name && !_left->_left && !_left->_right &&
+                                (_right->_type == ast_string_constant || (_right->_type == ast_variable && _right->_rettype == xpath_type_string)))
+                        {
+                                _type = ast_opt_compare_attribute;
+                        }
+                }
+
+                bool is_posinv_expr() const
                 {
                         switch (_type)
                         {
                         case ast_func_position:
+                        case ast_func_last:
                                 return false;
 
@@ -8076 +10866 @@
                         case ast_predicate:
                         case ast_filter:
-                        case ast_filter_posinv:
                                 return true;
 
                         default:
-                                if (_left && !_left->is_posinv()) return false;
-                                
+                                if (_left && !_left->is_posinv_expr()) return false;
+
                                 for (xpath_ast_node* n = _right; n; n = n->_next)
-                                        if (!n->is_posinv()) return false;
-                                        
+                                        if (!n->is_posinv_expr()) return false;
+
                                 return true;
                         }
                 }
 
+                bool is_posinv_step() const
+                {
+                        assert(_type == ast_step);
+
+                        for (xpath_ast_node* n = _right; n; n = n->_next)
+                        {
+                                assert(n->_type == ast_predicate);
+
+                                if (n->_test != predicate_posinv)
+                                        return false;
+                        }
+
+                        return true;
+                }
+
                 xpath_value_type rettype() const
                 {
@@ -8097 +10901 @@
         struct xpath_parser
         {
-            xpath_allocator* _alloc;
-            xpath_lexer _lexer;
+                xpath_allocator* _alloc;
+                xpath_lexer _lexer;
 
                 const char_t* _query;
@@ -8104 +10908 @@
 
                 xpath_parse_result* _result;
+
+                char_t _scratch[32];
 
         #ifdef PUGIXML_NO_EXCEPTIONS
@@ -8122 +10928 @@
 
                 void throw_error_oom()
-        {
-        #ifdef PUGIXML_NO_EXCEPTIONS
-            throw_error("Out of memory");
-        #else
-            throw std::bad_alloc();
-        #endif
-        }
+                {
+                #ifdef PUGIXML_NO_EXCEPTIONS
+                        throw_error("Out of memory");
+                #else
+                        throw std::bad_alloc();
+                #endif
+                }
 
                 void* alloc_node()
@@ -8147 +10953 @@
                                 char_t* c = static_cast<char_t*>(_alloc->allocate_nothrow((length + 1) * sizeof(char_t)));
                                 if (!c) throw_error_oom();
+                                assert(c); // workaround for clang static analysis
 
                                 memcpy(c, value.begin, length * sizeof(char_t));
@@ -8160 +10967 @@
                         assert(argc <= 1);
 
-                        if (argc == 1 && args[0]->rettype() != xpath_type_node_set) throw_error("Function has to be applied to node set");
+                        if (argc == 1 && args[0]->rettype() != xpath_type_node_set)
+                                throw_error("Function has to be applied to node set");
 
                         return new (alloc_node()) xpath_ast_node(argc == 0 ? type0 : type1, xpath_type_string, args[0]);
@@ -8172 +10980 @@
                                 if (name == PUGIXML_TEXT("boolean") && argc == 1)
                                         return new (alloc_node()) xpath_ast_node(ast_func_boolean, xpath_type_boolean, args[0]);
-                                        
+
                                 break;
-                        
+
                         case 'c':
                                 if (name == PUGIXML_TEXT("count") && argc == 1)
                                 {
-                                        if (args[0]->rettype() != xpath_type_node_set) throw_error("Function has to be applied to node set");
+                                        if (args[0]->rettype() != xpath_type_node_set)
+                                                throw_error("Function has to be applied to node set");
+
                                         return new (alloc_node()) xpath_ast_node(ast_func_count, xpath_type_number, args[0]);
                                 }
                                 else if (name == PUGIXML_TEXT("contains") && argc == 2)
-                                        return new (alloc_node()) xpath_ast_node(ast_func_contains, xpath_type_string, args[0], args[1]);
+                                        return new (alloc_node()) xpath_ast_node(ast_func_contains, xpath_type_boolean, args[0], args[1]);
                                 else if (name == PUGIXML_TEXT("concat") && argc >= 2)
                                         return new (alloc_node()) xpath_ast_node(ast_func_concat, xpath_type_string, args[0], args[1]);
                                 else if (name == PUGIXML_TEXT("ceiling") && argc == 1)
                                         return new (alloc_node()) xpath_ast_node(ast_func_ceiling, xpath_type_number, args[0]);
-                                        
+
                                 break;
-                        
+
                         case 'f':
                                 if (name == PUGIXML_TEXT("false") && argc == 0)
@@ -8195 +11005 @@
                                 else if (name == PUGIXML_TEXT("floor") && argc == 1)
                                         return new (alloc_node()) xpath_ast_node(ast_func_floor, xpath_type_number, args[0]);
-                                        
+
                                 break;
-                        
+
                         case 'i':
                                 if (name == PUGIXML_TEXT("id") && argc == 1)
                                         return new (alloc_node()) xpath_ast_node(ast_func_id, xpath_type_node_set, args[0]);
-                                        
+
                                 break;
-                        
+
                         case 'l':
                                 if (name == PUGIXML_TEXT("last") && argc == 0)
@@ -8211 +11021 @@
                                 else if (name == PUGIXML_TEXT("local-name") && argc <= 1)
                                         return parse_function_helper(ast_func_local_name_0, ast_func_local_name_1, argc, args);
-                        
+
                                 break;
-                        
+
                         case 'n':
                                 if (name == PUGIXML_TEXT("name") && argc <= 1)
@@ -8225 +11035 @@
                                 else if (name == PUGIXML_TEXT("number") && argc <= 1)
                                         return new (alloc_node()) xpath_ast_node(argc == 0 ? ast_func_number_0 : ast_func_number_1, xpath_type_number, args[0]);
-                        
+
                                 break;
-                        
+
                         case 'p':
                                 if (name == PUGIXML_TEXT("position") && argc == 0)
                                         return new (alloc_node()) xpath_ast_node(ast_func_position, xpath_type_number);
-                                
+
                                 break;
-                        
+
                         case 'r':
                                 if (name == PUGIXML_TEXT("round") && argc == 1)
@@ -8239 +11049 @@
 
                                 break;
-                        
+
                         case 's':
                                 if (name == PUGIXML_TEXT("string") && argc <= 1)
                                         return new (alloc_node()) xpath_ast_node(argc == 0 ? ast_func_string_0 : ast_func_string_1, xpath_type_string, args[0]);
                                 else if (name == PUGIXML_TEXT("string-length") && argc <= 1)
-                                        return new (alloc_node()) xpath_ast_node(argc == 0 ? ast_func_string_length_0 : ast_func_string_length_1, xpath_type_string, args[0]);
+                                        return new (alloc_node()) xpath_ast_node(argc == 0 ? ast_func_string_length_0 : ast_func_string_length_1, xpath_type_number, args[0]);
                                 else if (name == PUGIXML_TEXT("starts-with") && argc == 2)
                                         return new (alloc_node()) xpath_ast_node(ast_func_starts_with, xpath_type_boolean, args[0], args[1]);
@@ -8260 +11070 @@
 
                                 break;
-                        
+
                         case 't':
                                 if (name == PUGIXML_TEXT("translate") && argc == 3)
@@ -8266 +11076 @@
                                 else if (name == PUGIXML_TEXT("true") && argc == 0)
                                         return new (alloc_node()) xpath_ast_node(ast_func_true, xpath_type_boolean);
-                                        
+
+                                break;
+
+                        default:
                                 break;
                         }
@@ -8288 +11101 @@
                                 else if (name == PUGIXML_TEXT("attribute"))
                                         return axis_attribute;
-                                
+
                                 break;
-                        
+
                         case 'c':
                                 if (name == PUGIXML_TEXT("child"))
                                         return axis_child;
-                                
+
                                 break;
-                        
+
                         case 'd':
                                 if (name == PUGIXML_TEXT("descendant"))
@@ -8302 +11115 @@
                                 else if (name == PUGIXML_TEXT("descendant-or-self"))
                                         return axis_descendant_or_self;
-                                
+
                                 break;
-                        
+
                         case 'f':
                                 if (name == PUGIXML_TEXT("following"))
@@ -8310 +11123 @@
                                 else if (name == PUGIXML_TEXT("following-sibling"))
                                         return axis_following_sibling;
-                                
+
                                 break;
-                        
+
                         case 'n':
                                 if (name == PUGIXML_TEXT("namespace"))
                                         return axis_namespace;
-                                
+
                                 break;
-                        
+
                         case 'p':
                                 if (name == PUGIXML_TEXT("parent"))
@@ -8326 +11139 @@
                                 else if (name == PUGIXML_TEXT("preceding-sibling"))
                                         return axis_preceding_sibling;
-                                
+
                                 break;
-                        
+
                         case 's':
                                 if (name == PUGIXML_TEXT("self"))
                                         return axis_self;
-                                
+
+                                break;
+
+                        default:
                                 break;
                         }
@@ -8367 +11183 @@
 
                                 break;
+
+                        default:
+                                break;
                         }
 
@@ -8372 +11191 @@
                 }
 
-            // PrimaryExpr ::= VariableReference | '(' Expr ')' | Literal | Number | FunctionCall
-            xpath_ast_node* parse_primary_expression()
-            {
-                switch (_lexer.current())
-                {
-                case lex_var_ref:
-                {
+                // PrimaryExpr ::= VariableReference | '(' Expr ')' | Literal | Number | FunctionCall
+                xpath_ast_node* parse_primary_expression()
+                {
+                        switch (_lexer.current())
+                        {
+                        case lex_var_ref:
+                        {
                                 xpath_lexer_string name = _lexer.contents();
 
@@ -8384 +11203 @@
                                         throw_error("Unknown variable: variable set is not provided");
 
-                                xpath_variable* var = get_variable(_variables, name.begin, name.end);
+                                xpath_variable* var = 0;
+                                if (!get_variable_scratch(_scratch, _variables, name.begin, name.end, &var))
+                                        throw_error_oom();
 
                                 if (!var)
@@ -8391 +11212 @@
                                 _lexer.next();
 
-                        return new (alloc_node()) xpath_ast_node(ast_variable, var->type(), var);
+                                return new (alloc_node()) xpath_ast_node(ast_variable, var->type(), var);
                         }
 
@@ -8422 +11243 @@
                                 double value = 0;
 
-                                if (!convert_string_to_number(_lexer.contents().begin, _lexer.contents().end, &value))
+                                if (!convert_string_to_number_scratch(_scratch, _lexer.contents().begin, _lexer.contents().end, &value))
                                         throw_error_oom();
 
@@ -8435 +11256 @@
                                 xpath_ast_node* args[2] = {0};
                                 size_t argc = 0;
-                                
+
                                 xpath_lexer_string function = _lexer.contents();
                                 _lexer.next();
-                                
+
                                 xpath_ast_node* last_arg = 0;
-                                
+
                                 if (_lexer.current() != lex_open_brace)
                                         throw_error("Unrecognized function call");
@@ -8453 +11274 @@
                                                 throw_error("No comma between function arguments");
                                         _lexer.next();
-                                        
+
                                         xpath_ast_node* n = parse_expression();
-                                        
+
                                         if (argc < 2) args[argc] = n;
                                         else last_arg->set_next(n);
@@ -8462 +11283 @@
                                         last_arg = n;
                                 }
-                                
+
                                 _lexer.next();
 
@@ -8468 +11289 @@
                         }
 
-                default:
-                        throw_error("Unrecognizable primary expression");
-
-                        return 0;
-                }
-            }
-            
-            // FilterExpr ::= PrimaryExpr | FilterExpr Predicate
-            // Predicate ::= '[' PredicateExpr ']'
-            // PredicateExpr ::= Expr
-            xpath_ast_node* parse_filter_expression()
-            {
-                xpath_ast_node* n = parse_primary_expression();
-
-                while (_lexer.current() == lex_open_square_brace)
-                {
-                        _lexer.next();
+                        default:
+                                throw_error("Unrecognizable primary expression");
+
+                                return 0;
+                        }
+                }
+
+                // FilterExpr ::= PrimaryExpr | FilterExpr Predicate
+                // Predicate ::= '[' PredicateExpr ']'
+                // PredicateExpr ::= Expr
+                xpath_ast_node* parse_filter_expression()
+                {
+                        xpath_ast_node* n = parse_primary_expression();
+
+                        while (_lexer.current() == lex_open_square_brace)
+                        {
+                                _lexer.next();
 
                                 xpath_ast_node* expr = parse_expression();
 
-                                if (n->rettype() != xpath_type_node_set) throw_error("Predicate has to be applied to node set");
-
-                                bool posinv = expr->rettype() != xpath_type_number && expr->is_posinv();
-
-                        n = new (alloc_node()) xpath_ast_node(posinv ? ast_filter_posinv : ast_filter, xpath_type_node_set, n, expr);
-
-                        if (_lexer.current() != lex_close_square_brace)
-                                throw_error("Unmatched square brace");
-                
-                        _lexer.next();
-                }
-                
-                return n;
-            }
-            
-            // Step ::= AxisSpecifier NodeTest Predicate* | AbbreviatedStep
-            // AxisSpecifier ::= AxisName '::' | '@'?
-            // NodeTest ::= NameTest | NodeType '(' ')' | 'processing-instruction' '(' Literal ')'
-            // NameTest ::= '*' | NCName ':' '*' | QName
-            // AbbreviatedStep ::= '.' | '..'
-            xpath_ast_node* parse_step(xpath_ast_node* set)
-            {
+                                if (n->rettype() != xpath_type_node_set)
+                                        throw_error("Predicate has to be applied to node set");
+
+                                n = new (alloc_node()) xpath_ast_node(ast_filter, n, expr, predicate_default);
+
+                                if (_lexer.current() != lex_close_square_brace)
+                                        throw_error("Unmatched square brace");
+
+                                _lexer.next();
+                        }
+
+                        return n;
+                }
+
+                // Step ::= AxisSpecifier NodeTest Predicate* | AbbreviatedStep
+                // AxisSpecifier ::= AxisName '::' | '@'?
+                // NodeTest ::= NameTest | NodeType '(' ')' | 'processing-instruction' '(' Literal ')'
+                // NameTest ::= '*' | NCName ':' '*' | QName
+                // AbbreviatedStep ::= '.' | '..'
+                xpath_ast_node* parse_step(xpath_ast_node* set)
+                {
                         if (set && set->rettype() != xpath_type_node_set)
                                 throw_error("Step has to be applied to node set");
@@ -8520 +11340 @@
                                 axis = axis_attribute;
                                 axis_specified = true;
-                                
+
                                 _lexer.next();
                         }
@@ -8526 +11346 @@
                         {
                                 _lexer.next();
-                                
+
                                 return new (alloc_node()) xpath_ast_node(ast_step, set, axis_self, nodetest_type_node, 0);
                         }
@@ -8532 +11352 @@
                         {
                                 _lexer.next();
-                                
+
                                 return new (alloc_node()) xpath_ast_node(ast_step, set, axis_parent, nodetest_type_node, 0);
                         }
-            
+
                         nodetest_t nt_type = nodetest_none;
                         xpath_lexer_string nt_name;
-                        
+
                         if (_lexer.current() == lex_string)
                         {
@@ -8549 +11369 @@
                                 {
                                         // parse axis name
-                                        if (axis_specified) throw_error("Two axis specifiers in one step");
+                                        if (axis_specified)
+                                                throw_error("Two axis specifiers in one step");
 
                                         axis = parse_axis_name(nt_name, axis_specified);
 
-                                        if (!axis_specified) throw_error("Unknown axis");
+                                        if (!axis_specified)
+                                                throw_error("Unknown axis");
 
                                         // read actual node test
@@ -8571 +11393 @@
                                         else throw_error("Unrecognized node test");
                                 }
-                                
+
                                 if (nt_type == nodetest_none)
                                 {
@@ -8578 +11400 @@
                                         {
                                                 _lexer.next();
-                                                
+
                                                 if (_lexer.current() == lex_close_brace)
                                                 {
@@ -8585 +11407 @@
                                                         nt_type = parse_node_test_type(nt_name);
 
-                                                        if (nt_type == nodetest_none) throw_error("Unrecognized node type");
-                                                        
+                                                        if (nt_type == nodetest_none)
+                                                                throw_error("Unrecognized node type");
+
                                                         nt_name = xpath_lexer_string();
                                                 }
@@ -8593 +11416 @@
                                                         if (_lexer.current() != lex_quoted_string)
                                                                 throw_error("Only literals are allowed as arguments to processing-instruction()");
-                                                
+
                                                         nt_type = nodetest_pi;
                                                         nt_name = _lexer.contents();
                                                         _lexer.next();
-                                                        
+
                                                         if (_lexer.current() != lex_close_brace)
                                                                 throw_error("Unmatched brace near processing-instruction()");
@@ -8603 +11426 @@
                                                 }
                                                 else
+                                                {
                                                         throw_error("Unmatched brace near node type test");
-
+                                                }
                                         }
                                         // QName or NCName:*
@@ -8612 +11436 @@
                                                 {
                                                         nt_name.end--; // erase *
-                                                        
+
                                                         nt_type = nodetest_all_in_namespace;
                                                 }
-                                                else nt_type = nodetest_name;
+                                                else
+                                                {
+                                                        nt_type = nodetest_name;
+                                                }
                                         }
                                 }
@@ -8624 +11451 @@
                                 _lexer.next();
                         }
-                        else throw_error("Unrecognized node test");
-                        
-                        xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step, set, axis, nt_type, alloc_string(nt_name));
-                        
+                        else
+                        {
+                                throw_error("Unrecognized node test");
+                        }
+
+                        const char_t* nt_name_copy = alloc_string(nt_name);
+                        xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step, set, axis, nt_type, nt_name_copy);
+
                         xpath_ast_node* last = 0;
-                        
+
                         while (_lexer.current() == lex_open_square_brace)
                         {
                                 _lexer.next();
-                                
+
                                 xpath_ast_node* expr = parse_expression();
 
-                                xpath_ast_node* pred = new (alloc_node()) xpath_ast_node(ast_predicate, xpath_type_node_set, expr);
-                                
+                                xpath_ast_node* pred = new (alloc_node()) xpath_ast_node(ast_predicate, 0, expr, predicate_default);
+
                                 if (_lexer.current() != lex_close_square_brace)
-                                throw_error("Unmatched square brace");
+                                        throw_error("Unmatched square brace");
                                 _lexer.next();
-                                
+
                                 if (last) last->set_next(pred);
                                 else n->set_right(pred);
-                                
+
                                 last = pred;
                         }
-                        
+
                         return n;
-            }
-            
-            // RelativeLocationPath ::= Step | RelativeLocationPath '/' Step | RelativeLocationPath '//' Step
-            xpath_ast_node* parse_relative_location_path(xpath_ast_node* set)
-            {
+                }
+
+                // RelativeLocationPath ::= Step | RelativeLocationPath '/' Step | RelativeLocationPath '//' Step
+                xpath_ast_node* parse_relative_location_path(xpath_ast_node* set)
+                {
                         xpath_ast_node* n = parse_step(set);
-                        
+
                         while (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
                         {
@@ -8663 +11494 @@
                                 if (l == lex_double_slash)
                                         n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
-                                
+
                                 n = parse_step(n);
                         }
-                        
+
                         return n;
-            }
-            
-            // LocationPath ::= RelativeLocationPath | AbsoluteLocationPath
-            // AbsoluteLocationPath ::= '/' RelativeLocationPath? | '//' RelativeLocationPath
-            xpath_ast_node* parse_location_path()
-            {
+                }
+
+                // LocationPath ::= RelativeLocationPath | AbsoluteLocationPath
+                // AbsoluteLocationPath ::= '/' RelativeLocationPath? | '//' RelativeLocationPath
+                xpath_ast_node* parse_location_path()
+                {
                         if (_lexer.current() == lex_slash)
                         {
                                 _lexer.next();
-                                
+
                                 xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step_root, xpath_type_node_set);
 
@@ -8691 +11522 @@
                         {
                                 _lexer.next();
-                                
+
                                 xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step_root, xpath_type_node_set);
                                 n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
-                                
+
                                 return parse_relative_location_path(n);
                         }
@@ -8700 +11531 @@
                         // else clause moved outside of if because of bogus warning 'control may reach end of non-void function being inlined' in gcc 4.0.1
                         return parse_relative_location_path(0);
-            }
-            
-            // PathExpr ::= LocationPath
-            //                          | FilterExpr
-            //                          | FilterExpr '/' RelativeLocationPath
-            //                          | FilterExpr '//' RelativeLocationPath
-            xpath_ast_node* parse_path_expression()
-            {
+                }
+
+                // PathExpr ::= LocationPath
+                //                              | FilterExpr
+                //                              | FilterExpr '/' RelativeLocationPath
+                //                              | FilterExpr '//' RelativeLocationPath
+                // UnionExpr ::= PathExpr | UnionExpr '|' PathExpr
+                // UnaryExpr ::= UnionExpr | '-' UnaryExpr
+                xpath_ast_node* parse_path_or_unary_expression()
+                {
                         // Clarification.
                         // PathExpr begins with either LocationPath or FilterExpr.
@@ -8715 +11548 @@
                         // function call.
 
-                        if (_lexer.current() == lex_var_ref || _lexer.current() == lex_open_brace || 
+                        if (_lexer.current() == lex_var_ref || _lexer.current() == lex_open_brace ||
                                 _lexer.current() == lex_quoted_string || _lexer.current() == lex_number ||
                                 _lexer.current() == lex_string)
-                {
-                        if (_lexer.current() == lex_string)
-                        {
-                                // This is either a function call, or not - if not, we shall proceed with location path
-                                const char_t* state = _lexer.state();
-                                
-                                        while (IS_CHARTYPE(*state, ct_space)) ++state;
-                                
-                                if (*state != '(') return parse_location_path();
+                        {
+                                if (_lexer.current() == lex_string)
+                                {
+                                        // This is either a function call, or not - if not, we shall proceed with location path
+                                        const char_t* state = _lexer.state();
+
+                                        while (PUGI__IS_CHARTYPE(*state, ct_space)) ++state;
+
+                                        if (*state != '(') return parse_location_path();
 
                                         // This looks like a function call; however this still can be a node-test. Check it.
-                                        if (parse_node_test_type(_lexer.contents()) != nodetest_none) return parse_location_path();
-                        }
-                        
-                        xpath_ast_node* n = parse_filter_expression();
-
-                        if (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
-                        {
+                                        if (parse_node_test_type(_lexer.contents()) != nodetest_none)
+                                                return parse_location_path();
+                                }
+
+                                xpath_ast_node* n = parse_filter_expression();
+
+                                if (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
+                                {
                                         lexeme_t l = _lexer.current();
-                                _lexer.next();
-                                
+                                        _lexer.next();
+
                                         if (l == lex_double_slash)
                                         {
-                                                if (n->rettype() != xpath_type_node_set) throw_error("Step has to be applied to node set");
+                                                if (n->rettype() != xpath_type_node_set)
+                                                        throw_error("Step has to be applied to node set");
 
                                                 n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
                                         }
-        
-                                // select from location path
-                                return parse_relative_location_path(n);
-                        }
-
-                        return n;
-                }
-                else return parse_location_path();
-            }
-
-            // UnionExpr ::= PathExpr | UnionExpr '|' PathExpr
-            xpath_ast_node* parse_union_expression()
-            {
-                xpath_ast_node* n = parse_path_expression();
-
-                while (_lexer.current() == lex_union)
-                {
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_union_expression();
-
-                                if (n->rettype() != xpath_type_node_set || expr->rettype() != xpath_type_node_set)
+
+                                        // select from location path
+                                        return parse_relative_location_path(n);
+                                }
+
+                                return n;
+                        }
+                        else if (_lexer.current() == lex_minus)
+                        {
+                                _lexer.next();
+
+                                // precedence 7+ - only parses union expressions
+                                xpath_ast_node* expr = parse_expression_rec(parse_path_or_unary_expression(), 7);
+
+                                return new (alloc_node()) xpath_ast_node(ast_op_negate, xpath_type_number, expr);
+                        }
+                        else
+                        {
+                                return parse_location_path();
+                        }
+                }
+
+                struct binary_op_t
+                {
+                        ast_type_t asttype;
+                        xpath_value_type rettype;
+                        int precedence;
+
+                        binary_op_t(): asttype(ast_unknown), rettype(xpath_type_none), precedence(0)
+                        {
+                        }
+
+                        binary_op_t(ast_type_t asttype_, xpath_value_type rettype_, int precedence_): asttype(asttype_), rettype(rettype_), precedence(precedence_)
+                        {
+                        }
+
+                        static binary_op_t parse(xpath_lexer& lexer)
+                        {
+                                switch (lexer.current())
+                                {
+                                case lex_string:
+                                        if (lexer.contents() == PUGIXML_TEXT("or"))
+                                                return binary_op_t(ast_op_or, xpath_type_boolean, 1);
+                                        else if (lexer.contents() == PUGIXML_TEXT("and"))
+                                                return binary_op_t(ast_op_and, xpath_type_boolean, 2);
+                                        else if (lexer.contents() == PUGIXML_TEXT("div"))
+                                                return binary_op_t(ast_op_divide, xpath_type_number, 6);
+                                        else if (lexer.contents() == PUGIXML_TEXT("mod"))
+                                                return binary_op_t(ast_op_mod, xpath_type_number, 6);
+                                        else
+                                                return binary_op_t();
+
+                                case lex_equal:
+                                        return binary_op_t(ast_op_equal, xpath_type_boolean, 3);
+
+                                case lex_not_equal:
+                                        return binary_op_t(ast_op_not_equal, xpath_type_boolean, 3);
+
+                                case lex_less:
+                                        return binary_op_t(ast_op_less, xpath_type_boolean, 4);
+
+                                case lex_greater:
+                                        return binary_op_t(ast_op_greater, xpath_type_boolean, 4);
+
+                                case lex_less_or_equal:
+                                        return binary_op_t(ast_op_less_or_equal, xpath_type_boolean, 4);
+
+                                case lex_greater_or_equal:
+                                        return binary_op_t(ast_op_greater_or_equal, xpath_type_boolean, 4);
+
+                                case lex_plus:
+                                        return binary_op_t(ast_op_add, xpath_type_number, 5);
+
+                                case lex_minus:
+                                        return binary_op_t(ast_op_subtract, xpath_type_number, 5);
+
+                                case lex_multiply:
+                                        return binary_op_t(ast_op_multiply, xpath_type_number, 6);
+
+                                case lex_union:
+                                        return binary_op_t(ast_op_union, xpath_type_node_set, 7);
+
+                                default:
+                                        return binary_op_t();
+                                }
+                        }
+                };
+
+                xpath_ast_node* parse_expression_rec(xpath_ast_node* lhs, int limit)
+                {
+                        binary_op_t op = binary_op_t::parse(_lexer);
+
+                        while (op.asttype != ast_unknown && op.precedence >= limit)
+                        {
+                                _lexer.next();
+
+                                xpath_ast_node* rhs = parse_path_or_unary_expression();
+
+                                binary_op_t nextop = binary_op_t::parse(_lexer);
+
+                                while (nextop.asttype != ast_unknown && nextop.precedence > op.precedence)
+                                {
+                                        rhs = parse_expression_rec(rhs, nextop.precedence);
+
+                                        nextop = binary_op_t::parse(_lexer);
+                                }
+
+                                if (op.asttype == ast_op_union && (lhs->rettype() != xpath_type_node_set || rhs->rettype() != xpath_type_node_set))
                                         throw_error("Union operator has to be applied to node sets");
 
-                        n = new (alloc_node()) xpath_ast_node(ast_op_union, xpath_type_node_set, n, expr);
-                }
-
-                return n;
-            }
-
-            // UnaryExpr ::= UnionExpr | '-' UnaryExpr
-            xpath_ast_node* parse_unary_expression()
-            {
-                if (_lexer.current() == lex_minus)
-                {
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_unary_expression();
-
-                        return new (alloc_node()) xpath_ast_node(ast_op_negate, xpath_type_number, expr);
-                }
-                else return parse_union_expression();
-            }
-            
-            // MultiplicativeExpr ::= UnaryExpr
-            //                                            | MultiplicativeExpr '*' UnaryExpr
-            //                                            | MultiplicativeExpr 'div' UnaryExpr
-            //                                            | MultiplicativeExpr 'mod' UnaryExpr
-            xpath_ast_node* parse_multiplicative_expression()
-            {
-                xpath_ast_node* n = parse_unary_expression();
-
-                while (_lexer.current() == lex_multiply || (_lexer.current() == lex_string &&
-                           (_lexer.contents() == PUGIXML_TEXT("mod") || _lexer.contents() == PUGIXML_TEXT("div"))))
-                {
-                        ast_type_t op = _lexer.current() == lex_multiply ? ast_op_multiply :
-                                _lexer.contents().begin[0] == 'd' ? ast_op_divide : ast_op_mod;
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_unary_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(op, xpath_type_number, n, expr);
-                }
-
-                return n;
-            }
-
-            // AdditiveExpr ::= MultiplicativeExpr
-            //                                  | AdditiveExpr '+' MultiplicativeExpr
-            //                                  | AdditiveExpr '-' MultiplicativeExpr
-            xpath_ast_node* parse_additive_expression()
-            {
-                xpath_ast_node* n = parse_multiplicative_expression();
-
-                while (_lexer.current() == lex_plus || _lexer.current() == lex_minus)
-                {
-                        lexeme_t l = _lexer.current();
-
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_multiplicative_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(l == lex_plus ? ast_op_add : ast_op_subtract, xpath_type_number, n, expr);
-                }
-
-                return n;
-            }
-
-            // RelationalExpr ::= AdditiveExpr
-            //                                    | RelationalExpr '<' AdditiveExpr
-            //                                    | RelationalExpr '>' AdditiveExpr
-            //                                    | RelationalExpr '<=' AdditiveExpr
-            //                                    | RelationalExpr '>=' AdditiveExpr
-            xpath_ast_node* parse_relational_expression()
-            {
-                xpath_ast_node* n = parse_additive_expression();
-
-                while (_lexer.current() == lex_less || _lexer.current() == lex_less_or_equal || 
-                           _lexer.current() == lex_greater || _lexer.current() == lex_greater_or_equal)
-                {
-                        lexeme_t l = _lexer.current();
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_additive_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(l == lex_less ? ast_op_less : l == lex_greater ? ast_op_greater :
-                                                        l == lex_less_or_equal ? ast_op_less_or_equal : ast_op_greater_or_equal, xpath_type_boolean, n, expr);
-                }
-
-                return n;
-            }
-            
-            // EqualityExpr ::= RelationalExpr
-            //                                  | EqualityExpr '=' RelationalExpr
-            //                                  | EqualityExpr '!=' RelationalExpr
-            xpath_ast_node* parse_equality_expression()
-            {
-                xpath_ast_node* n = parse_relational_expression();
-
-                while (_lexer.current() == lex_equal || _lexer.current() == lex_not_equal)
-                {
-                        lexeme_t l = _lexer.current();
-
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_relational_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(l == lex_equal ? ast_op_equal : ast_op_not_equal, xpath_type_boolean, n, expr);
-                }
-
-                return n;
-            }
-            
-            // AndExpr ::= EqualityExpr | AndExpr 'and' EqualityExpr
-            xpath_ast_node* parse_and_expression()
-            {
-                xpath_ast_node* n = parse_equality_expression();
-
-                while (_lexer.current() == lex_string && _lexer.contents() == PUGIXML_TEXT("and"))
-                {
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_equality_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(ast_op_and, xpath_type_boolean, n, expr);
-                }
-
-                return n;
-            }
-
-            // OrExpr ::= AndExpr | OrExpr 'or' AndExpr
-            xpath_ast_node* parse_or_expression()
-            {
-                xpath_ast_node* n = parse_and_expression();
-
-                while (_lexer.current() == lex_string && _lexer.contents() == PUGIXML_TEXT("or"))
-                {
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_and_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(ast_op_or, xpath_type_boolean, n, expr);
-                }
-
-                return n;
-            }
-                
+                                lhs = new (alloc_node()) xpath_ast_node(op.asttype, op.rettype, lhs, rhs);
+
+                                op = binary_op_t::parse(_lexer);
+                        }
+
+                        return lhs;
+                }
+
                 // Expr ::= OrExpr
+                // OrExpr ::= AndExpr | OrExpr 'or' AndExpr
+                // AndExpr ::= EqualityExpr | AndExpr 'and' EqualityExpr
+                // EqualityExpr ::= RelationalExpr
+                //                                      | EqualityExpr '=' RelationalExpr
+                //                                      | EqualityExpr '!=' RelationalExpr
+                // RelationalExpr ::= AdditiveExpr
+                //                                        | RelationalExpr '<' AdditiveExpr
+                //                                        | RelationalExpr '>' AdditiveExpr
+                //                                        | RelationalExpr '<=' AdditiveExpr
+                //                                        | RelationalExpr '>=' AdditiveExpr
+                // AdditiveExpr ::= MultiplicativeExpr
+                //                                      | AdditiveExpr '+' MultiplicativeExpr
+                //                                      | AdditiveExpr '-' MultiplicativeExpr
+                // MultiplicativeExpr ::= UnaryExpr
+                //                                                | MultiplicativeExpr '*' UnaryExpr
+                //                                                | MultiplicativeExpr 'div' UnaryExpr
+                //                                                | MultiplicativeExpr 'mod' UnaryExpr
                 xpath_ast_node* parse_expression()
                 {
-                        return parse_or_expression();
+                        return parse_expression_rec(parse_path_or_unary_expression(), 0);
                 }
 
@@ -8925 +11728 @@
                 {
                         xpath_ast_node* result = parse_expression();
-                        
+
+                        // check if there are unparsed tokens left
                         if (_lexer.current() != lex_eof)
-                        {
-                                // there are still unparsed tokens left, error
                                 throw_error("Incorrect query");
-                        }
-                        
+
                         return result;
                 }
@@ -8949 +11750 @@
         };
 
-    struct xpath_query_impl
-    {
+        struct xpath_query_impl
+        {
                 static xpath_query_impl* create()
                 {
-                        void* memory = global_allocate(sizeof(xpath_query_impl));
-
-            return new (memory) xpath_query_impl();
-                }
-
-                static void destroy(void* ptr)
-                {
-                        if (!ptr) return;
-                        
+                        void* memory = xml_memory::allocate(sizeof(xpath_query_impl));
+                        if (!memory) return 0;
+
+                        return new (memory) xpath_query_impl();
+                }
+
+                static void destroy(xpath_query_impl* impl)
+                {
                         // free all allocated pages
-                        static_cast<xpath_query_impl*>(ptr)->alloc.release();
+                        impl->alloc.release();
 
                         // free allocator memory (with the first page)
-                        global_deallocate(ptr);
-                }
-
-        xpath_query_impl(): root(0), alloc(&block)
-        {
-            block.next = 0;
-        }
-
-        xpath_ast_node* root;
-        xpath_allocator alloc;
-        xpath_memory_block block;
-    };
-
-        xpath_string evaluate_string_impl(xpath_query_impl* impl, const xpath_node& n, xpath_stack_data& sd)
+                        xml_memory::deallocate(impl);
+                }
+
+                xpath_query_impl(): root(0), alloc(&block)
+                {
+                        block.next = 0;
+                        block.capacity = sizeof(block.data);
+                }
+
+                xpath_ast_node* root;
+                xpath_allocator alloc;
+                xpath_memory_block block;
+        };
+
+        PUGI__FN xpath_string evaluate_string_impl(xpath_query_impl* impl, const xpath_node& n, xpath_stack_data& sd)
         {
                 if (!impl) return xpath_string();
@@ -8991 +11792 @@
                 return impl->root->eval_string(c, sd.stack);
         }
-}
+
+        PUGI__FN impl::xpath_ast_node* evaluate_node_set_prepare(xpath_query_impl* impl)
+        {
+                if (!impl) return 0;
+
+                if (impl->root->rettype() != xpath_type_node_set)
+                {
+                #ifdef PUGIXML_NO_EXCEPTIONS
+                        return 0;
+                #else
+                        xpath_parse_result res;
+                        res.error = "Expression does not evaluate to node set";
+
+                        throw xpath_exception(res);
+                #endif
+                }
+
+                return impl->root;
+        }
+PUGI__NS_END
 
 namespace pugi
 {
 #ifndef PUGIXML_NO_EXCEPTIONS
-        xpath_exception::xpath_exception(const xpath_parse_result& result): _result(result)
-        {
-                assert(result.error);
-        }
-        
-        const char* xpath_exception::what() const throw()
+        PUGI__FN xpath_exception::xpath_exception(const xpath_parse_result& result_): _result(result_)
+        {
+                assert(_result.error);
+        }
+
+        PUGI__FN const char* xpath_exception::what() const throw()
         {
                 return _result.error;
         }
 
-        const xpath_parse_result& xpath_exception::result() const
+        PUGI__FN const xpath_parse_result& xpath_exception::result() const
         {
                 return _result;
         }
 #endif
-        
-        xpath_node::xpath_node()
-        {
-        }
-                
-        xpath_node::xpath_node(const xml_node& node): _node(node)
-        {
-        }
-                
-        xpath_node::xpath_node(const xml_attribute& attribute, const xml_node& parent): _node(attribute ? parent : xml_node()), _attribute(attribute)
-        {
-        }
-
-        xml_node xpath_node::node() const
+
+        PUGI__FN xpath_node::xpath_node()
+        {
+        }
+
+        PUGI__FN xpath_node::xpath_node(const xml_node& node_): _node(node_)
+        {
+        }
+
+        PUGI__FN xpath_node::xpath_node(const xml_attribute& attribute_, const xml_node& parent_): _node(attribute_ ? parent_ : xml_node()), _attribute(attribute_)
+        {
+        }
+
+        PUGI__FN xml_node xpath_node::node() const
         {
                 return _attribute ? xml_node() : _node;
         }
-                
-        xml_attribute xpath_node::attribute() const
+
+        PUGI__FN xml_attribute xpath_node::attribute() const
         {
                 return _attribute;
         }
-        
-        xml_node xpath_node::parent() const
+
+        PUGI__FN xml_node xpath_node::parent() const
         {
                 return _attribute ? _node : _node.parent();
         }
 
-        xpath_node::operator xpath_node::unspecified_bool_type() const
-        {
-                return (_node || _attribute) ? &xpath_node::_node : 0;
-        }
-        
-        bool xpath_node::operator!() const
+        PUGI__FN static void unspecified_bool_xpath_node(xpath_node***)
+        {
+        }
+
+        PUGI__FN xpath_node::operator xpath_node::unspecified_bool_type() const
+        {
+                return (_node || _attribute) ? unspecified_bool_xpath_node : 0;
+        }
+
+        PUGI__FN bool xpath_node::operator!() const
         {
                 return !(_node || _attribute);
         }
 
-        bool xpath_node::operator==(const xpath_node& n) const
+        PUGI__FN bool xpath_node::operator==(const xpath_node& n) const
         {
                 return _node == n._node && _attribute == n._attribute;
         }
-        
-        bool xpath_node::operator!=(const xpath_node& n) const
+
+        PUGI__FN bool xpath_node::operator!=(const xpath_node& n) const
         {
                 return _node != n._node || _attribute != n._attribute;
@@ -9060 +11884 @@
 
 #ifdef __BORLANDC__
-        bool operator&&(const xpath_node& lhs, bool rhs)
+        PUGI__FN bool operator&&(const xpath_node& lhs, bool rhs)
         {
                 return (bool)lhs && rhs;
         }
 
-        bool operator||(const xpath_node& lhs, bool rhs)
+        PUGI__FN bool operator||(const xpath_node& lhs, bool rhs)
         {
                 return (bool)lhs || rhs;
@@ -9071 +11895 @@
 #endif
 
-        void xpath_node_set::_assign(const_iterator begin, const_iterator end)
-        {
-                assert(begin <= end);
-
-                size_t size = static_cast<size_t>(end - begin);
-
-                if (size <= 1)
+        PUGI__FN void xpath_node_set::_assign(const_iterator begin_, const_iterator end_, type_t type_)
+        {
+                assert(begin_ <= end_);
+
+                size_t size_ = static_cast<size_t>(end_ - begin_);
+
+                if (size_ <= 1)
                 {
                         // deallocate old buffer
-                        if (_begin != &_storage) global_deallocate(_begin);
+                        if (_begin != &_storage) impl::xml_memory::deallocate(_begin);
 
                         // use internal buffer
-                        if (begin != end) _storage = *begin;
+                        if (begin_ != end_) _storage = *begin_;
 
                         _begin = &_storage;
-                        _end = &_storage + size;
+                        _end = &_storage + size_;
+                        _type = type_;
                 }
                 else
                 {
                         // make heap copy
-                        xpath_node* storage = static_cast<xpath_node*>(global_allocate(size * sizeof(xpath_node)));
+                        xpath_node* storage = static_cast<xpath_node*>(impl::xml_memory::allocate(size_ * sizeof(xpath_node)));
 
                         if (!storage)
@@ -9102 +11927 @@
                         }
 
-                        memcpy(storage, begin, size * sizeof(xpath_node));
-                        
+                        memcpy(storage, begin_, size_ * sizeof(xpath_node));
+
                         // deallocate old buffer
-                        if (_begin != &_storage) global_deallocate(_begin);
+                        if (_begin != &_storage) impl::xml_memory::deallocate(_begin);
 
                         // finalize
                         _begin = storage;
-                        _end = storage + size;
-                }
-        }
-
-        xpath_node_set::xpath_node_set(): _type(type_unsorted), _begin(&_storage), _end(&_storage)
-        {
-        }
-
-        xpath_node_set::xpath_node_set(const_iterator begin, const_iterator end, type_t type): _type(type), _begin(&_storage), _end(&_storage)
-        {
-                _assign(begin, end);
-        }
-
-        xpath_node_set::~xpath_node_set()
-        {
-                if (_begin != &_storage) global_deallocate(_begin);
-        }
-                
-        xpath_node_set::xpath_node_set(const xpath_node_set& ns): _type(ns._type), _begin(&_storage), _end(&_storage)
-        {
-                _assign(ns._begin, ns._end);
-        }
-        
-        xpath_node_set& xpath_node_set::operator=(const xpath_node_set& ns)
+                        _end = storage + size_;
+                        _type = type_;
+                }
+        }
+
+#ifdef PUGIXML_HAS_MOVE
+        PUGI__FN void xpath_node_set::_move(xpath_node_set& rhs)
+        {
+                _type = rhs._type;
+                _storage = rhs._storage;
+                _begin = (rhs._begin == &rhs._storage) ? &_storage : rhs._begin;
+                _end = _begin + (rhs._end - rhs._begin);
+
+                rhs._type = type_unsorted;
+                rhs._begin = &rhs._storage;
+                rhs._end = rhs._begin;
+        }
+#endif
+
+        PUGI__FN xpath_node_set::xpath_node_set(): _type(type_unsorted), _begin(&_storage), _end(&_storage)
+        {
+        }
+
+        PUGI__FN xpath_node_set::xpath_node_set(const_iterator begin_, const_iterator end_, type_t type_): _type(type_unsorted), _begin(&_storage), _end(&_storage)
+        {
+                _assign(begin_, end_, type_);
+        }
+
+        PUGI__FN xpath_node_set::~xpath_node_set()
+        {
+                if (_begin != &_storage)
+                        impl::xml_memory::deallocate(_begin);
+        }
+
+        PUGI__FN xpath_node_set::xpath_node_set(const xpath_node_set& ns): _type(type_unsorted), _begin(&_storage), _end(&_storage)
+        {
+                _assign(ns._begin, ns._end, ns._type);
+        }
+
+        PUGI__FN xpath_node_set& xpath_node_set::operator=(const xpath_node_set& ns)
         {
                 if (this == &ns) return *this;
-                
-                _type = ns._type;
-                _assign(ns._begin, ns._end);
+
+                _assign(ns._begin, ns._end, ns._type);
 
                 return *this;
         }
 
-        xpath_node_set::type_t xpath_node_set::type() const
+#ifdef PUGIXML_HAS_MOVE
+        PUGI__FN xpath_node_set::xpath_node_set(xpath_node_set&& rhs): _type(type_unsorted), _begin(&_storage), _end(&_storage)
+        {
+                _move(rhs);
+        }
+
+        PUGI__FN xpath_node_set& xpath_node_set::operator=(xpath_node_set&& rhs)
+        {
+                if (this == &rhs) return *this;
+
+                if (_begin != &_storage)
+                        impl::xml_memory::deallocate(_begin);
+
+                _move(rhs);
+
+                return *this;
+        }
+#endif
+
+        PUGI__FN xpath_node_set::type_t xpath_node_set::type() const
         {
                 return _type;
         }
-                
-        size_t xpath_node_set::size() const
+
+        PUGI__FN size_t xpath_node_set::size() const
         {
                 return _end - _begin;
         }
-                
-        bool xpath_node_set::empty() const
+
+        PUGI__FN bool xpath_node_set::empty() const
         {
                 return _begin == _end;
         }
-                
-        const xpath_node& xpath_node_set::operator[](size_t index) const
+
+        PUGI__FN const xpath_node& xpath_node_set::operator[](size_t index) const
         {
                 assert(index < size());
@@ -9163 +12022 @@
         }
 
-        xpath_node_set::const_iterator xpath_node_set::begin() const
+        PUGI__FN xpath_node_set::const_iterator xpath_node_set::begin() const
         {
                 return _begin;
         }
-                
-        xpath_node_set::const_iterator xpath_node_set::end() const
+
+        PUGI__FN xpath_node_set::const_iterator xpath_node_set::end() const
         {
                 return _end;
         }
-        
-        void xpath_node_set::sort(bool reverse)
-        {
-                _type = xpath_sort(_begin, _end, _type, reverse);
-        }
-
-        xpath_node xpath_node_set::first() const
-        {
-                return xpath_first(_begin, _end, _type);
-        }
-
-    xpath_parse_result::xpath_parse_result(): error("Internal error"), offset(0)
-    {
-    }
-
-    xpath_parse_result::operator bool() const
-    {
-        return error == 0;
-    }
-        const char* xpath_parse_result::description() const
+
+        PUGI__FN void xpath_node_set::sort(bool reverse)
+        {
+                _type = impl::xpath_sort(_begin, _end, _type, reverse);
+        }
+
+        PUGI__FN xpath_node xpath_node_set::first() const
+        {
+                return impl::xpath_first(_begin, _end, _type);
+        }
+
+        PUGI__FN xpath_parse_result::xpath_parse_result(): error("Internal error"), offset(0)
+        {
+        }
+
+        PUGI__FN xpath_parse_result::operator bool() const
+        {
+                return error == 0;
+        }
+
+        PUGI__FN const char* xpath_parse_result::description() const
         {
                 return error ? error : "No error";
         }
 
-        xpath_variable::xpath_variable()
-    {
-    }
-
-        const char_t* xpath_variable::name() const
+        PUGI__FN xpath_variable::xpath_variable(xpath_value_type type_): _type(type_), _next(0)
+        {
+        }
+
+        PUGI__FN const char_t* xpath_variable::name() const
         {
                 switch (_type)
                 {
                 case xpath_type_node_set:
-                        return static_cast<const xpath_variable_node_set*>(this)->name;
+                        return static_cast<const impl::xpath_variable_node_set*>(this)->name;
 
                 case xpath_type_number:
-                        return static_cast<const xpath_variable_number*>(this)->name;
+                        return static_cast<const impl::xpath_variable_number*>(this)->name;
 
                 case xpath_type_string:
-                        return static_cast<const xpath_variable_string*>(this)->name;
+                        return static_cast<const impl::xpath_variable_string*>(this)->name;
 
                 case xpath_type_boolean:
-                        return static_cast<const xpath_variable_boolean*>(this)->name;
+                        return static_cast<const impl::xpath_variable_boolean*>(this)->name;
 
                 default:
-                        assert(!"Invalid variable type");
+                        assert(false && "Invalid variable type");
                         return 0;
                 }
         }
 
-        xpath_value_type xpath_variable::type() const
+        PUGI__FN xpath_value_type xpath_variable::type() const
         {
                 return _type;
         }
 
-        bool xpath_variable::get_boolean() const
-        {
-                return (_type == xpath_type_boolean) ? static_cast<const xpath_variable_boolean*>(this)->value : false;
-        }
-
-        double xpath_variable::get_number() const
-        {
-                return (_type == xpath_type_number) ? static_cast<const xpath_variable_number*>(this)->value : gen_nan();
-        }
-
-        const char_t* xpath_variable::get_string() const
-        {
-                const char_t* value = (_type == xpath_type_string) ? static_cast<const xpath_variable_string*>(this)->value : 0;
+        PUGI__FN bool xpath_variable::get_boolean() const
+        {
+                return (_type == xpath_type_boolean) ? static_cast<const impl::xpath_variable_boolean*>(this)->value : false;
+        }
+
+        PUGI__FN double xpath_variable::get_number() const
+        {
+                return (_type == xpath_type_number) ? static_cast<const impl::xpath_variable_number*>(this)->value : impl::gen_nan();
+        }
+
+        PUGI__FN const char_t* xpath_variable::get_string() const
+        {
+                const char_t* value = (_type == xpath_type_string) ? static_cast<const impl::xpath_variable_string*>(this)->value : 0;
                 return value ? value : PUGIXML_TEXT("");
         }
 
-        const xpath_node_set& xpath_variable::get_node_set() const
-        {
-                return (_type == xpath_type_node_set) ? static_cast<const xpath_variable_node_set*>(this)->value : dummy_node_set;
-        }
-
-        bool xpath_variable::set(bool value)
+        PUGI__FN const xpath_node_set& xpath_variable::get_node_set() const
+        {
+                return (_type == xpath_type_node_set) ? static_cast<const impl::xpath_variable_node_set*>(this)->value : impl::dummy_node_set;
+        }
+
+        PUGI__FN bool xpath_variable::set(bool value)
         {
                 if (_type != xpath_type_boolean) return false;
 
-                static_cast<xpath_variable_boolean*>(this)->value = value;
+                static_cast<impl::xpath_variable_boolean*>(this)->value = value;
                 return true;
         }
 
-        bool xpath_variable::set(double value)
+        PUGI__FN bool xpath_variable::set(double value)
         {
                 if (_type != xpath_type_number) return false;
 
-                static_cast<xpath_variable_number*>(this)->value = value;
+                static_cast<impl::xpath_variable_number*>(this)->value = value;
                 return true;
         }
 
-        bool xpath_variable::set(const char_t* value)
+        PUGI__FN bool xpath_variable::set(const char_t* value)
         {
                 if (_type != xpath_type_string) return false;
 
-                xpath_variable_string* var = static_cast<xpath_variable_string*>(this);
+                impl::xpath_variable_string* var = static_cast<impl::xpath_variable_string*>(this);
 
                 // duplicate string
-                size_t size = (strlength(value) + 1) * sizeof(char_t);
-
-                char_t* copy = static_cast<char_t*>(global_allocate(size));
+                size_t size = (impl::strlength(value) + 1) * sizeof(char_t);
+
+                char_t* copy = static_cast<char_t*>(impl::xml_memory::allocate(size));
                 if (!copy) return false;
 
@@ -9279 +12139 @@
 
                 // replace old string
-                if (var->value) global_deallocate(var->value);
+                if (var->value) impl::xml_memory::deallocate(var->value);
                 var->value = copy;
 
@@ -9285 +12145 @@
         }
 
-        bool xpath_variable::set(const xpath_node_set& value)
+        PUGI__FN bool xpath_variable::set(const xpath_node_set& value)
         {
                 if (_type != xpath_type_node_set) return false;
 
-                static_cast<xpath_variable_node_set*>(this)->value = value;
+                static_cast<impl::xpath_variable_node_set*>(this)->value = value;
                 return true;
         }
 
-        xpath_variable_set::xpath_variable_set()
-        {
-                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i) _data[i] = 0;
-        }
-
-        xpath_variable_set::~xpath_variable_set()
+        PUGI__FN xpath_variable_set::xpath_variable_set()
         {
                 for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
-                {
-                        xpath_variable* var = _data[i];
-
-                        while (var)
-                        {
-                                xpath_variable* next = var->_next;
-
-                                delete_xpath_variable(var->_type, var);
-
-                                var = next;
-                        }
-                }
-        }
-
-        xpath_variable* xpath_variable_set::find(const char_t* name) const
+                        _data[i] = 0;
+        }
+
+        PUGI__FN xpath_variable_set::~xpath_variable_set()
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                        _destroy(_data[i]);
+        }
+
+        PUGI__FN xpath_variable_set::xpath_variable_set(const xpath_variable_set& rhs)
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                        _data[i] = 0;
+
+                _assign(rhs);
+        }
+
+        PUGI__FN xpath_variable_set& xpath_variable_set::operator=(const xpath_variable_set& rhs)
+        {
+                if (this == &rhs) return *this;
+
+                _assign(rhs);
+
+                return *this;
+        }
+
+#ifdef PUGIXML_HAS_MOVE
+        PUGI__FN xpath_variable_set::xpath_variable_set(xpath_variable_set&& rhs)
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                {
+                        _data[i] = rhs._data[i];
+                        rhs._data[i] = 0;
+                }
+        }
+
+        PUGI__FN xpath_variable_set& xpath_variable_set::operator=(xpath_variable_set&& rhs)
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                {
+                        _destroy(_data[i]);
+
+                        _data[i] = rhs._data[i];
+                        rhs._data[i] = 0;
+                }
+
+                return *this;
+        }
+#endif
+
+        PUGI__FN void xpath_variable_set::_assign(const xpath_variable_set& rhs)
+        {
+                xpath_variable_set temp;
+
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                        if (rhs._data[i] && !_clone(rhs._data[i], &temp._data[i]))
+                                return;
+
+                _swap(temp);
+        }
+
+        PUGI__FN void xpath_variable_set::_swap(xpath_variable_set& rhs)
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                {
+                        xpath_variable* chain = _data[i];
+
+                        _data[i] = rhs._data[i];
+                        rhs._data[i] = chain;
+                }
+        }
+
+        PUGI__FN xpath_variable* xpath_variable_set::_find(const char_t* name) const
         {
                 const size_t hash_size = sizeof(_data) / sizeof(_data[0]);
-                size_t hash = hash_string(name) % hash_size;
+                size_t hash = impl::hash_string(name) % hash_size;
 
                 // look for existing variable
                 for (xpath_variable* var = _data[hash]; var; var = var->_next)
-                        if (strequal(var->name(), name))
+                        if (impl::strequal(var->name(), name))
                                 return var;
 
@@ -9328 +12241 @@
         }
 
-        xpath_variable* xpath_variable_set::add(const char_t* name, xpath_value_type type)
+        PUGI__FN bool xpath_variable_set::_clone(xpath_variable* var, xpath_variable** out_result)
+        {
+                xpath_variable* last = 0;
+
+                while (var)
+                {
+                        // allocate storage for new variable
+                        xpath_variable* nvar = impl::new_xpath_variable(var->_type, var->name());
+                        if (!nvar) return false;
+
+                        // link the variable to the result immediately to handle failures gracefully
+                        if (last)
+                                last->_next = nvar;
+                        else
+                                *out_result = nvar;
+
+                        last = nvar;
+
+                        // copy the value; this can fail due to out-of-memory conditions
+                        if (!impl::copy_xpath_variable(nvar, var)) return false;
+
+                        var = var->_next;
+                }
+
+                return true;
+        }
+
+        PUGI__FN void xpath_variable_set::_destroy(xpath_variable* var)
+        {
+                while (var)
+                {
+                        xpath_variable* next = var->_next;
+
+                        impl::delete_xpath_variable(var->_type, var);
+
+                        var = next;
+                }
+        }
+
+        PUGI__FN xpath_variable* xpath_variable_set::add(const char_t* name, xpath_value_type type)
         {
                 const size_t hash_size = sizeof(_data) / sizeof(_data[0]);
-                size_t hash = hash_string(name) % hash_size;
+                size_t hash = impl::hash_string(name) % hash_size;
 
                 // look for existing variable
                 for (xpath_variable* var = _data[hash]; var; var = var->_next)
-                        if (strequal(var->name(), name))
+                        if (impl::strequal(var->name(), name))
                                 return var->type() == type ? var : 0;
 
                 // add new variable
-                xpath_variable* result = new_xpath_variable(type, name);
+                xpath_variable* result = impl::new_xpath_variable(type, name);
 
                 if (result)
                 {
-                        result->_type = type;
                         result->_next = _data[hash];
 
@@ -9352 +12303 @@
         }
 
-        bool xpath_variable_set::set(const char_t* name, bool value)
+        PUGI__FN bool xpath_variable_set::set(const char_t* name, bool value)
         {
                 xpath_variable* var = add(name, xpath_type_boolean);
@@ -9358 +12309 @@
         }
 
-        bool xpath_variable_set::set(const char_t* name, double value)
+        PUGI__FN bool xpath_variable_set::set(const char_t* name, double value)
         {
                 xpath_variable* var = add(name, xpath_type_number);
@@ -9364 +12315 @@
         }
 
-        bool xpath_variable_set::set(const char_t* name, const char_t* value)
+        PUGI__FN bool xpath_variable_set::set(const char_t* name, const char_t* value)
         {
                 xpath_variable* var = add(name, xpath_type_string);
@@ -9370 +12321 @@
         }
 
-        bool xpath_variable_set::set(const char_t* name, const xpath_node_set& value)
+        PUGI__FN bool xpath_variable_set::set(const char_t* name, const xpath_node_set& value)
         {
                 xpath_variable* var = add(name, xpath_type_node_set);
@@ -9376 +12327 @@
         }
 
-        xpath_variable* xpath_variable_set::get(const char_t* name)
-        {
-                return find(name);
-        }
-
-        const xpath_variable* xpath_variable_set::get(const char_t* name) const
-        {
-                return find(name);
-        }
-
-        xpath_query::xpath_query(const char_t* query, xpath_variable_set* variables): _impl(0)
-        {
-                xpath_query_impl* impl = xpath_query_impl::create();
-
-                if (!impl)
+        PUGI__FN xpath_variable* xpath_variable_set::get(const char_t* name)
+        {
+                return _find(name);
+        }
+
+        PUGI__FN const xpath_variable* xpath_variable_set::get(const char_t* name) const
+        {
+                return _find(name);
+        }
+
+        PUGI__FN xpath_query::xpath_query(const char_t* query, xpath_variable_set* variables): _impl(0)
+        {
+                impl::xpath_query_impl* qimpl = impl::xpath_query_impl::create();
+
+                if (!qimpl)
                 {
                 #ifdef PUGIXML_NO_EXCEPTIONS
                         _result.error = "Out of memory";
-        #else
+                #else
                         throw std::bad_alloc();
                 #endif
@@ -9400 +12351 @@
                 else
                 {
-                        buffer_holder impl_holder(impl, xpath_query_impl::destroy);
-
-                        impl->root = xpath_parser::parse(query, variables, &impl->alloc, &_result);
-
-                        if (impl->root)
-                        {
-                _impl = static_cast<xpath_query_impl*>(impl_holder.release());
+                        using impl::auto_deleter; // MSVC7 workaround
+                        auto_deleter<impl::xpath_query_impl> impl(qimpl, impl::xpath_query_impl::destroy);
+
+                        qimpl->root = impl::xpath_parser::parse(query, variables, &qimpl->alloc, &_result);
+
+                        if (qimpl->root)
+                        {
+                                qimpl->root->optimize(&qimpl->alloc);
+
+                                _impl = impl.release();
                                 _result.error = 0;
                         }
@@ -9412 +12366 @@
         }
 
-        xpath_query::~xpath_query()
-        {
-                xpath_query_impl::destroy(_impl);
-        }
-
-        xpath_value_type xpath_query::return_type() const
+        PUGI__FN xpath_query::xpath_query(): _impl(0)
+        {
+        }
+
+        PUGI__FN xpath_query::~xpath_query()
+        {
+                if (_impl)
+                        impl::xpath_query_impl::destroy(static_cast<impl::xpath_query_impl*>(_impl));
+        }
+
+#ifdef PUGIXML_HAS_MOVE
+        PUGI__FN xpath_query::xpath_query(xpath_query&& rhs)
+        {
+                _impl = rhs._impl;
+                _result = rhs._result;
+                rhs._impl = 0;
+                rhs._result = xpath_parse_result();
+        }
+
+        PUGI__FN xpath_query& xpath_query::operator=(xpath_query&& rhs)
+        {
+                if (this == &rhs) return *this;
+
+                if (_impl)
+                        impl::xpath_query_impl::destroy(static_cast<impl::xpath_query_impl*>(_impl));
+
+                _impl = rhs._impl;
+                _result = rhs._result;
+                rhs._impl = 0;
+                rhs._result = xpath_parse_result();
+
+                return *this;
+        }
+#endif
+
+        PUGI__FN xpath_value_type xpath_query::return_type() const
         {
                 if (!_impl) return xpath_type_none;
 
-                return static_cast<xpath_query_impl*>(_impl)->root->rettype();
-        }
-
-        bool xpath_query::evaluate_boolean(const xpath_node& n) const
+                return static_cast<impl::xpath_query_impl*>(_impl)->root->rettype();
+        }
+
+        PUGI__FN bool xpath_query::evaluate_boolean(const xpath_node& n) const
         {
                 if (!_impl) return false;
-                
-                xpath_context c(n, 1, 1);
-                xpath_stack_data sd;
+
+                impl::xpath_context c(n, 1, 1);
+                impl::xpath_stack_data sd;
 
         #ifdef PUGIXML_NO_EXCEPTIONS
                 if (setjmp(sd.error_handler)) return false;
         #endif
-                
-                return static_cast<xpath_query_impl*>(_impl)->root->eval_boolean(c, sd.stack);
-        }
-        
-        double xpath_query::evaluate_number(const xpath_node& n) const
-        {
-                if (!_impl) return gen_nan();
-                
-                xpath_context c(n, 1, 1);
-                xpath_stack_data sd;
+
+                return static_cast<impl::xpath_query_impl*>(_impl)->root->eval_boolean(c, sd.stack);
+        }
+
+        PUGI__FN double xpath_query::evaluate_number(const xpath_node& n) const
+        {
+                if (!_impl) return impl::gen_nan();
+
+                impl::xpath_context c(n, 1, 1);
+                impl::xpath_stack_data sd;
 
         #ifdef PUGIXML_NO_EXCEPTIONS
-                if (setjmp(sd.error_handler)) return gen_nan();
+                if (setjmp(sd.error_handler)) return impl::gen_nan();
         #endif
 
-                return static_cast<xpath_query_impl*>(_impl)->root->eval_number(c, sd.stack);
+                return static_cast<impl::xpath_query_impl*>(_impl)->root->eval_number(c, sd.stack);
         }
 
 #ifndef PUGIXML_NO_STL
-        string_t xpath_query::evaluate_string(const xpath_node& n) const
-        {
-                xpath_stack_data sd;
-
-                return evaluate_string_impl(static_cast<xpath_query_impl*>(_impl), n, sd).c_str();
+        PUGI__FN string_t xpath_query::evaluate_string(const xpath_node& n) const
+        {
+                impl::xpath_stack_data sd;
+
+                impl::xpath_string r = impl::evaluate_string_impl(static_cast<impl::xpath_query_impl*>(_impl), n, sd);
+
+                return string_t(r.c_str(), r.length());
         }
 #endif
 
-        size_t xpath_query::evaluate_string(char_t* buffer, size_t capacity, const xpath_node& n) const
-        {
-                xpath_stack_data sd;
-
-                xpath_string r = evaluate_string_impl(static_cast<xpath_query_impl*>(_impl), n, sd);
+        PUGI__FN size_t xpath_query::evaluate_string(char_t* buffer, size_t capacity, const xpath_node& n) const
+        {
+                impl::xpath_stack_data sd;
+
+                impl::xpath_string r = impl::evaluate_string_impl(static_cast<impl::xpath_query_impl*>(_impl), n, sd);
 
                 size_t full_size = r.length() + 1;
-                
+
                 if (capacity > 0)
-        {
-            size_t size = (full_size < capacity) ? full_size : capacity;
-            assert(size > 0);
-
-            memcpy(buffer, r.c_str(), (size - 1) * sizeof(char_t));
-            buffer[size - 1] = 0;
-        }
-                
+                {
+                        size_t size = (full_size < capacity) ? full_size : capacity;
+                        assert(size > 0);
+
+                        memcpy(buffer, r.c_str(), (size - 1) * sizeof(char_t));
+                        buffer[size - 1] = 0;
+                }
+
                 return full_size;
         }
 
-        xpath_node_set xpath_query::evaluate_node_set(const xpath_node& n) const
-        {
-                if (!_impl) return xpath_node_set();
-
-        xpath_ast_node* root = static_cast<xpath_query_impl*>(_impl)->root;
-
-                if (root->rettype() != xpath_type_node_set)
-                {
-                #ifdef PUGIXML_NO_EXCEPTIONS
-                        return xpath_node_set();
-                #else
-                        xpath_parse_result result;
-                        result.error = "Expression does not evaluate to node set";
-
-                        throw xpath_exception(result);
-                #endif
-                }
-                
-                xpath_context c(n, 1, 1);
-                xpath_stack_data sd;
+        PUGI__FN xpath_node_set xpath_query::evaluate_node_set(const xpath_node& n) const
+        {
+                impl::xpath_ast_node* root = impl::evaluate_node_set_prepare(static_cast<impl::xpath_query_impl*>(_impl));
+                if (!root) return xpath_node_set();
+
+                impl::xpath_context c(n, 1, 1);
+                impl::xpath_stack_data sd;
 
         #ifdef PUGIXML_NO_EXCEPTIONS
@@ -9506 +12479 @@
         #endif
 
-                xpath_node_set_raw r = root->eval_node_set(c, sd.stack);
+                impl::xpath_node_set_raw r = root->eval_node_set(c, sd.stack, impl::nodeset_eval_all);
 
                 return xpath_node_set(r.begin(), r.end(), r.type());
         }
 
-        const xpath_parse_result& xpath_query::result() const
+        PUGI__FN xpath_node xpath_query::evaluate_node(const xpath_node& n) const
+        {
+                impl::xpath_ast_node* root = impl::evaluate_node_set_prepare(static_cast<impl::xpath_query_impl*>(_impl));
+                if (!root) return xpath_node();
+
+                impl::xpath_context c(n, 1, 1);
+                impl::xpath_stack_data sd;
+
+        #ifdef PUGIXML_NO_EXCEPTIONS
+                if (setjmp(sd.error_handler)) return xpath_node();
+        #endif
+
+                impl::xpath_node_set_raw r = root->eval_node_set(c, sd.stack, impl::nodeset_eval_first);
+
+                return r.first();
+        }
+
+        PUGI__FN const xpath_parse_result& xpath_query::result() const
         {
                 return _result;
         }
 
-        xpath_query::operator xpath_query::unspecified_bool_type() const
-        {
-                return _impl ? &xpath_query::_impl : 0;
-        }
-
-        bool xpath_query::operator!() const
+        PUGI__FN static void unspecified_bool_xpath_query(xpath_query***)
+        {
+        }
+
+        PUGI__FN xpath_query::operator xpath_query::unspecified_bool_type() const
+        {
+                return _impl ? unspecified_bool_xpath_query : 0;
+        }
+
+        PUGI__FN bool xpath_query::operator!() const
         {
                 return !_impl;
         }
 
-        xpath_node xml_node::select_single_node(const char_t* query, xpath_variable_set* variables) const
+        PUGI__FN xpath_node xml_node::select_node(const char_t* query, xpath_variable_set* variables) const
+        {
+                xpath_query q(query, variables);
+                return select_node(q);
+        }
+
+        PUGI__FN xpath_node xml_node::select_node(const xpath_query& query) const
+        {
+                return query.evaluate_node(*this);
+        }
+
+        PUGI__FN xpath_node_set xml_node::select_nodes(const char_t* query, xpath_variable_set* variables) const
+        {
+                xpath_query q(query, variables);
+                return select_nodes(q);
+        }
+
+        PUGI__FN xpath_node_set xml_node::select_nodes(const xpath_query& query) const
+        {
+                return query.evaluate_node_set(*this);
+        }
+
+        PUGI__FN xpath_node xml_node::select_single_node(const char_t* query, xpath_variable_set* variables) const
         {
                 xpath_query q(query, variables);
@@ -9532 +12548 @@
         }
 
-        xpath_node xml_node::select_single_node(const xpath_query& query) const
-        {
-                xpath_node_set s = query.evaluate_node_set(*this);
-                return s.empty() ? xpath_node() : s.first();
-        }
-
-        xpath_node_set xml_node::select_nodes(const char_t* query, xpath_variable_set* variables) const
-        {
-                xpath_query q(query, variables);
-                return select_nodes(q);
-        }
-
-        xpath_node_set xml_node::select_nodes(const xpath_query& query) const
-        {
-                return query.evaluate_node_set(*this);
+        PUGI__FN xpath_node xml_node::select_single_node(const xpath_query& query) const
+        {
+                return query.evaluate_node(*this);
         }
 }
@@ -9552 +12556 @@
 #endif
 
+#ifdef __BORLANDC__
+#       pragma option pop
+#endif
+
+// Intel C++ does not properly keep warning state for function templates,
+// so popping warning state at the end of translation unit leads to warnings in the middle.
+#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
+#       pragma warning(pop)
+#endif
+
+// Undefine all local macros (makes sure we're not leaking macros in header-only mode)
+#undef PUGI__NO_INLINE
+#undef PUGI__UNLIKELY
+#undef PUGI__STATIC_ASSERT
+#undef PUGI__DMC_VOLATILE
+#undef PUGI__MSVC_CRT_VERSION
+#undef PUGI__NS_BEGIN
+#undef PUGI__NS_END
+#undef PUGI__FN
+#undef PUGI__FN_NO_INLINE
+#undef PUGI__GETHEADER_IMPL
+#undef PUGI__GETPAGE_IMPL
+#undef PUGI__GETPAGE
+#undef PUGI__NODETYPE
+#undef PUGI__IS_CHARTYPE_IMPL
+#undef PUGI__IS_CHARTYPE
+#undef PUGI__IS_CHARTYPEX
+#undef PUGI__ENDSWITH
+#undef PUGI__SKIPWS
+#undef PUGI__OPTSET
+#undef PUGI__PUSHNODE
+#undef PUGI__POPNODE
+#undef PUGI__SCANFOR
+#undef PUGI__SCANWHILE
+#undef PUGI__SCANWHILE_UNROLL
+#undef PUGI__ENDSEG
+#undef PUGI__THROW_ERROR
+#undef PUGI__CHECK_ERROR
+
+#endif
+
 /**
- * Copyright (c) 2006-2010 Arseny Kapoulkine
+ * Copyright (c) 2006-2016 Arseny Kapoulkine
  *
  * Permission is hereby granted, free of charge, to any person
@@ -9566 +12611 @@
  * The above copyright notice and this permission notice shall be
  * included in all copies or substantial portions of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES

ThirdParty/vmg/src/thirdparty/pugixml/pugixml.hpp

@@ -1 +1 @@
 /**
- * pugixml parser - version 1.0
+ * pugixml parser - version 1.8
  * --------------------------------------------------------
- * Copyright (C) 2006-2010, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
+ * Copyright (C) 2006-2016, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
  * Report bugs and download new versions at http://pugixml.org/
  *
@@ -12 +12 @@
  */
 
+#ifndef PUGIXML_VERSION
+// Define version macro; evaluates to major * 100 + minor so that it's safe to use in less-than comparisons
+#       define PUGIXML_VERSION 180
+#endif
+
+// Include user configuration file (this can define various configuration macros)
+#include "pugiconfig.hpp"
+
 #ifndef HEADER_PUGIXML_HPP
 #define HEADER_PUGIXML_HPP
 
-#include "pugiconfig.hpp"
-
+// Include stddef.h for size_t and ptrdiff_t
+#include <stddef.h>
+
+// Include exception header for XPath
+#if !defined(PUGIXML_NO_XPATH) && !defined(PUGIXML_NO_EXCEPTIONS)
+#       include <exception>
+#endif
+
+// Include STL headers
 #ifndef PUGIXML_NO_STL
-namespace std
-{
-        struct bidirectional_iterator_tag;
-
-#ifdef __SUNPRO_CC
-        // Sun C++ compiler has a bug which forces template argument names in forward declarations to be the same as in actual definitions
-        template <class _T> class allocator;
-        template <class _charT> struct char_traits;
-        template <class _charT, class _Traits> class basic_istream;
-        template <class _charT, class _Traits> class basic_ostream;
-        template <class _charT, class _Traits, class _Allocator> class basic_string;
-#else
-        // Borland C++ compiler has a bug which forces template argument names in forward declarations to be the same as in actual definitions
-        template <class _Ty> class allocator;
-        template <class _Ty> struct char_traits;
-        template <class _Elem, class _Traits> class basic_istream;
-        template <class _Elem, class _Traits> class basic_ostream;
-        template <class _Elem, class _Traits, class _Ax> class basic_string;
-#endif
-
-        // Digital Mars compiler has a bug which requires a forward declaration for explicit instantiation (otherwise type selection is messed up later, producing link errors)
-        // Also note that we have to declare char_traits as a class here, since it's defined that way
-#ifdef __DMC__
-        template <> class char_traits<char>;
-#endif
-}
+#       include <iterator>
+#       include <iosfwd>
+#       include <string>
 #endif
 
@@ -57 +49 @@
 #endif
 
-// Include exception header for XPath
-#if !defined(PUGIXML_NO_XPATH) && !defined(PUGIXML_NO_EXCEPTIONS)
-#       include <exception>
-#endif
-
 // If no API is defined, assume default
 #ifndef PUGIXML_API
-#   define PUGIXML_API
+#       define PUGIXML_API
 #endif
 
 // If no API for classes is defined, assume default
 #ifndef PUGIXML_CLASS
-#   define PUGIXML_CLASS PUGIXML_API
+#       define PUGIXML_CLASS PUGIXML_API
 #endif
 
 // If no API for functions is defined, assume default
 #ifndef PUGIXML_FUNCTION
-#   define PUGIXML_FUNCTION PUGIXML_API
-#endif
-
-#include <stddef.h>
+#       define PUGIXML_FUNCTION PUGIXML_API
+#endif
+
+// If the platform is known to have long long support, enable long long functions
+#ifndef PUGIXML_HAS_LONG_LONG
+#       if __cplusplus >= 201103
+#               define PUGIXML_HAS_LONG_LONG
+#       elif defined(_MSC_VER) && _MSC_VER >= 1400
+#               define PUGIXML_HAS_LONG_LONG
+#       endif
+#endif
+
+// If the platform is known to have move semantics support, compile move ctor/operator implementation
+#ifndef PUGIXML_HAS_MOVE
+#       if __cplusplus >= 201103
+#               define PUGIXML_HAS_MOVE
+#       elif defined(_MSC_VER) && _MSC_VER >= 1600
+#               define PUGIXML_HAS_MOVE
+#       endif
+#endif
+
+// If C++ is 2011 or higher, add 'override' qualifiers
+#ifndef PUGIXML_OVERRIDE
+#       if __cplusplus >= 201103
+#               define PUGIXML_OVERRIDE override
+#       else
+#               define PUGIXML_OVERRIDE
+#       endif
+#endif
 
 // Character interface macros
@@ -105 +117 @@
         enum xml_node_type
         {
-                node_null,          // Empty (null) node handle
+                node_null,                      // Empty (null) node handle
                 node_document,          // A document tree's absolute root
                 node_element,           // Element tag, i.e. '<node/>'
@@ -113 +125 @@
                 node_pi,                        // Processing instruction, i.e. '<?name?>'
                 node_declaration,       // Document declaration, i.e. '<?xml version="1.0"?>'
-        node_doctype        // Document type declaration, i.e. '<!DOCTYPE doc>'
+                node_doctype            // Document type declaration, i.e. '<!DOCTYPE doc>'
         };
 
@@ -119 +131 @@
 
         // Minimal parsing mode (equivalent to turning all other flags off).
-    // Only elements and PCDATA sections are added to the DOM tree, no text conversions are performed.
+        // Only elements and PCDATA sections are added to the DOM tree, no text conversions are performed.
         const unsigned int parse_minimal = 0x0000;
 
@@ -132 +144 @@
 
         // This flag determines if plain character data (node_pcdata) that consist only of whitespace are added to the DOM tree.
-    // This flag is off by default; turning it on usually results in slower parsing and more memory consumption.
+        // This flag is off by default; turning it on usually results in slower parsing and more memory consumption.
         const unsigned int parse_ws_pcdata = 0x0008;
 
@@ -140 +152 @@
         // This flag determines if EOL characters are normalized (converted to #xA) during parsing. This flag is on by default.
         const unsigned int parse_eol = 0x0020;
-        
-        // This flag determines if attribute values are normalized using CDATA normalization rules during parsing. This flag is on by default.
-        const unsigned int parse_wconv_attribute = 0x0040;
-
-        // This flag determines if attribute values are normalized using NMTOKENS normalization rules during parsing. This flag is off by default.
-        const unsigned int parse_wnorm_attribute = 0x0080;
-        
-    // This flag determines if document declaration (node_declaration) is added to the DOM tree. This flag is off by default.
+
+        // This flag determines if attribute values are normalized using CDATA normalization rules during parsing. This flag is on by default.
+        const unsigned int parse_wconv_attribute = 0x0040;
+
+        // This flag determines if attribute values are normalized using NMTOKENS normalization rules during parsing. This flag is off by default.
+        const unsigned int parse_wnorm_attribute = 0x0080;
+
+        // This flag determines if document declaration (node_declaration) is added to the DOM tree. This flag is off by default.
         const unsigned int parse_declaration = 0x0100;
 
-    // This flag determines if document type declaration (node_doctype) is added to the DOM tree. This flag is off by default.
+        // This flag determines if document type declaration (node_doctype) is added to the DOM tree. This flag is off by default.
         const unsigned int parse_doctype = 0x0200;
 
+        // This flag determines if plain character data (node_pcdata) that is the only child of the parent node and that consists only
+        // of whitespace is added to the DOM tree.
+        // This flag is off by default; turning it on may result in slower parsing and more memory consumption.
+        const unsigned int parse_ws_pcdata_single = 0x0400;
+
+        // This flag determines if leading and trailing whitespace is to be removed from plain character data. This flag is off by default.
+        const unsigned int parse_trim_pcdata = 0x0800;
+
+        // This flag determines if plain character data that does not have a parent node is added to the DOM tree, and if an empty document
+        // is a valid document. This flag is off by default.
+        const unsigned int parse_fragment = 0x1000;
+
+        // This flag determines if plain character data is be stored in the parent element's value. This significantly changes the structure of
+        // the document; this flag is only recommended for parsing documents with many PCDATA nodes in memory-constrained environments.
+        // This flag is off by default.
+        const unsigned int parse_embed_pcdata = 0x2000;
+
         // The default parsing mode.
-    // Elements, PCDATA and CDATA sections are added to the DOM tree, character/reference entities are expanded,
-    // End-of-Line characters are normalized, attribute values are normalized using CDATA normalization rules.
+        // Elements, PCDATA and CDATA sections are added to the DOM tree, character/reference entities are expanded,
+        // End-of-Line characters are normalized, attribute values are normalized using CDATA normalization rules.
         const unsigned int parse_default = parse_cdata | parse_escapes | parse_wconv_attribute | parse_eol;
 
-    // The full parsing mode.
-    // Nodes of all types are added to the DOM tree, character/reference entities are expanded,
-    // End-of-Line characters are normalized, attribute values are normalized using CDATA normalization rules.
-    const unsigned int parse_full = parse_default | parse_pi | parse_comments | parse_declaration | parse_doctype;
+        // The full parsing mode.
+        // Nodes of all types are added to the DOM tree, character/reference entities are expanded,
+        // End-of-Line characters are normalized, attribute values are normalized using CDATA normalization rules.
+        const unsigned int parse_full = parse_default | parse_pi | parse_comments | parse_declaration | parse_doctype;
 
         // These flags determine the encoding of input data for XML document
         enum xml_encoding
         {
-                encoding_auto,      // Auto-detect input encoding using BOM or < / <? detection; use UTF8 if BOM is not found
-                encoding_utf8,      // UTF8 encoding
-                encoding_utf16_le,  // Little-endian UTF16
-                encoding_utf16_be,  // Big-endian UTF16
-                encoding_utf16,     // UTF16 with native endianness
-                encoding_utf32_le,  // Little-endian UTF32
-                encoding_utf32_be,  // Big-endian UTF32
-                encoding_utf32,     // UTF32 with native endianness
-                encoding_wchar      // The same encoding wchar_t has (either UTF16 or UTF32)
+                encoding_auto,          // Auto-detect input encoding using BOM or < / <? detection; use UTF8 if BOM is not found
+                encoding_utf8,          // UTF8 encoding
+                encoding_utf16_le,      // Little-endian UTF16
+                encoding_utf16_be,      // Big-endian UTF16
+                encoding_utf16,         // UTF16 with native endianness
+                encoding_utf32_le,      // Little-endian UTF32
+                encoding_utf32_be,      // Big-endian UTF32
+                encoding_utf32,         // UTF32 with native endianness
+                encoding_wchar,         // The same encoding wchar_t has (either UTF16 or UTF32)
+                encoding_latin1
         };
 
         // Formatting flags
-        
+
         // Indent the nodes that are written to output stream with as many indentation strings as deep the node is in DOM tree. This flag is on by default.
         const unsigned int format_indent = 0x01;
-        
+
         // Write encoding-specific BOM to the output stream. This flag is off by default.
         const unsigned int format_write_bom = 0x02;
@@ -187 +217 @@
         // Use raw output mode (no indentation and no line breaks are written). This flag is off by default.
         const unsigned int format_raw = 0x04;
-        
+
         // Omit default XML declaration even if there is no declaration in the document. This flag is off by default.
         const unsigned int format_no_declaration = 0x08;
 
+        // Don't escape attribute values and PCDATA contents. This flag is off by default.
+        const unsigned int format_no_escapes = 0x10;
+
+        // Open file using text mode in xml_document::save_file. This enables special character (i.e. new-line) conversions on some systems. This flag is off by default.
+        const unsigned int format_save_file_text = 0x20;
+
+        // Write every attribute on a new line with appropriate indentation. This flag is off by default.
+        const unsigned int format_indent_attributes = 0x40;
+
+        // Don't output empty element tags, instead writing an explicit start and end tag even if there are no children. This flag is off by default.
+        const unsigned int format_no_empty_element_tags = 0x80;
+
         // The default set of formatting flags.
-    // Nodes are indented depending on their depth in DOM tree, a default declaration is output if document has none.
+        // Nodes are indented depending on their depth in DOM tree, a default declaration is output if document has none.
         const unsigned int format_default = format_indent;
-                
+
         // Forward declarations
         struct xml_attribute_struct;
@@ -201 +243 @@
         class xml_node_iterator;
         class xml_attribute_iterator;
+        class xml_named_node_iterator;
 
         class xml_tree_walker;
-        
+
+        struct xml_parse_result;
+
         class xml_node;
+
+        class xml_text;
 
         #ifndef PUGIXML_NO_XPATH
@@ -213 +260 @@
         #endif
 
+        // Range-based for loop support
+        template <typename It> class xml_object_range
+        {
+        public:
+                typedef It const_iterator;
+                typedef It iterator;
+
+                xml_object_range(It b, It e): _begin(b), _end(e)
+                {
+                }
+
+                It begin() const { return _begin; }
+                It end() const { return _end; }
+
+        private:
+                It _begin, _end;
+        };
+
         // Writer interface for node printing (see xml_node::print)
         class PUGIXML_CLASS xml_writer
@@ -227 +292 @@
         {
         public:
-        // Construct writer from a FILE* object; void* is used to avoid header dependencies on stdio
+                // Construct writer from a FILE* object; void* is used to avoid header dependencies on stdio
                 xml_writer_file(void* file);
 
-                virtual void write(const void* data, size_t size);
+                virtual void write(const void* data, size_t size) PUGIXML_OVERRIDE;
 
         private:
@@ -241 +306 @@
         {
         public:
-        // Construct writer from an output stream object
+                // Construct writer from an output stream object
                 xml_writer_stream(std::basic_ostream<char, std::char_traits<char> >& stream);
                 xml_writer_stream(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream);
 
-                virtual void write(const void* data, size_t size);
+                virtual void write(const void* data, size_t size) PUGIXML_OVERRIDE;
 
         private:
@@ -261 +326 @@
         private:
                 xml_attribute_struct* _attr;
-        
-        typedef xml_attribute_struct* xml_attribute::*unspecified_bool_type;
-
-        public:
-        // Default constructor. Constructs an empty attribute.
+
+                typedef void (*unspecified_bool_type)(xml_attribute***);
+
+        public:
+                // Default constructor. Constructs an empty attribute.
                 xml_attribute();
-                
-        // Constructs attribute from internal pointer
+
+                // Constructs attribute from internal pointer
                 explicit xml_attribute(xml_attribute_struct* attr);
 
-        // Safe bool conversion operator
-        operator unspecified_bool_type() const;
-
-        // Borland C++ workaround
-        bool operator!() const;
+                // Safe bool conversion operator
+                operator unspecified_bool_type() const;
+
+                // Borland C++ workaround
+                bool operator!() const;
 
                 // Comparison operators (compares wrapped attribute pointers)
@@ -292 +357 @@
                 const char_t* value() const;
 
-                // Get attribute value as a number, or 0 if conversion did not succeed or attribute is empty
-                int as_int() const;
-                unsigned int as_uint() const;
-                double as_double() const;
-                float as_float() const;
-
-        // Get attribute value as bool (returns true if first character is in '1tTyY' set), or false if attribute is empty
-                bool as_bool() const;
-
-        // Set attribute name/value (returns false if attribute is empty or there is not enough memory)
+                // Get attribute value, or the default value if attribute is empty
+                const char_t* as_string(const char_t* def = PUGIXML_TEXT("")) const;
+
+                // Get attribute value as a number, or the default value if conversion did not succeed or attribute is empty
+                int as_int(int def = 0) const;
+                unsigned int as_uint(unsigned int def = 0) const;
+                double as_double(double def = 0) const;
+                float as_float(float def = 0) const;
+
+        #ifdef PUGIXML_HAS_LONG_LONG
+                long long as_llong(long long def = 0) const;
+                unsigned long long as_ullong(unsigned long long def = 0) const;
+        #endif
+
+                // Get attribute value as bool (returns true if first character is in '1tTyY' set), or the default value if attribute is empty
+                bool as_bool(bool def = false) const;
+
+                // Set attribute name/value (returns false if attribute is empty or there is not enough memory)
                 bool set_name(const char_t* rhs);
                 bool set_value(const char_t* rhs);
 
-        // Set attribute value with type conversion (numbers are converted to strings, boolean is converted to "true"/"false")
+                // Set attribute value with type conversion (numbers are converted to strings, boolean is converted to "true"/"false")
                 bool set_value(int rhs);
                 bool set_value(unsigned int rhs);
+                bool set_value(long rhs);
+                bool set_value(unsigned long rhs);
                 bool set_value(double rhs);
+                bool set_value(float rhs);
                 bool set_value(bool rhs);
+
+        #ifdef PUGIXML_HAS_LONG_LONG
+                bool set_value(long long rhs);
+                bool set_value(unsigned long long rhs);
+        #endif
 
                 // Set attribute value (equivalent to set_value without error checking)
@@ -315 +396 @@
                 xml_attribute& operator=(int rhs);
                 xml_attribute& operator=(unsigned int rhs);
+                xml_attribute& operator=(long rhs);
+                xml_attribute& operator=(unsigned long rhs);
                 xml_attribute& operator=(double rhs);
+                xml_attribute& operator=(float rhs);
                 xml_attribute& operator=(bool rhs);
 
-        // Get next/previous attribute in the attribute list of the parent node
-        xml_attribute next_attribute() const;
-        xml_attribute previous_attribute() const;
-
-        // Get hash value (unique for handles to the same object)
-        size_t hash_value() const;
+        #ifdef PUGIXML_HAS_LONG_LONG
+                xml_attribute& operator=(long long rhs);
+                xml_attribute& operator=(unsigned long long rhs);
+        #endif
+
+                // Get next/previous attribute in the attribute list of the parent node
+                xml_attribute next_attribute() const;
+                xml_attribute previous_attribute() const;
+
+                // Get hash value (unique for handles to the same object)
+                size_t hash_value() const;
 
                 // Get internal pointer
@@ -340 +429 @@
                 friend class xml_attribute_iterator;
                 friend class xml_node_iterator;
+                friend class xml_named_node_iterator;
 
         protected:
                 xml_node_struct* _root;
 
-        typedef xml_node_struct* xml_node::*unspecified_bool_type;
+                typedef void (*unspecified_bool_type)(xml_node***);
 
         public:
@@ -350 +440 @@
                 xml_node();
 
-        // Constructs node from internal pointer
+                // Constructs node from internal pointer
                 explicit xml_node(xml_node_struct* p);
 
-        // Safe bool conversion operator
+                // Safe bool conversion operator
                 operator unspecified_bool_type() const;
 
                 // Borland C++ workaround
                 bool operator!() const;
-        
+
                 // Comparison operators (compares wrapped node pointers)
                 bool operator==(const xml_node& r) const;
@@ -373 +463 @@
                 xml_node_type type() const;
 
-                // Get node name/value, or "" if node is empty or it has no name/value
+                // Get node name, or "" if node is empty or it has no name
                 const char_t* name() const;
+
+                // Get node value, or "" if node is empty or it has no value
+                // Note: For <node>text</node> node.value() does not return "text"! Use child_value() or text() methods to access text inside nodes.
                 const char_t* value() const;
-        
+
                 // Get attribute list
                 xml_attribute first_attribute() const;
-        xml_attribute last_attribute() const;
-
-        // Get children list
+                xml_attribute last_attribute() const;
+
+                // Get children list
                 xml_node first_child() const;
-        xml_node last_child() const;
-
-        // Get next/previous sibling in the children list of the parent node
+                xml_node last_child() const;
+
+                // Get next/previous sibling in the children list of the parent node
                 xml_node next_sibling() const;
                 xml_node previous_sibling() const;
-                
-        // Get parent node
+
+                // Get parent node
                 xml_node parent() const;
 
                 // Get root of DOM tree this node belongs to
                 xml_node root() const;
+
+                // Get text object for the current node
+                xml_text text() const;
 
                 // Get child, attribute or next/previous sibling with the specified name
@@ -401 +497 @@
                 xml_node previous_sibling(const char_t* name) const;
 
+                // Get attribute, starting the search from a hint (and updating hint so that searching for a sequence of attributes is fast)
+                xml_attribute attribute(const char_t* name, xml_attribute& hint) const;
+
                 // Get child value of current node; that is, value of the first child node of type PCDATA/CDATA
                 const char_t* child_value() const;
@@ -410 +509 @@
                 bool set_name(const char_t* rhs);
                 bool set_value(const char_t* rhs);
-                
+
                 // Add attribute with specified name. Returns added attribute, or empty attribute on errors.
                 xml_attribute append_attribute(const char_t* name);
@@ -441 +540 @@
                 xml_node insert_copy_before(const xml_node& proto, const xml_node& node);
 
+                // Move the specified node to become a child of this node. Returns moved node, or empty node on errors.
+                xml_node append_move(const xml_node& moved);
+                xml_node prepend_move(const xml_node& moved);
+                xml_node insert_move_after(const xml_node& moved, const xml_node& node);
+                xml_node insert_move_before(const xml_node& moved, const xml_node& node);
+
                 // Remove specified attribute
                 bool remove_attribute(const xml_attribute& a);
@@ -448 +553 @@
                 bool remove_child(const xml_node& n);
                 bool remove_child(const char_t* name);
+
+                // Parses buffer as an XML document fragment and appends all nodes as children of the current node.
+                // Copies/converts the buffer, so it may be deleted or changed after the function returns.
+                // Note: append_buffer allocates memory that has the lifetime of the owning document; removing the appended nodes does not immediately reclaim that memory.
+                xml_parse_result append_buffer(const void* contents, size_t size, unsigned int options = parse_default, xml_encoding encoding = encoding_auto);
 
                 // Find attribute using predicate. Returns first attribute for which predicate returned true.
@@ -453 +563 @@
                 {
                         if (!_root) return xml_attribute();
-                        
+
                         for (xml_attribute attrib = first_attribute(); attrib; attrib = attrib.next_attribute())
                                 if (pred(attrib))
                                         return attrib;
-                
+
                         return xml_attribute();
                 }
@@ -465 +575 @@
                 {
                         if (!_root) return xml_node();
-        
+
                         for (xml_node node = first_child(); node; node = node.next_sibling())
                                 if (pred(node))
                                         return node;
-        
-                return xml_node();
+
+                        return xml_node();
                 }
 
@@ -479 +589 @@
 
                         xml_node cur = first_child();
-                        
+
                         while (cur._root && cur._root != _root)
                         {
@@ -511 +621 @@
                 // Recursively traverse subtree with xml_tree_walker
                 bool traverse(xml_tree_walker& walker);
-        
+
         #ifndef PUGIXML_NO_XPATH
                 // Select single node by evaluating XPath query. Returns first node from the resulting node set.
-                xpath_node select_single_node(const char_t* query, xpath_variable_set* variables = 0) const;
-                xpath_node select_single_node(const xpath_query& query) const;
+                xpath_node select_node(const char_t* query, xpath_variable_set* variables = 0) const;
+                xpath_node select_node(const xpath_query& query) const;
 
                 // Select node set by evaluating XPath query
                 xpath_node_set select_nodes(const char_t* query, xpath_variable_set* variables = 0) const;
                 xpath_node_set select_nodes(const xpath_query& query) const;
-        #endif
-                
+
+                // (deprecated: use select_node instead) Select single node by evaluating XPath query.
+                xpath_node select_single_node(const char_t* query, xpath_variable_set* variables = 0) const;
+                xpath_node select_single_node(const xpath_query& query) const;
+
+        #endif
+
                 // Print subtree using a writer object
                 void print(xml_writer& writer, const char_t* indent = PUGIXML_TEXT("\t"), unsigned int flags = format_default, xml_encoding encoding = encoding_auto, unsigned int depth = 0) const;
@@ -543 +658 @@
                 attribute_iterator attributes_end() const;
 
+                // Range-based for support
+                xml_object_range<xml_node_iterator> children() const;
+                xml_object_range<xml_named_node_iterator> children(const char_t* name) const;
+                xml_object_range<xml_attribute_iterator> attributes() const;
+
                 // Get node offset in parsed file/string (in char_t units) for debugging purposes
                 ptrdiff_t offset_debug() const;
 
-        // Get hash value (unique for handles to the same object)
-        size_t hash_value() const;
+                // Get hash value (unique for handles to the same object)
+                size_t hash_value() const;
 
                 // Get internal pointer
@@ -559 +679 @@
 #endif
 
+        // A helper for working with text inside PCDATA nodes
+        class PUGIXML_CLASS xml_text
+        {
+                friend class xml_node;
+
+                xml_node_struct* _root;
+
+                typedef void (*unspecified_bool_type)(xml_text***);
+
+                explicit xml_text(xml_node_struct* root);
+
+                xml_node_struct* _data_new();
+                xml_node_struct* _data() const;
+
+        public:
+                // Default constructor. Constructs an empty object.
+                xml_text();
+
+                // Safe bool conversion operator
+                operator unspecified_bool_type() const;
+
+                // Borland C++ workaround
+                bool operator!() const;
+
+                // Check if text object is empty
+                bool empty() const;
+
+                // Get text, or "" if object is empty
+                const char_t* get() const;
+
+                // Get text, or the default value if object is empty
+                const char_t* as_string(const char_t* def = PUGIXML_TEXT("")) const;
+
+                // Get text as a number, or the default value if conversion did not succeed or object is empty
+                int as_int(int def = 0) const;
+                unsigned int as_uint(unsigned int def = 0) const;
+                double as_double(double def = 0) const;
+                float as_float(float def = 0) const;
+
+        #ifdef PUGIXML_HAS_LONG_LONG
+                long long as_llong(long long def = 0) const;
+                unsigned long long as_ullong(unsigned long long def = 0) const;
+        #endif
+
+                // Get text as bool (returns true if first character is in '1tTyY' set), or the default value if object is empty
+                bool as_bool(bool def = false) const;
+
+                // Set text (returns false if object is empty or there is not enough memory)
+                bool set(const char_t* rhs);
+
+                // Set text with type conversion (numbers are converted to strings, boolean is converted to "true"/"false")
+                bool set(int rhs);
+                bool set(unsigned int rhs);
+                bool set(long rhs);
+                bool set(unsigned long rhs);
+                bool set(double rhs);
+                bool set(float rhs);
+                bool set(bool rhs);
+
+        #ifdef PUGIXML_HAS_LONG_LONG
+                bool set(long long rhs);
+                bool set(unsigned long long rhs);
+        #endif
+
+                // Set text (equivalent to set without error checking)
+                xml_text& operator=(const char_t* rhs);
+                xml_text& operator=(int rhs);
+                xml_text& operator=(unsigned int rhs);
+                xml_text& operator=(long rhs);
+                xml_text& operator=(unsigned long rhs);
+                xml_text& operator=(double rhs);
+                xml_text& operator=(float rhs);
+                xml_text& operator=(bool rhs);
+
+        #ifdef PUGIXML_HAS_LONG_LONG
+                xml_text& operator=(long long rhs);
+                xml_text& operator=(unsigned long long rhs);
+        #endif
+
+                // Get the data node (node_pcdata or node_cdata) for this object
+                xml_node data() const;
+        };
+
+#ifdef __BORLANDC__
+        // Borland C++ workaround
+        bool PUGIXML_FUNCTION operator&&(const xml_text& lhs, bool rhs);
+        bool PUGIXML_FUNCTION operator||(const xml_text& lhs, bool rhs);
+#endif
+
         // Child node iterator (a bidirectional iterator over a collection of xml_node)
         class PUGIXML_CLASS xml_node_iterator
@@ -565 +774 @@
 
         private:
-                xml_node _wrap;
+                mutable xml_node _wrap;
                 xml_node _parent;
 
@@ -581 +790 @@
         #endif
 
-        // Default constructor
+                // Default constructor
                 xml_node_iterator();
 
-        // Construct an iterator which points to the specified node
+                // Construct an iterator which points to the specified node
                 xml_node_iterator(const xml_node& node);
 
-        // Iterator operators
+                // Iterator operators
                 bool operator==(const xml_node_iterator& rhs) const;
                 bool operator!=(const xml_node_iterator& rhs) const;
 
-                xml_node& operator*();
-                xml_node* operator->();
+                xml_node& operator*() const;
+                xml_node* operator->() const;
 
                 const xml_node_iterator& operator++();
@@ -607 +816 @@
 
         private:
-                xml_attribute _wrap;
+                mutable xml_attribute _wrap;
                 xml_node _parent;
 
@@ -623 +832 @@
         #endif
 
-        // Default constructor
+                // Default constructor
                 xml_attribute_iterator();
 
-        // Construct an iterator which points to the specified attribute
+                // Construct an iterator which points to the specified attribute
                 xml_attribute_iterator(const xml_attribute& attr, const xml_node& parent);
 
@@ -633 +842 @@
                 bool operator!=(const xml_attribute_iterator& rhs) const;
 
-                xml_attribute& operator*();
-                xml_attribute* operator->();
+                xml_attribute& operator*() const;
+                xml_attribute* operator->() const;
 
                 const xml_attribute_iterator& operator++();
@@ -643 +852 @@
         };
 
+        // Named node range helper
+        class PUGIXML_CLASS xml_named_node_iterator
+        {
+                friend class xml_node;
+
+        public:
+                // Iterator traits
+                typedef ptrdiff_t difference_type;
+                typedef xml_node value_type;
+                typedef xml_node* pointer;
+                typedef xml_node& reference;
+
+        #ifndef PUGIXML_NO_STL
+                typedef std::bidirectional_iterator_tag iterator_category;
+        #endif
+
+                // Default constructor
+                xml_named_node_iterator();
+
+                // Construct an iterator which points to the specified node
+                xml_named_node_iterator(const xml_node& node, const char_t* name);
+
+                // Iterator operators
+                bool operator==(const xml_named_node_iterator& rhs) const;
+                bool operator!=(const xml_named_node_iterator& rhs) const;
+
+                xml_node& operator*() const;
+                xml_node* operator->() const;
+
+                const xml_named_node_iterator& operator++();
+                xml_named_node_iterator operator++(int);
+
+                const xml_named_node_iterator& operator--();
+                xml_named_node_iterator operator--(int);
+
+        private:
+                mutable xml_node _wrap;
+                xml_node _parent;
+                const char_t* _name;
+
+                xml_named_node_iterator(xml_node_struct* ref, xml_node_struct* parent, const char_t* name);
+        };
+
         // Abstract tree walker class (see xml_node::traverse)
         class PUGIXML_CLASS xml_tree_walker
@@ -650 +902 @@
         private:
                 int _depth;
-        
+
         protected:
                 // Get current traversal depth
                 int depth() const;
-        
+
         public:
                 xml_tree_walker();
@@ -672 +924 @@
         enum xml_parse_status
         {
-                status_ok = 0,              // No error
-
-                status_file_not_found,      // File was not found during load_file()
-                status_io_error,            // Error reading from file/stream
-                status_out_of_memory,       // Could not allocate memory
-                status_internal_error,      // Internal error occurred
-
-                status_unrecognized_tag,    // Parser could not determine tag type
-
-                status_bad_pi,              // Parsing error occurred while parsing document declaration/processing instruction
-                status_bad_comment,         // Parsing error occurred while parsing comment
-                status_bad_cdata,           // Parsing error occurred while parsing CDATA section
-                status_bad_doctype,         // Parsing error occurred while parsing document type declaration
-                status_bad_pcdata,          // Parsing error occurred while parsing PCDATA section
-                status_bad_start_element,   // Parsing error occurred while parsing start element tag
-                status_bad_attribute,       // Parsing error occurred while parsing element attribute
-                status_bad_end_element,     // Parsing error occurred while parsing end element tag
-                status_end_element_mismatch // There was a mismatch of start-end tags (closing tag had incorrect name, some tag was not closed or there was an excessive closing tag)
+                status_ok = 0,                          // No error
+
+                status_file_not_found,          // File was not found during load_file()
+                status_io_error,                        // Error reading from file/stream
+                status_out_of_memory,           // Could not allocate memory
+                status_internal_error,          // Internal error occurred
+
+                status_unrecognized_tag,        // Parser could not determine tag type
+
+                status_bad_pi,                          // Parsing error occurred while parsing document declaration/processing instruction
+                status_bad_comment,                     // Parsing error occurred while parsing comment
+                status_bad_cdata,                       // Parsing error occurred while parsing CDATA section
+                status_bad_doctype,                     // Parsing error occurred while parsing document type declaration
+                status_bad_pcdata,                      // Parsing error occurred while parsing PCDATA section
+                status_bad_start_element,       // Parsing error occurred while parsing start element tag
+                status_bad_attribute,           // Parsing error occurred while parsing element attribute
+                status_bad_end_element,         // Parsing error occurred while parsing end element tag
+                status_end_element_mismatch,// There was a mismatch of start-end tags (closing tag had incorrect name, some tag was not closed or there was an excessive closing tag)
+
+                status_append_invalid_root,     // Unable to append nodes since root type is not node_element or node_document (exclusive to xml_node::append_buffer)
+
+                status_no_document_element      // Parsing resulted in a document without element nodes
         };
 
@@ -704 +960 @@
                 xml_encoding encoding;
 
-        // Default constructor, initializes object to failed state
+                // Default constructor, initializes object to failed state
                 xml_parse_result();
 
@@ -721 +977 @@
 
                 char _memory[192];
-                
+
                 // Non-copyable semantics
                 xml_document(const xml_document&);
-                const xml_document& operator=(const xml_document&);
-
-                void create();
-                void destroy();
-
-                xml_parse_result load_buffer_impl(void* contents, size_t size, unsigned int options, xml_encoding encoding, bool is_mutable, bool own);
+                xml_document& operator=(const xml_document&);
+
+                void _create();
+                void _destroy();
 
         public:
@@ -738 +992 @@
                 ~xml_document();
 
-        // Removes all nodes, leaving the empty document
+                // Removes all nodes, leaving the empty document
                 void reset();
 
-        // Removes all nodes, then copies the entire contents of the specified document
+                // Removes all nodes, then copies the entire contents of the specified document
                 void reset(const xml_document& proto);
 
@@ -750 +1004 @@
         #endif
 
+                // (deprecated: use load_string instead) Load document from zero-terminated string. No encoding conversions are applied.
+                xml_parse_result load(const char_t* contents, unsigned int options = parse_default);
+
                 // Load document from zero-terminated string. No encoding conversions are applied.
-                xml_parse_result load(const char_t* contents, unsigned int options = parse_default);
+                xml_parse_result load_string(const char_t* contents, unsigned int options = parse_default);
 
                 // Load document from file
@@ -761 +1018 @@
 
                 // Load document from buffer, using the buffer for in-place parsing (the buffer is modified and used for storage of document data).
-        // You should ensure that buffer data will persist throughout the document's lifetime, and free the buffer memory manually once document is destroyed.
+                // You should ensure that buffer data will persist throughout the document's lifetime, and free the buffer memory manually once document is destroyed.
                 xml_parse_result load_buffer_inplace(void* contents, size_t size, unsigned int options = parse_default, xml_encoding encoding = encoding_auto);
 
                 // Load document from buffer, using the buffer for in-place parsing (the buffer is modified and used for storage of document data).
-        // You should allocate the buffer with pugixml allocation function; document will free the buffer when it is no longer needed (you can't use it anymore).
+                // You should allocate the buffer with pugixml allocation function; document will free the buffer when it is no longer needed (you can't use it anymore).
                 xml_parse_result load_buffer_inplace_own(void* contents, size_t size, unsigned int options = parse_default, xml_encoding encoding = encoding_auto);
 
@@ -781 +1038 @@
                 bool save_file(const wchar_t* path, const char_t* indent = PUGIXML_TEXT("\t"), unsigned int flags = format_default, xml_encoding encoding = encoding_auto) const;
 
-        // Get document element
-        xml_node document_element() const;
+                // Get document element
+                xml_node document_element() const;
         };
 
@@ -789 +1046 @@
         enum xpath_value_type
         {
-                xpath_type_none,      // Unknown type (query failed to compile)
+                xpath_type_none,          // Unknown type (query failed to compile)
                 xpath_type_node_set,  // Node set (xpath_node_set)
-                xpath_type_number,    // Number
-                xpath_type_string,    // String
-                xpath_type_boolean    // Boolean
-        };
-
-    // XPath parsing result
+                xpath_type_number,        // Number
+                xpath_type_string,        // String
+                xpath_type_boolean        // Boolean
+        };
+
+        // XPath parsing result
         struct PUGIXML_CLASS xpath_parse_result
         {
@@ -805 +1062 @@
                 ptrdiff_t offset;
 
-        // Default constructor, initializes object to failed state
+                // Default constructor, initializes object to failed state
                 xpath_parse_result();
 
@@ -824 +1081 @@
                 xpath_variable* _next;
 
-                xpath_variable();
+                xpath_variable(xpath_value_type type);
 
                 // Non-copyable semantics
                 xpath_variable(const xpath_variable&);
                 xpath_variable& operator=(const xpath_variable&);
-                
-        public:
-        // Get variable name
+
+        public:
+                // Get variable name
                 const char_t* name() const;
 
-        // Get variable type
+                // Get variable type
                 xpath_value_type type() const;
 
-        // Get variable value; no type conversion is performed, default value (false, NaN, empty string, empty node set) is returned on type mismatch error
+                // Get variable value; no type conversion is performed, default value (false, NaN, empty string, empty node set) is returned on type mismatch error
                 bool get_boolean() const;
                 double get_number() const;
@@ -843 +1100 @@
                 const xpath_node_set& get_node_set() const;
 
-        // Set variable value; no type conversion is performed, false is returned on type mismatch error
+                // Set variable value; no type conversion is performed, false is returned on type mismatch error
                 bool set(bool value);
                 bool set(double value);
@@ -856 +1113 @@
                 xpath_variable* _data[64];
 
-                // Non-copyable semantics
-                xpath_variable_set(const xpath_variable_set&);
-                xpath_variable_set& operator=(const xpath_variable_set&);
-
-                xpath_variable* find(const char_t* name) const;
-
-        public:
-        // Default constructor/destructor
+                void _assign(const xpath_variable_set& rhs);
+                void _swap(xpath_variable_set& rhs);
+
+                xpath_variable* _find(const char_t* name) const;
+
+                static bool _clone(xpath_variable* var, xpath_variable** out_result);
+                static void _destroy(xpath_variable* var);
+
+        public:
+                // Default constructor/destructor
                 xpath_variable_set();
                 ~xpath_variable_set();
 
-        // Add a new variable or get the existing one, if the types match
+                // Copy constructor/assignment operator
+                xpath_variable_set(const xpath_variable_set& rhs);
+                xpath_variable_set& operator=(const xpath_variable_set& rhs);
+
+        #ifdef PUGIXML_HAS_MOVE
+                // Move semantics support
+                xpath_variable_set(xpath_variable_set&& rhs);
+                xpath_variable_set& operator=(xpath_variable_set&& rhs);
+        #endif
+
+                // Add a new variable or get the existing one, if the types match
                 xpath_variable* add(const char_t* name, xpath_value_type type);
 
-        // Set value of an existing variable; no type conversion is performed, false is returned if there is no such variable or if types mismatch
+                // Set value of an existing variable; no type conversion is performed, false is returned if there is no such variable or if types mismatch
                 bool set(const char_t* name, bool value);
                 bool set(const char_t* name, double value);
@@ -876 +1145 @@
                 bool set(const char_t* name, const xpath_node_set& value);
 
-        // Get existing variable by name
+                // Get existing variable by name
                 xpath_variable* get(const char_t* name);
                 const xpath_variable* get(const char_t* name) const;
@@ -888 +1157 @@
                 xpath_parse_result _result;
 
-        typedef void* xpath_query::*unspecified_bool_type;
+                typedef void (*unspecified_bool_type)(xpath_query***);
 
                 // Non-copyable semantics
@@ -895 +1164 @@
 
         public:
-        // Construct a compiled object from XPath expression.
-        // If PUGIXML_NO_EXCEPTIONS is not defined, throws xpath_exception on compilation errors.
+                // Construct a compiled object from XPath expression.
+                // If PUGIXML_NO_EXCEPTIONS is not defined, throws xpath_exception on compilation errors.
                 explicit xpath_query(const char_t* query, xpath_variable_set* variables = 0);
+
+                // Constructor
+                xpath_query();
 
                 // Destructor
                 ~xpath_query();
 
+        #ifdef PUGIXML_HAS_MOVE
+                // Move semantics support
+                xpath_query(xpath_query&& rhs);
+                xpath_query& operator=(xpath_query&& rhs);
+        #endif
+
                 // Get query expression return type
                 xpath_value_type return_type() const;
-                
+
                 // Evaluate expression as boolean value in the specified context; performs type conversion if necessary.
-        // If PUGIXML_NO_EXCEPTIONS is not defined, throws std::bad_alloc on out of memory errors.
+                // If PUGIXML_NO_EXCEPTIONS is not defined, throws std::bad_alloc on out of memory errors.
                 bool evaluate_boolean(const xpath_node& n) const;
-                
+
                 // Evaluate expression as double value in the specified context; performs type conversion if necessary.
-        // If PUGIXML_NO_EXCEPTIONS is not defined, throws std::bad_alloc on out of memory errors.
+                // If PUGIXML_NO_EXCEPTIONS is not defined, throws std::bad_alloc on out of memory errors.
                 double evaluate_number(const xpath_node& n) const;
-                
+
         #ifndef PUGIXML_NO_STL
                 // Evaluate expression as string value in the specified context; performs type conversion if necessary.
-        // If PUGIXML_NO_EXCEPTIONS is not defined, throws std::bad_alloc on out of memory errors.
+                // If PUGIXML_NO_EXCEPTIONS is not defined, throws std::bad_alloc on out of memory errors.
                 string_t evaluate_string(const xpath_node& n) const;
         #endif
-                
+
                 // Evaluate expression as string value in the specified context; performs type conversion if necessary.
-        // At most capacity characters are written to the destination buffer, full result size is returned (includes terminating zero).
-        // If PUGIXML_NO_EXCEPTIONS is not defined, throws std::bad_alloc on out of memory errors.
-        // If PUGIXML_NO_EXCEPTIONS is defined, returns empty  set instead.
+                // At most capacity characters are written to the destination buffer, full result size is returned (includes terminating zero).
+                // If PUGIXML_NO_EXCEPTIONS is not defined, throws std::bad_alloc on out of memory errors.
+                // If PUGIXML_NO_EXCEPTIONS is defined, returns empty  set instead.
                 size_t evaluate_string(char_t* buffer, size_t capacity, const xpath_node& n) const;
 
                 // Evaluate expression as node set in the specified context.
-        // If PUGIXML_NO_EXCEPTIONS is not defined, throws xpath_exception on type mismatch and std::bad_alloc on out of memory errors.
-        // If PUGIXML_NO_EXCEPTIONS is defined, returns empty node set instead.
+                // If PUGIXML_NO_EXCEPTIONS is not defined, throws xpath_exception on type mismatch and std::bad_alloc on out of memory errors.
+                // If PUGIXML_NO_EXCEPTIONS is defined, returns empty node set instead.
                 xpath_node_set evaluate_node_set(const xpath_node& n) const;
+
+                // Evaluate expression as node set in the specified context.
+                // Return first node in document order, or empty node if node set is empty.
+                // If PUGIXML_NO_EXCEPTIONS is not defined, throws xpath_exception on type mismatch and std::bad_alloc on out of memory errors.
+                // If PUGIXML_NO_EXCEPTIONS is defined, returns empty node instead.
+                xpath_node evaluate_node(const xpath_node& n) const;
 
                 // Get parsing result (used to get compilation errors in PUGIXML_NO_EXCEPTIONS mode)
@@ -936 +1220 @@
                 operator unspecified_bool_type() const;
 
-        // Borland C++ workaround
+                // Borland C++ workaround
                 bool operator!() const;
         };
-        
+
         #ifndef PUGIXML_NO_EXCEPTIONS
         // XPath exception class
@@ -952 +1236 @@
 
                 // Get error message
-                virtual const char* what() const throw();
-
-        // Get parse result
+                virtual const char* what() const throw() PUGIXML_OVERRIDE;
+
+                // Get parse result
                 const xpath_parse_result& result() const;
         };
         #endif
-        
+
         // XPath node class (either xml_node or xml_attribute)
         class PUGIXML_CLASS xpath_node
@@ -965 +1249 @@
                 xml_node _node;
                 xml_attribute _attribute;
-        
-        typedef xml_node xpath_node::*unspecified_bool_type;
+
+                typedef void (*unspecified_bool_type)(xpath_node***);
 
         public:
                 // Default constructor; constructs empty XPath node
                 xpath_node();
-                
+
                 // Construct XPath node from XML node/attribute
                 xpath_node(const xml_node& node);
@@ -979 +1263 @@
                 xml_node node() const;
                 xml_attribute attribute() const;
-                
+
                 // Get parent of contained node/attribute
                 xml_node parent() const;
 
-        // Safe bool conversion operator
+                // Safe bool conversion operator
                 operator unspecified_bool_type() const;
-                
-        // Borland C++ workaround
-        bool operator!() const;
+
+                // Borland C++ workaround
+                bool operator!() const;
 
                 // Comparison operators
@@ -1011 +1295 @@
                         type_sorted_reverse             // Sorted by document order (descending)
                 };
-                
+
                 // Constant iterator type
                 typedef const xpath_node* const_iterator;
-        
+
+                // We define non-constant iterator to be the same as constant iterator so that various generic algorithms (i.e. boost foreach) work
+                typedef const xpath_node* iterator;
+
                 // Default constructor. Constructs empty set.
                 xpath_node_set();
@@ -1023 +1310 @@
                 // Destructor
                 ~xpath_node_set();
-                
+
                 // Copy constructor/assignment operator
                 xpath_node_set(const xpath_node_set& ns);
                 xpath_node_set& operator=(const xpath_node_set& ns);
 
+        #ifdef PUGIXML_HAS_MOVE
+                // Move semantics support
+                xpath_node_set(xpath_node_set&& rhs);
+                xpath_node_set& operator=(xpath_node_set&& rhs);
+        #endif
+
                 // Get collection type
                 type_t type() const;
-                
+
                 // Get collection size
                 size_t size() const;
 
-        // Indexing operator
+                // Indexing operator
                 const xpath_node& operator[](size_t index) const;
-                
+
                 // Collection iterators
                 const_iterator begin() const;
@@ -1043 +1336 @@
                 // Sort the collection in ascending/descending order by document order
                 void sort(bool reverse = false);
-                
+
                 // Get first node in the collection by document order
                 xpath_node first() const;
-                
+
                 // Check if collection is empty
                 bool empty() const;
-    
+
         private:
                 type_t _type;
-                
+
                 xpath_node _storage;
-                
+
                 xpath_node* _begin;
                 xpath_node* _end;
 
-                void _assign(const_iterator begin, const_iterator end);
+                void _assign(const_iterator begin, const_iterator end, type_t type);
+                void _move(xpath_node_set& rhs);
         };
 #endif
@@ -1066 +1360 @@
         std::basic_string<char, std::char_traits<char>, std::allocator<char> > PUGIXML_FUNCTION as_utf8(const wchar_t* str);
         std::basic_string<char, std::char_traits<char>, std::allocator<char> > PUGIXML_FUNCTION as_utf8(const std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> >& str);
-        
+
         // Convert UTF8 to wide string
         std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> > PUGIXML_FUNCTION as_wide(const char* str);
@@ -1074 +1368 @@
         // Memory allocation function interface; returns pointer to allocated memory or NULL on failure
         typedef void* (*allocation_function)(size_t size);
-        
+
         // Memory deallocation function interface
-    typedef void (*deallocation_function)(void* ptr);
-
-    // Override default memory management functions. All subsequent allocations/deallocations will be performed via supplied functions.
-    void PUGIXML_FUNCTION set_memory_management_functions(allocation_function allocate, deallocation_function deallocate);
-    
-    // Get current memory management functions
-    allocation_function PUGIXML_FUNCTION get_memory_allocation_function();
-    deallocation_function PUGIXML_FUNCTION get_memory_deallocation_function();
+        typedef void (*deallocation_function)(void* ptr);
+
+        // Override default memory management functions. All subsequent allocations/deallocations will be performed via supplied functions.
+        void PUGIXML_FUNCTION set_memory_management_functions(allocation_function allocate, deallocation_function deallocate);
+
+        // Get current memory management functions
+        allocation_function PUGIXML_FUNCTION get_memory_allocation_function();
+        deallocation_function PUGIXML_FUNCTION get_memory_deallocation_function();
 }
 
@@ -1092 +1386 @@
         std::bidirectional_iterator_tag PUGIXML_FUNCTION _Iter_cat(const pugi::xml_node_iterator&);
         std::bidirectional_iterator_tag PUGIXML_FUNCTION _Iter_cat(const pugi::xml_attribute_iterator&);
+        std::bidirectional_iterator_tag PUGIXML_FUNCTION _Iter_cat(const pugi::xml_named_node_iterator&);
 }
 #endif
@@ -1101 +1396 @@
         std::bidirectional_iterator_tag PUGIXML_FUNCTION __iterator_category(const pugi::xml_node_iterator&);
         std::bidirectional_iterator_tag PUGIXML_FUNCTION __iterator_category(const pugi::xml_attribute_iterator&);
+        std::bidirectional_iterator_tag PUGIXML_FUNCTION __iterator_category(const pugi::xml_named_node_iterator&);
 }
 #endif
@@ -1106 +1402 @@
 #endif
 
+// Make sure implementation is included in header-only mode
+// Use macro expansion in #include to work around QMake (QTBUG-11923)
+#if defined(PUGIXML_HEADER_ONLY) && !defined(PUGIXML_SOURCE)
+#       define PUGIXML_SOURCE "pugixml.cpp"
+#       include PUGIXML_SOURCE
+#endif
+
 /**
- * Copyright (c) 2006-2010 Arseny Kapoulkine
+ * Copyright (c) 2006-2016 Arseny Kapoulkine
  *
  * Permission is hereby granted, free of charge, to any person
@@ -1120 +1423 @@
  * The above copyright notice and this permission notice shall be
  * included in all copies or substantial portions of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES

ThirdParty/vmg/src/thirdparty/pugixml/readme.txt

@@ -1 @@
-pugixml 1.0 - an XML processing library
+pugixml 1.8 - an XML processing library
 
-Copyright (C) 2006-2010, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
+Copyright (C) 2006-2016, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
 Report bugs and download new versions at http://pugixml.org/
 
@@ -29 +29 @@
 This library is distributed under the MIT License:
 
-Copyright (c) 2006-2010 Arseny Kapoulkine
+Copyright (c) 2006-2016 Arseny Kapoulkine
 
 Permission is hereby granted, free of charge, to any person