xmlscanner.hpp

Go to the documentation of this file.

/*
 * Copyright (c) 2014 Patrick P. Frey
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#ifndef __TEXTWOLF_XML_SCANNER_HPP__
#define __TEXTWOLF_XML_SCANNER_HPP__
#include "textwolf/char.hpp"
#include "textwolf/charset_interface.hpp"
#include "textwolf/exception.hpp"
#include "textwolf/textscanner.hpp"
#include "textwolf/traits.hpp"
#include <map>
#include <cstddef>

namespace textwolf {

class ScannerStatemachine :public throws_exception
{
public:
        enum
        {
                MaxNofStates=64                         
        };
        struct Element
        {
                int fallbackState;                      
                int missError;                          

                struct Action
                {
                        int op;                         
                        int arg;                        
                };
                Action action;                          
                unsigned char nofnext;                  
                signed char next[ NofControlCharacter]; 

                Element() :fallbackState(-1),missError(-1),nofnext(0)
                {
                        action.op = -1;
                        action.arg = 0;
                        for (unsigned int ii=0; ii<NofControlCharacter; ii++) next[ii] = -1;
                }
        };
        Element* get( int stateIdx) throw(exception)
        {
                if ((unsigned int)stateIdx>size) throw exception(InvalidState);
                return tab + stateIdx;
        }

private:
        Element tab[ MaxNofStates];     
        unsigned int size;              

        void newState( int stateIdx) throw(exception)
        {
                if (size != (unsigned int)stateIdx) throw exception( StateNumbersNotAscending);
                if (size >= MaxNofStates) throw exception( DimOutOfRange);
                size++;
        }

        void addOtherTransition( int nextState) throw(exception)
        {
                if (size == 0) throw exception( InvalidState);
                if (nextState < 0 || nextState > MaxNofStates) throw exception( InvalidParamState);
                for (unsigned int inputchr=0; inputchr<NofControlCharacter; inputchr++)
                {
                        if (tab[ size-1].next[ inputchr] == -1) tab[ size-1].next[ inputchr] = (unsigned char)nextState;
                }
                tab[ size-1].nofnext = NofControlCharacter;
        }

        void addTransition( ControlCharacter inputchr, int nextState) throw(exception)
        {
                if (size == 0) throw exception( InvalidState);
                if ((int)inputchr >= (int)NofControlCharacter) throw exception( InvalidParamChar);
                if (nextState < 0 || nextState > MaxNofStates) throw exception( InvalidParamState);
                if (tab[ size-1].next[ inputchr] != -1) throw exception( DuplicateStateTransition);
                tab[ size-1].next[ inputchr] = (unsigned char)nextState;
                tab[ size-1].nofnext += 1;
        }

        void addTransition( ControlCharacter inputchr) throw(exception)
        {
                addTransition( inputchr, size-1);
        }

        void addAction( int action_op, int action_arg=0) throw(exception)
        {
                if (size == 0) throw exception( InvalidState);
                if (tab[ size-1].action.op != -1) throw exception( InvalidState);
                tab[ size-1].action.op = action_op;
                tab[ size-1].action.arg = action_arg;
        }

        void addMiss( int error) throw(exception)
        {
                if (size == 0) throw exception( InvalidState);
                if (tab[ size-1].missError != -1) throw exception( InvalidState);
                tab[ size-1].missError = error;
        }

        void addFallback( int stateIdx) throw(exception)
        {
                if (size == 0) throw exception( InvalidState);
                if (tab[ size-1].fallbackState != -1) throw exception( InvalidState);
                if (stateIdx < 0 || stateIdx > MaxNofStates) throw exception( InvalidParamState);
                tab[ size-1].fallbackState = stateIdx;
        }
public:
        ScannerStatemachine() :size(0){}

        ScannerStatemachine& operator[]( int stateIdx)                                                                  {newState(stateIdx); return *this;}
        ScannerStatemachine& operator()( ControlCharacter inputchr, int ns)                                             {addTransition(inputchr,ns); return *this;}
        ScannerStatemachine& operator()( ControlCharacter i1, ControlCharacter i2, int ns)                              {addTransition(i1,ns); addTransition(i2,ns); return *this;}
        ScannerStatemachine& operator()( ControlCharacter i1, ControlCharacter i2, ControlCharacter i3, int ns)         {addTransition(i1,ns); addTransition(i2,ns); addTransition(i3,ns); return *this;}
        ScannerStatemachine& operator()( ControlCharacter inputchr)                                                     {addTransition(inputchr); return *this;}
        ScannerStatemachine& action( int aa, int arg=0)                                                                 {addAction(aa,arg); return *this;}
        ScannerStatemachine& miss( int ee)                                                                              {addMiss(ee); return *this;}
        ScannerStatemachine& fallback( int stateIdx)                                                                    {addFallback(stateIdx); return *this;}
        ScannerStatemachine& other( int stateIdx)                                                                       {addOtherTransition(stateIdx); return *this;}
};

class XMLScannerBase
{
public:
        enum ElementType
        {
                None,                                   
                ErrorOccurred,                          
                HeaderStart,                            
                HeaderAttribName,                       
                HeaderAttribValue,                      
                HeaderEnd,                              
                DocAttribValue,                         
                DocAttribEnd,                           
                TagAttribName,                          
                TagAttribValue,                         
                OpenTag,                                
                CloseTag,                               
                CloseTagIm,                             
                Content,                                
                Exit                                    
        };
        enum
        {
                NofElementTypes=Exit+1                  
        };

        static const char* getElementTypeName( ElementType ee)
        {
                static const char* names[ NofElementTypes] = {"None","ErrorOccurred","HeaderStart","HeaderAttribName","HeaderAttribValue","HeaderEnd", "DocAttribValue", "DocAttribEnd", "TagAttribName","TagAttribValue","OpenTag","CloseTag","CloseTagIm","Content","Exit"};
                return names[ (unsigned int)ee];
        }

        enum Error
        {
                Ok,                                     
                ErrIllegalDocumentAttributeDef,         
                ErrExpectedOpenTag,                     
                ErrExpectedXMLTag,                      
                ErrUnexpectedEndOfText,                 
                ErrSyntaxToken,                         
                ErrStringNotTerminated,                 
                ErrUndefinedCharacterEntity,            
                ErrExpectedTagEnd,                      
                ErrExpectedEqual,                       
                ErrExpectedTagAttribute,                
                ErrExpectedCDATATag,                    
                ErrInternal,                            
                ErrUnexpectedEndOfInput,                
                ErrExpectedEndOfLine,                   
                ErrExpectedDash2                        
        };

        static const char* getErrorString( Error ee)
        {
                enum {NofErrors=16};
                static const char* sError[NofErrors]
                        = {0,"illegal document attribute definition",
                                "expected open tag",
                                "expected XML tag",
                                "unexpected end of text",
                                "syntax token",
                                "string not terminated",
                                "undefined character entity",
                                "expected tag end",
                                "expected equal",
                                "expected tag attribute",
                                "expected CDATA tag",
                                "internal (illegal state)",
                                "unexpected end of input",
                                "expected end of line",
                                "expected 2nd '-' to complete marker for start of comment '<!--'"
                };
                return sError[(unsigned int)ee];
        }

        enum STMState
        {
                START, STARTTAG, XTAG, PITAG, PITAGEND, XTAGEND, XTAGDONE, XTAGAISK, XTAGANAM, XTAGAESK, XTAGAVSK, XTAGAVID, XTAGAVSQ, XTAGAVDQ, XTAGAVQE,
                DOCSTART, CONTENT, TOKEN, SEEKTOK, XMLTAG, OPENTAG, CLOSETAG, TAGCLSK, TAGAISK, TAGANAM, TAGAESK, TAGAVSK, TAGAVID, TAGAVSQ, TAGAVDQ, TAGAVQE,
                TAGCLIM, ENTITYSL, ENTITY, ENTITYE, ENTITYID, ENTITYSQ, ENTITYDQ, ENTITYLC, 
                COMDASH2, COMSEEKE, COMENDD2, COMENDCL, CDATA, CDATA1, CDATA2, CDATA3, EXIT
        };

        static const char* getStateString( STMState s)
        {
                enum Constant {NofStates=48};
                static const char* sState[NofStates]
                = {
                        "START", "STARTTAG", "XTAG", "PITAG", "PITAGEND",
                        "XTAGEND", "XTAGDONE", "XTAGAISK", "XTAGANAM",
                        "XTAGAESK", "XTAGAVSK", "XTAGAVID", "XTAGAVSQ", "XTAGAVDQ",
                        "XTAGAVQE", "DOCSTART", "CONTENT", "TOKEN", "SEEKTOK", "XMLTAG",
                        "OPENTAG", "CLOSETAG", "TAGCLSK", "TAGAISK", "TAGANAM",
                        "TAGAESK", "TAGAVSK", "TAGAVID", "TAGAVSQ", "TAGAVDQ",
                        "TAGAVQE", "TAGCLIM", "ENTITYSL", "ENTITY", "ENTITYE",
                        "ENTITYID", "ENTITYSQ", "ENTITYDQ",  "ENTITYLC",
                        "COMDASH2", "COMSEEKE", "COMENDD2", "COMENDCL",
                        "CDATA", "CDATA1", "CDATA2", "CDATA3", "EXIT"
                };
                return sState[(unsigned int)s];
        }

        enum STMAction
        {
                Return, ReturnWord, ReturnContent, ReturnIdentifier, ReturnSQString, ReturnDQString, ExpectIdentifierXML, ExpectIdentifierCDATA, ReturnEOF,
                NofSTMActions = 9
        };

        static const char* getActionString( STMAction a)
        {
                static const char* name[ NofSTMActions] = {"Return", "ReturnWord", "ReturnContent", "ReturnIdentifier", "ReturnSQString", "ReturnDQString", "ExpectIdentifierXML", "ExpectIdentifierCDATA", "ReturnEOF"};
                return name[ (unsigned int)a];
        };

        struct Statemachine :public ScannerStatemachine
        {
                Statemachine()
                {
                        (*this)
                        [ START    ](EndOfText,EXIT)(EndOfLine)(Cntrl)(Space)(Lt,STARTTAG).miss(ErrExpectedOpenTag)
                        [ STARTTAG ](EndOfLine)(Cntrl)(Space)(Questm,XTAG)(Exclam,ENTITYSL).fallback(OPENTAG)
                        [ XTAG     ].action(ExpectIdentifierXML)(EndOfLine,Cntrl,Space,XTAGAISK)(Questm,XTAGEND).miss(ErrExpectedXMLTag)
                        [ PITAG    ](Questm,PITAGEND).other(PITAG)
                        [ PITAGEND ](Gt,CONTENT).miss(ErrExpectedTagEnd)
                        [ XTAGEND  ](Gt,XTAGDONE)(EndOfLine)(Cntrl)(Space).miss(ErrExpectedTagEnd)
                        [ XTAGDONE ].action(Return,HeaderEnd).fallback(DOCSTART)
                        [ XTAGAISK ](EndOfLine)(Cntrl)(Space)(Questm,XTAGEND).fallback(XTAGANAM)
                        [ XTAGANAM ].action(ReturnIdentifier,HeaderAttribName)(EndOfLine,Cntrl,Space,XTAGAESK)(Equal,XTAGAVSK).miss(ErrExpectedEqual)
                        [ XTAGAESK ](EndOfLine)(Cntrl)(Space)(Equal,XTAGAVSK).miss(ErrExpectedEqual)
                        [ XTAGAVSK ](EndOfLine)(Cntrl)(Space)(Sq,XTAGAVSQ)(Dq,XTAGAVDQ).fallback(XTAGAVID)
                        [ XTAGAVID ].action(ReturnIdentifier,HeaderAttribValue)(EndOfLine,Cntrl,Space,XTAGAISK)(Questm,XTAGEND).miss(ErrExpectedTagAttribute)
                        [ XTAGAVSQ ].action(ReturnSQString,HeaderAttribValue)(Sq,XTAGAVQE).miss(ErrStringNotTerminated)
                        [ XTAGAVDQ ].action(ReturnDQString,HeaderAttribValue)(Dq,XTAGAVQE).miss(ErrStringNotTerminated)
                        [ XTAGAVQE ](EndOfLine,Cntrl,Space,XTAGAISK)(Questm,XTAGEND).miss(ErrExpectedTagAttribute)
                        [ DOCSTART ](EndOfText,EXIT)(EndOfLine)(Cntrl)(Space)(Lt,XMLTAG).fallback(TOKEN)
                        [ CONTENT  ](EndOfText,EXIT)(Lt,XMLTAG).fallback(TOKEN)
                        [ TOKEN    ].action(ReturnContent,Content)(EndOfText,EXIT)(EndOfLine,Cntrl,Space,CONTENT)(Lt,XMLTAG).fallback(CONTENT)
                        [ SEEKTOK  ](EndOfText,EXIT)(EndOfLine)(Cntrl)(Space)(Lt,XMLTAG).fallback(TOKEN)
                        [ XMLTAG   ](EndOfLine)(Cntrl)(Space)(Questm,PITAG)(Exclam,ENTITYSL)(Slash,CLOSETAG).fallback(OPENTAG)
                        [ OPENTAG  ].action(ReturnIdentifier,OpenTag)(EndOfLine,Cntrl,Space,TAGAISK)(Slash,TAGCLIM)(Gt,CONTENT).miss(ErrExpectedTagAttribute)
                        [ CLOSETAG ].action(ReturnIdentifier,CloseTag)(EndOfLine,Cntrl,Space,TAGCLSK)(Gt,CONTENT).miss(ErrExpectedTagEnd)
                        [ TAGCLSK  ](EndOfLine)(Cntrl)(Space)(Gt,CONTENT).miss(ErrExpectedTagEnd)
                        [ TAGAISK  ](EndOfLine)(Cntrl)(Space)(Gt,CONTENT)(Slash,TAGCLIM).fallback(TAGANAM)
                        [ TAGANAM  ].action(ReturnIdentifier,TagAttribName)(EndOfLine,Cntrl,Space,TAGAESK)(Equal,TAGAVSK).miss(ErrExpectedEqual)
                        [ TAGAESK  ](EndOfLine)(Cntrl)(Space)(Equal,TAGAVSK).miss(ErrExpectedEqual)
                        [ TAGAVSK  ](EndOfLine)(Cntrl)(Space)(Sq,TAGAVSQ)(Dq,TAGAVDQ).fallback(TAGAVID)
                        [ TAGAVID  ].action(ReturnIdentifier,TagAttribValue)(EndOfLine,Cntrl,Space,TAGAISK)(Slash,TAGCLIM)(Gt,CONTENT).miss(ErrExpectedTagAttribute)
                        [ TAGAVSQ  ].action(ReturnSQString,TagAttribValue)(Sq,TAGAVQE).miss(ErrStringNotTerminated)
                        [ TAGAVDQ  ].action(ReturnDQString,TagAttribValue)(Dq,TAGAVQE).miss(ErrStringNotTerminated)
                        [ TAGAVQE  ](EndOfLine,Cntrl,Space,TAGAISK)(Slash,TAGCLIM)(Gt,CONTENT).miss(ErrExpectedTagAttribute)
                        [ TAGCLIM  ].action(Return,CloseTagIm)(EndOfLine)(Cntrl)(Space)(Gt,CONTENT).miss(ErrExpectedTagEnd)
                        [ ENTITYSL ](Osb,CDATA)(Dash,COMDASH2).fallback(ENTITY)
                        [ ENTITY   ](Gt,ENTITYE)(EndOfLine)(Cntrl)(Space)(Dq,ENTITYDQ)(Sq,ENTITYSQ)(Osb,ENTITYLC).fallback(ENTITYID)
                        [ ENTITYE  ].action(Return,DocAttribEnd).fallback(SEEKTOK)
                        [ ENTITYID ].action(ReturnIdentifier,DocAttribValue)(EndOfLine,Cntrl,Space,ENTITY)(Gt,ENTITYE).miss(ErrIllegalDocumentAttributeDef)
                        [ ENTITYSQ ].action(ReturnSQString,DocAttribValue)(Sq,ENTITY).miss(ErrStringNotTerminated)
                        [ ENTITYDQ ].action(ReturnDQString,DocAttribValue)(Dq,ENTITY).miss(ErrStringNotTerminated)
                        [ ENTITYLC ](Csb,ENTITY).other( ENTITYLC)
                        [ COMDASH2 ](Dash,COMSEEKE).miss(ErrExpectedDash2)
                        [ COMSEEKE ](Dash,COMENDD2).other(COMSEEKE)
                        [ COMENDD2 ](Dash,COMENDCL).other(COMSEEKE)
                        [ COMENDCL ](Gt,SEEKTOK)(Dash,COMENDD2).other(COMSEEKE)
                        [ CDATA    ].action(ExpectIdentifierCDATA)(Osb,CDATA1).miss(ErrExpectedCDATATag)
                        [ CDATA1   ](Csb,CDATA2).other(CDATA1)
                        [ CDATA2   ](Csb,CDATA3).other(CDATA1)
                        [ CDATA3   ](Gt,CONTENT).other(CDATA1)
                        [ EXIT     ].action(Return,Exit);
                }
        };

        typedef CharMap<bool,false,NofControlCharacter> IsTokenCharMap;

        struct IsTagCharMap :public IsTokenCharMap
        {
                IsTagCharMap()
                {
                        (*this)(Undef,true)(Any,true)(Dash,true);
                }
        };

        struct IsWordCharMap :public IsTokenCharMap
        {
                IsWordCharMap()
                {
                        (*this)(Undef,true)(Equal,true)(Gt,true)(Slash,true)(Dash,true)(Exclam,true)(Questm,true)(Sq,true)(Dq,true)(Osb,true)(Csb,true)(Any,true);
                }
        };

        struct IsContentCharMap :public IsTokenCharMap
        {
                IsContentCharMap()
                {
                        (*this)(Cntrl,true)(Space,true)(EndOfLine,true)(Undef,true)(Equal,true)(Gt,true)(Slash,true)(Dash,true)(Exclam,true)(Questm,true)(Sq,true)(Dq,true)(Osb,true)(Csb,true)(Any,true);
                }
        };

        struct IsSQStringCharMap :public IsContentCharMap
        {
                IsSQStringCharMap()
                {
                        (*this)(Sq,false)(Space,true);
                }
        };

        struct IsDQStringCharMap :public IsContentCharMap
        {
                IsDQStringCharMap()
                {
                        (*this)(Dq,false)(Space,true);
                }
        };
};


template
<
                class InputIterator,
                class InputCharSet_,
                class OutputCharSet_,
                class OutputBuffer_
>
class XMLScanner :public XMLScannerBase
{
private:
        struct TokState
        {
                enum Id
                {
                        Start,                          
                        ParsingDone,                    
                        ParsingKey,                     
                        ParsingEntity,                  
                        ParsingNumericEntity,           
                        ParsingNumericBaseEntity,       
                        ParsingNamedEntity,             
                        ParsingToken                    
                };
                Id id;                                  

                enum EolnState                          
                {
                        SRC,CR
                };
                EolnState eolnState;                    

                unsigned int pos;                       
                unsigned int base;                      
                EChar value;                            
                char buf[ 16];                          
                UChar curchr_saved;                     

                TokState()                              :id(Start),eolnState(SRC),pos(0),base(0),value(0),curchr_saved(0) {}

                void init(Id id_=Start, EolnState eolnState_=SRC)
                {
                        id=id_;eolnState=eolnState_;pos=0;base=0;value=0;curchr_saved=0;
                }
        };
        TokState tokstate;                              

public:
        typedef InputCharSet_ InputCharSet;
        typedef OutputCharSet_ OutputCharSet;
        class iterator;

public:
        typedef TextScanner<InputIterator,InputCharSet_> InputReader;
        typedef XMLScanner<InputIterator,InputCharSet_,OutputCharSet_,OutputBuffer_> ThisXMLScanner;
        typedef std::map<const char*,UChar> EntityMap;
        typedef OutputBuffer_ OutputBuffer;

private:
        void push( UChar ch)
        {
                m_output.print( ch, m_outputBuf);
        }

        void copychar_impl( const traits::TypeCheck::YES&)
        {
                m_src.copychar( m_output, m_outputBuf);
        }

        void copychar_impl( const traits::TypeCheck::NO&)
        {
                push( m_src.chr());
        }

        void copychar()
        {
                copychar_impl( traits::TypeCheck::is_same<InputCharSet,OutputCharSet>::type());
        }

        static unsigned char HEX( unsigned char ch)
        {
                struct HexCharMap :public CharMap<unsigned char, 0xFF>
                {
                        HexCharMap()
                        {
                                (*this)
                                        ('0',0) ('1', 1)('2', 2)('3', 3)('4', 4)('5', 5)('6', 6)('7', 7)('8', 8)('9', 9)
                                        ('A',10)('B',11)('C',12)('D',13)('E',14)('F',15)('a',10)('b',11)('c',12)('d',13)('e',14)('f',15);
                        }
                };
                static HexCharMap hexCharMap;
                return hexCharMap[ch];
        }

        static UChar parseStaticNumericEntityValue( InputReader& ir)
        {
                EChar value = 0;
                unsigned char ch = ir.ascii();
                unsigned int base;
                if (ch != '#') return 0;
                ir.skip();
                ch = ir.ascii();
                if (ch == 'x')
                {
                        ir.skip();
                        ch = ir.ascii();
                        base = 16;
                }
                else
                {
                        base = 10;
                }
                while (ch != ';')
                {
                        unsigned char chval = HEX(ch);
                        if (value >= base) return 0;
                        value = value * base + chval;
                        if (value >= 0xFFFFFFFF) return 0;
                        ir.skip();
                        ch = ir.ascii();
                }
                return (UChar)value;
        }

        void fallbackEntity()
        {
                switch (tokstate.id)
                {
                        case TokState::Start:
                        case TokState::ParsingDone:
                        case TokState::ParsingKey:
                        case TokState::ParsingToken:
                                break;
                        case TokState::ParsingEntity:
                                push('&');
                                break;
                        case TokState::ParsingNumericEntity:
                                push('&');
                                push('#');
                                break;
                        case TokState::ParsingNumericBaseEntity:
                                push('&');
                                push('#');
                                for (unsigned int ii=0; ii<tokstate.pos; ii++) push( tokstate.buf[ii]);
                                break;
                        case TokState::ParsingNamedEntity:
                                push('&');
                                for (unsigned int ii=0; ii<tokstate.pos; ii++) push( tokstate.buf[ii]);
                                break;
                }
        }

        bool parseEntity()
        {
                unsigned char ch;
                tokstate.id = TokState::ParsingEntity;
                ch = m_src.ascii();
                if (ch == '#')
                {
                        m_src.skip();
                        return parseNumericEntity();
                }
                else
                {
                        return parseNamedEntity();
                }
        }

        bool parseNumericEntity()
        {
                unsigned char ch;
                tokstate.id = TokState::ParsingNumericEntity;
                ch = m_src.ascii();
                if (ch == 'x')
                {
                        tokstate.base = 16;
                        m_src.skip();
                        return parseNumericBaseEntity();
                }
                else
                {
                        tokstate.base = 10;
                        return parseNumericBaseEntity();
                }
        }

        bool parseNumericBaseEntity()
        {
                unsigned char ch;
                tokstate.id = TokState::ParsingNumericBaseEntity;

                while (tokstate.pos < sizeof(tokstate.buf))
                {
                        ch = m_src.ascii();
                        if (ch == ';')
                        {
                                if (tokstate.value > 0xFFFFFFFF)
                                {
                                        tokstate.buf[ tokstate.pos++] = ch;
                                        fallbackEntity();
                                        return true;
                                }
                                push( (UChar)tokstate.value);
                                tokstate.init( TokState::ParsingToken);
                                m_src.skip();
                                return true;
                        }
                        else
                        {
                                unsigned char chval = HEX(ch);
                                if (chval >= tokstate.base)
                                {
                                        fallbackEntity();
                                        return true;
                                }
                                tokstate.buf[ tokstate.pos++] = ch;
                                tokstate.value = tokstate.value * tokstate.base + chval;
                                m_src.skip();
                        }
                }
                fallbackEntity();
                return true;
        }

        bool parseNamedEntity()
        {
                unsigned char ch;
                tokstate.id = TokState::ParsingNamedEntity;
                ch = m_src.ascii();
                while (tokstate.pos < sizeof(tokstate.buf)-1 && ch != ';' && m_src.control() == Any)
                {
                        tokstate.buf[ tokstate.pos] = ch;
                        m_src.skip();
                        tokstate.pos++;
                        ch = m_src.ascii();
                }
                if (ch == ';')
                {
                        tokstate.buf[ tokstate.pos] = '\0';
                        if (!pushEntity( tokstate.buf)) return false;
                        tokstate.init( TokState::ParsingToken);
                        m_src.skip();
                        return true;
                }
                else
                {
                        fallbackEntity();
                        return true;
                }
        }

        bool parseTokenRecover()
        {
                bool rt = false;
                if (tokstate.curchr_saved)
                {
                        push( tokstate.curchr_saved);
                        tokstate.curchr_saved = 0;
                }
                switch (tokstate.id)
                {
                        case TokState::Start:
                        case TokState::ParsingDone:
                        case TokState::ParsingKey:
                        case TokState::ParsingToken:
                                error = ErrInternal;
                                return false;
                        case TokState::ParsingEntity: rt = parseEntity(); break;
                        case TokState::ParsingNumericEntity: rt = parseNumericEntity(); break;
                        case TokState::ParsingNumericBaseEntity: rt = parseNumericBaseEntity(); break;
                        case TokState::ParsingNamedEntity: rt = parseNamedEntity(); break;
                }
                tokstate.init( TokState::ParsingToken);
                return rt;
        }

        bool parseToken( const IsTokenCharMap& isTok)
        {
                if (tokstate.id == TokState::Start)
                {
                        m_tokenpos = m_src.getPosition();
                        tokstate.id = TokState::ParsingToken;
                        m_outputBuf.clear();
                }
                else if (tokstate.id != TokState::ParsingToken)
                {
                        if (!parseTokenRecover())
                        {
                                tokstate.init();
                                return false;
                        }
                }
                for (;;)
                {
                        ControlCharacter ch;
                        while (isTok[ (unsigned char)(ch=m_src.control())])
                        {
                                unsigned char aa = m_src.ascii();
                                if (aa <= 0xD)
                                {
                                        //handling W3C requirements for end of line translation in XML:
                                        if (aa == '\r')
                                        {
                                                push( (unsigned char)'\n');
                                                tokstate.eolnState = TokState::CR;
                                        }
                                        else if (aa == '\n')
                                        {
                                                if (tokstate.eolnState != TokState::CR)
                                                {
                                                        push( (unsigned char)'\n');
                                                }
                                                tokstate.eolnState = TokState::SRC;
                                        }
                                        else
                                        {
                                                copychar();
                                                tokstate.eolnState = TokState::SRC;
                                        }
                                }
                                else
                                {
                                        copychar();
                                        tokstate.eolnState = TokState::SRC;
                                }
                                m_src.skip();
                        }
                        if (ch == Amp)
                        {
                                m_src.skip();
                                if (!parseEntity()) break;
                                tokstate.init( TokState::ParsingToken);
                                continue;
                        }
                        else
                        {
                                tokstate.init( TokState::ParsingDone);
                                return true;
                        }
                }
                tokstate.init();
                return false;
        }

public:
        template <class OutputBufferType>
        static bool parseStaticToken( const IsTokenCharMap& isTok, InputReader ir, OutputBufferType& buf)
        {
                static OutputCharSet output;
                buf.clear();
                for (;;)
                {
                        ControlCharacter ch;
                        for (;;)
                        {
                                UChar pc;
                                if (isTok[ (unsigned char)(ch=ir.control())])
                                {
                                        pc = ir.chr();
                                }
                                else if (ch == Amp)
                                {
                                        pc = parseStaticNumericEntityValue( ir);
                                }
                                else
                                {
                                        return true;
                                }
                                output.print( pc, buf);
                                ir.skip();
                        }
                }
        }

private:
        bool skipToken( const IsTokenCharMap& isTok)
        {
                do
                {
                        ControlCharacter ch;
                        while (isTok[ (unsigned char)(ch=m_src.control())] || ch == Amp)
                        {
                                m_src.skip();
                        }
                }
                while (m_src.control() == Any);
                return true;
        }

        bool expectStr( const char* str)
        {
                bool rt = true;
                tokstate.id = TokState::ParsingKey;
                for (; str[tokstate.pos] != '\0'; m_src.skip(),tokstate.pos++)
                {
                        if (m_src.ascii() == str[ tokstate.pos]) continue;
                        ControlCharacter ch = m_src.control();
                        if (ch == EndOfText)
                        {
                                error = ErrUnexpectedEndOfText;
                        }
                        else
                        {
                                error = ErrSyntaxToken;
                        }
                        rt = false;
                        break;
                }
                tokstate.init( TokState::ParsingDone);
                return rt;
        }

        bool pushPredefinedEntity( const char* str)
        {
                switch (str[0])
                {
                        case 'q':
                                if (str[1] == 'u' && str[2] == 'o' && str[3] == 't' && str[4] == '\0')
                                {
                                        push( '\"');
                                        return true;
                                }
                                break;

                        case 'a':
                                if (str[1] == 'm')
                                {
                                        if (str[2] == 'p' && str[3] == '\0')
                                        {
                                                push( '&');
                                                return true;
                                        }
                                }
                                else if (str[1] == 'p')
                                {
                                        if (str[2] == 'o' && str[3] == 's' && str[4] == '\0')
                                        {
                                                push( '\'');
                                                return true;
                                        }
                                }
                                break;

                        case 'l':
                                if (str[1] == 't' && str[2] == '\0')
                                {
                                        push( '<');
                                        return true;
                                }
                                break;

                        case 'g':
                                if (str[1] == 't' && str[2] == '\0')
                                {
                                        push( '>');
                                        return true;
                                }
                                break;

                        case 'n':
                                if (str[1] == 'b' && str[2] == 's' && str[3] == 'p' && str[4] == '\0')
                                {
                                        push( ' ');
                                        return true;
                                }
                                break;
                }
                return false;
        }

        bool pushEntity( const char* str)
        {
                if (pushPredefinedEntity( str))
                {
                        return true;
                }
                else if (m_entityMap)
                {
                        EntityMap::const_iterator itr = m_entityMap->find( str);
                        if (itr == m_entityMap->end())
                        {
                                error = ErrUndefinedCharacterEntity;
                                return false;
                        }
                        else
                        {
                                UChar ch = itr->second;
                                push( ch);
                                return true;
                        }
                }
                else
                {
                        error = ErrUndefinedCharacterEntity;
                        return false;
                }
        }

private:
        STMState state;                 
        Error error;                    
        InputReader m_src;              
        const EntityMap* m_entityMap;   
        OutputBuffer m_outputBuf;       
        OutputCharSet m_output;         
        std::size_t m_tokenpos;         

public:
        XMLScanner( const InputIterator& p_src, const EntityMap& p_entityMap)
                        :state(START),error(Ok),m_src(InputCharSet(),p_src),m_entityMap(&p_entityMap),m_output(OutputCharSet()),m_tokenpos(0)
        {}
        explicit XMLScanner( const InputIterator& p_src)
                        :state(START),error(Ok),m_src(InputCharSet(),p_src),m_entityMap(0),m_output(OutputCharSet()),m_tokenpos(0)
        {}
        XMLScanner( const InputCharSet& p_charset, const InputIterator& p_src, const EntityMap& p_entityMap)
                        :state(START),error(Ok),m_src(p_charset,p_src),m_entityMap(&p_entityMap),m_output(OutputCharSet()),m_tokenpos(0)
        {}
        XMLScanner( const InputCharSet& p_charset, const InputIterator& p_src)
                        :state(START),error(Ok),m_src(p_charset,p_src),m_entityMap(0),m_output(OutputCharSet()),m_tokenpos(0)
        {}
        explicit XMLScanner( const InputCharSet& p_charset)
                        :state(START),error(Ok),m_src(p_charset),m_entityMap(0),m_tokenpos(0)
        {}
        XMLScanner()
                        :state(START),error(Ok),m_src(InputCharSet()),m_entityMap(0),m_tokenpos(0)
        {}

        XMLScanner( const XMLScanner& o)
                :state(o.state)
                ,error(o.error)
                ,m_src(o.m_src)
                ,m_entityMap(o.m_entityMap)
                ,m_outputBuf(o.m_outputBuf)
                ,m_tokenpos(o.m_tokenpos)
        {}

        template <class IteratorAssignment>
        void setSource( const IteratorAssignment& a)
        {
                m_src.setSource( a);
        }

        std::size_t getPosition() const
        {
                return m_src.getPosition();
        }
        std::size_t getTokenPosition() const
        {
                return m_tokenpos;
        }

        const char* getItemPtr() const {return m_outputBuf.size()?&m_outputBuf.at(0):"\0\0\0\0";}

        std::size_t getItemSize() const {return m_outputBuf.size();}

        const OutputBuffer& getItem() const
        {
                return m_outputBuf;
        }

        ScannerStatemachine::Element* getState()
        {
                static Statemachine stm;
                return stm.get( state);
        }

        Error getError( const char** str=0)
        {
                Error rt = error;
                error = Ok;
                if (str) *str=getErrorString(rt);
                return rt;
        }

        const InputIterator& getIterator() const
        {
                return m_src.getIterator();
        }

        InputIterator& getIterator()
        {
                return m_src.getIterator();
        }

        ElementType nextItem( unsigned short mask=0xFFFF)
        {
                static const IsWordCharMap wordC;
                static const IsContentCharMap contentC;
                static const IsTagCharMap tagC;
                static const IsSQStringCharMap sqC;
                static const IsDQStringCharMap dqC;
                static const IsTokenCharMap* tokenDefs[ NofSTMActions] = {0,&wordC,&contentC,&tagC,&sqC,&dqC,0,0,0};
                static const char* stringDefs[ NofSTMActions] = {0,0,0,0,0,0,"xml","CDATA",0};

                ElementType rt = None;
                ControlCharacter ch;
                do
                {
                        ScannerStatemachine::Element* sd = getState();
                        if (sd->action.op != -1)
                        {
                                if (tokenDefs[sd->action.op])
                                {
                                        if (tokstate.id != TokState::ParsingDone)
                                        {
                                                if ((mask&(1<<sd->action.arg)) != 0)
                                                {
                                                        if (!parseToken( *tokenDefs[ sd->action.op])) return ErrorOccurred;
                                                }
                                                else
                                                {
                                                        if (!skipToken( *tokenDefs[ sd->action.op])) return ErrorOccurred;
                                                }
                                        }
                                        rt = (ElementType)sd->action.arg;
                                }
                                else if (stringDefs[sd->action.op])
                                {
                                        if (tokstate.id != TokState::ParsingDone)
                                        {
                                                if (!expectStr( stringDefs[sd->action.op])) return ErrorOccurred;
                                                if (sd->action.op == ExpectIdentifierXML)
                                                {
                                                        //... special treatement for xml header for not
                                                        //    enforcing the model too much just for this case
                                                        push( '?'); push( 'x'); push( 'm'); push( 'l');
                                                        rt = HeaderStart;
                                                }
                                        }
                                        else if (sd->action.op == ExpectIdentifierXML)
                                        {
                                                //... special treatement for xml header for not
                                                //    enforcing the model too much just for this case
                                                rt = HeaderStart;
                                        }
                                }
                                else
                                {
                                        m_tokenpos = m_src.getPosition();
                                        m_outputBuf.clear();
                                        rt = (ElementType)sd->action.arg;
                                }
                                if (sd->nofnext == 0)
                                {
                                        if (sd->fallbackState != -1)
                                        {
                                                state = (STMState)sd->fallbackState;
                                        }
                                        return rt;
                                }
                        }
                        ch = m_src.control();
                        tokstate.id = TokState::Start;

                        if (sd->next[ ch] != -1)
                        {
                                state = (STMState)sd->next[ ch];
                                m_src.skip();
                        }
                        else if (sd->fallbackState != -1)
                        {
                                state = (STMState)sd->fallbackState;
                        }
                        else if (sd->missError != -1)
                        {
                                error = (Error)sd->missError;
                                return ErrorOccurred;
                        }
                        else if (ch == EndOfText)
                        {
                                error = ErrUnexpectedEndOfText;
                                return ErrorOccurred;
                        }
                        else
                        {
                                error = ErrInternal;
                                return ErrorOccurred;
                        }
                }
                while (rt == None);
                return rt;
        }

        struct End {};

        class iterator
        {
        public:
                class Element
                {
                private:
                        friend class iterator;
                        ElementType m_type;             
                        const char* m_content;          
                        std::size_t m_size;             
                public:
                        bool valid() const              {return m_type != Exit && m_type != ErrorOccurred;}
                        const char* error() const       {return m_type == ErrorOccurred ? m_content : 0;}
                        const char* name() const        {return getElementTypeName( m_type);}
                        ElementType type() const        {return m_type;}
                        const char* content() const     {return m_content;}
                        std::size_t size() const        {return m_size;}
                        Element()                       :m_type(None),m_content(0),m_size(0) {}
                        Element( const End&)            :m_type(Exit),m_content(0),m_size(0) {}
                        Element( const Element& orig)   :m_type(orig.m_type),m_content(orig.m_content),m_size(orig.m_size) {}
                };
                // input iterator traits
                typedef Element value_type;
                typedef std::size_t difference_type;
                typedef std::size_t size_type;
                typedef Element* pointer;
                typedef Element& reference;
                typedef std::input_iterator_tag iterator_category;

        private:
                Element element;                                
                ThisXMLScanner* input;                          

                iterator& skip( unsigned short mask=0xFFFF)
                {
                        if (input != 0)
                        {
                                element.m_type = input->nextItem(mask);
                                element.m_content = input->getItemPtr();
                                element.m_size = input->getItemSize();
                        }
                        return *this;
                }

                bool compare( const iterator& iter) const
                {
                        if (element.type() == iter.element.type())
                        {
                                if (element.type() == Exit || element.type() == None) return true;  //equal only at beginning and end
                        }
                        return false;
                }
        public:
                void assign( const iterator& orig)
                {
                        input = orig.input;
                        element = orig.element;
                }
                iterator( const iterator& orig)
                {
                        assign( orig);
                }
                iterator( ThisXMLScanner& p_input, bool doSkipToFirst=true)
                                :input( &p_input)
                {
                        if (doSkipToFirst)
                        {
                                element.m_type = input->nextItem();
                                element.m_content = input->getItemPtr();
                                element.m_size = input->getItemSize();
                        }
                }
                iterator( const End& et)  :element(et),input(0) {}
                iterator()  :input(0) {}
                iterator& operator = (const iterator& orig)
                {
                        assign( orig);
                        return *this;
                }
                const Element& operator*() const
                {
                        return element;
                }
                const Element* operator->() const
                {
                        return &element;
                }
                iterator& operator++()                          {return skip();}
                iterator operator++(int)                        {iterator tmp(*this); skip(); return tmp;}

                bool operator==( const iterator& iter) const    {return compare( iter);}
                bool operator!=( const iterator& iter) const    {return !compare( iter);}
        };

        iterator begin( bool doSkipToFirst=true)
        {
                return iterator( *this, doSkipToFirst);
        }
        iterator end()
        {
                return iterator( End());
        }
};

}//namespace
#endif