10 #ifndef _STRUS_ANALYZER_TOKENIZER_TOKEN_HPP_INCLUDED
11 #define _STRUS_ANALYZER_TOKENIZER_TOKEN_HPP_INCLUDED
26 :m_ordpos(0),m_origsize(0),m_origpos(){}
32 :m_ordpos(ordpos_),m_origsize(origsize_),m_origpos(origpos_){}
34 #if __cplusplus >= 201103L
41 :m_ordpos(o.m_ordpos),m_origsize(o.m_origsize),m_origpos(o.m_origpos){}
45 int ordpos()
const {
return m_ordpos;}
64 m_origsize = origsize_ > 0 ? (uint32_t)origsize_ : 0;
77 int rt = m_origpos.
compare( o.m_origpos);
79 return ((m_origsize < o.m_origsize) - (m_origsize > o.m_origsize));
84 int cmp = m_origpos.
compare( o.m_origpos);
85 return (cmp == 0) ? (m_origsize < o.m_origsize) : cmp < 0;
void setOrigPosition(const Position &origpos_)
Set the original segment index of the token in the source.
Definition: token.hpp:55
Token(const Token &o)
Copy constructor.
Definition: token.hpp:40
int ordpos() const
Get the ordinal (counting) position in the document. This value is used to assign the term position...
Definition: token.hpp:45
Position & origpos()
Get the start position of this token in the original document.
Definition: token.hpp:49
int compare(const Token &o) const
Compare with another token.
Definition: token.hpp:75
bool operator<(const Token &o) const
Compare operator for sort.
Definition: token.hpp:82
int origsize() const
Get the byte size of the translated token string (UTF-8) in the original document segment...
Definition: token.hpp:51
int compare(const Position &o) const
Compare with another position.
Definition: position.hpp:61
void setOrigSize(int origsize_)
Set the original size of the token in the source.
Definition: token.hpp:62
Token()
Default constructor.
Definition: token.hpp:25
Structure describing a position in a document source by segment and offset.
Token(int ordpos_, const Position &origpos_, int origsize_)
Constructor.
Definition: token.hpp:31
Structure describing a position in a document source by segment and offset.
Definition: position.hpp:22
Structure describing a token in the document by its start and end position.
Definition: token.hpp:21
void setOrdpos(int ordpos_)
Set the ordinal position of the token in the source (adjusted in case of multiple segments) ...
Definition: token.hpp:69
const Position & origpos() const
Get the start position of this token in the original document.
Definition: token.hpp:47