strusAnalyzer  0.17
token.hpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2014 Patrick P. Frey
3  *
4  * This Source Code Form is subject to the terms of the Mozilla Public
5  * License, v. 2.0. If a copy of the MPL was not distributed with this
6  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7  */
10 #ifndef _STRUS_ANALYZER_TOKENIZER_TOKEN_HPP_INCLUDED
11 #define _STRUS_ANALYZER_TOKENIZER_TOKEN_HPP_INCLUDED
13 #include <string>
14 
16 namespace strus {
18 namespace analyzer {
19 
21 struct Token
22 {
23 public:
26  :m_ordpos(0),m_origsize(0),m_origpos(){}
31  Token( int ordpos_, const Position& origpos_, int origsize_)
32  :m_ordpos(ordpos_),m_origsize(origsize_),m_origpos(origpos_){}
34 #if __cplusplus >= 201103L
35  Token( Token&& ) = default;
36  Token( const Token& ) = default;
37  Token& operator= ( Token&& ) = default;
38  Token& operator= ( const Token& ) = default;
39 #else
40  Token( const Token& o)
41  :m_ordpos(o.m_ordpos),m_origsize(o.m_origsize),m_origpos(o.m_origpos){}
42 #endif
43 
45  int ordpos() const {return m_ordpos;}
47  const Position& origpos() const {return m_origpos;}
49  Position& origpos() {return m_origpos;}
51  int origsize() const {return m_origsize;}
52 
55  void setOrigPosition( const Position& origpos_)
56  {
57  m_origpos = origpos_;
58  }
59 
62  void setOrigSize( int origsize_)
63  {
64  m_origsize = origsize_ > 0 ? (uint32_t)origsize_ : 0;
65  }
66 
69  void setOrdpos( int ordpos_)
70  {
71  m_ordpos = ordpos_;
72  }
73 
75  int compare( const Token& o) const
76  {
77  int rt = m_origpos.compare( o.m_origpos);
78  if (rt) return rt;
79  return ((m_origsize < o.m_origsize) - (m_origsize > o.m_origsize));
80  }
82  bool operator < (const Token& o) const
83  {
84  int cmp = m_origpos.compare( o.m_origpos);
85  return (cmp == 0) ? (m_origsize < o.m_origsize) : cmp < 0;
86  }
87 
88 private:
89  uint32_t m_ordpos;
90  uint32_t m_origsize;
91  Position m_origpos;
92 };
93 
94 }}//namespace
95 #endif
96 
void setOrigPosition(const Position &origpos_)
Set the original segment index of the token in the source.
Definition: token.hpp:55
Token(const Token &o)
Copy constructor.
Definition: token.hpp:40
int ordpos() const
Get the ordinal (counting) position in the document. This value is used to assign the term position...
Definition: token.hpp:45
Position & origpos()
Get the start position of this token in the original document.
Definition: token.hpp:49
int compare(const Token &o) const
Compare with another token.
Definition: token.hpp:75
bool operator<(const Token &o) const
Compare operator for sort.
Definition: token.hpp:82
int origsize() const
Get the byte size of the translated token string (UTF-8) in the original document segment...
Definition: token.hpp:51
int compare(const Position &o) const
Compare with another position.
Definition: position.hpp:61
void setOrigSize(int origsize_)
Set the original size of the token in the source.
Definition: token.hpp:62
Token()
Default constructor.
Definition: token.hpp:25
Structure describing a position in a document source by segment and offset.
Token(int ordpos_, const Position &origpos_, int origsize_)
Constructor.
Definition: token.hpp:31
Structure describing a position in a document source by segment and offset.
Definition: position.hpp:22
Structure describing a token in the document by its start and end position.
Definition: token.hpp:21
void setOrdpos(int ordpos_)
Set the ordinal position of the token in the source (adjusted in case of multiple segments) ...
Definition: token.hpp:69
const Position & origpos() const
Get the start position of this token in the original document.
Definition: token.hpp:47