7 #ifndef STANDARDTOKENIZERIMPL_H 8 #define STANDARDTOKENIZERIMPL_H 36 static const wchar_t*
ZZ_CMAP();
162 void yybegin(int32_t newState);
static const int32_t YYEOF
This character denotes the end of file.
Definition: StandardTokenizerImpl.h:129
int32_t zzLexicalState
The current lexical state.
Definition: StandardTokenizerImpl.h:92
static const wchar_t * ZZ_CMAP()
virtual ~StandardTokenizerImpl()
static void ZZ_ACTION_INIT()
Translates DFA states to action switch labels.
int32_t yylength()
Returns the length of the matched text region.
bool zzAtBOL
zzAtBOL == true if the scanner is currently at the beginning of a line
Definition: StandardTokenizerImpl.h:122
boost::shared_ptr< Reader > ReaderPtr
Definition: LuceneTypes.h:547
static IntArray _ZZ_TRANS
The transition table of the DFA.
Definition: StandardTokenizerImpl.h:59
static const int32_t ZZ_ACTION_LENGTH
Definition: StandardTokenizerImpl.h:41
static CharArray _ZZ_CMAP
Translates characters to character classes.
Definition: StandardTokenizerImpl.h:29
void yybegin(int32_t newState)
Enters a new lexical state.
static const int32_t ZZ_TRANS_LENGTH
Definition: StandardTokenizerImpl.h:61
static const int32_t * ZZ_ACTION()
static const int32_t * ZZ_ROWMAP()
int32_t yyline
Number of newlines encountered up to the start of the matched text.
Definition: StandardTokenizerImpl.h:113
bool zzRefill()
Refills the input buffer.
static const int32_t ZZ_NO_MATCH
Definition: StandardTokenizerImpl.h:70
void yyreset(const ReaderPtr &reader)
Resets the scanner to read from a new input stream. Does not close the old reader.
boost::shared_ptr< Token > TokenPtr
Definition: LuceneTypes.h:59
boost::shared_ptr< TermAttribute > TermAttributePtr
Definition: LuceneTypes.h:58
static const int32_t YYINITIAL
Lexical states.
Definition: StandardTokenizerImpl.h:132
int32_t zzMarkedPos
The text position at the last accepting state.
Definition: StandardTokenizerImpl.h:98
static const int32_t ZZ_ATTRIBUTE_LENGTH
Definition: StandardTokenizerImpl.h:78
StandardTokenizerImpl(const ReaderPtr &in)
Creates a new scanner.
static const int32_t ZZ_CMAP_LENGTH
Definition: StandardTokenizerImpl.h:31
static const int32_t ZZ_CMAP_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:32
CharArray zzBuffer
This buffer contains the current text to be matched and is the source of the yytext() string...
Definition: StandardTokenizerImpl.h:95
int32_t zzStartRead
StartRead marks the beginning of the yytext() string in the buffer.
Definition: StandardTokenizerImpl.h:107
static const int32_t * ZZ_TRANS()
static IntArray _ZZ_ROWMAP
Translates a state to a row index in the transition table.
Definition: StandardTokenizerImpl.h:49
static void ZZ_ROWMAP_INIT()
Translates a state to a row index in the transition table.
int32_t getNextToken()
Resumes scanning until the next regular expression is matched, the end of input is encountered or an ...
static const int32_t ZZ_PUSHBACK_2BIG
Definition: StandardTokenizerImpl.h:71
static void ZZ_TRANS_INIT()
The transition table of the DFA.
Base class for all Lucene classes.
Definition: LuceneObject.h:31
static const wchar_t * ZZ_ERROR_MSG[]
Definition: StandardTokenizerImpl.h:73
int32_t yystate()
Returns the current lexical state.
#define LUCENE_CLASS(Name)
Definition: LuceneObject.h:24
Definition: AbstractAllTermDocs.h:12
void reset(const ReaderPtr &r)
Resets the Tokenizer to a new Reader.
wchar_t yycharat(int32_t pos)
Returns the character at position pos from the matched text.
static void ZZ_CMAP_INIT()
Translates characters to character classes.
static const int32_t ZZ_UNKNOWN_ERROR
Definition: StandardTokenizerImpl.h:69
Definition: StandardTokenizerImpl.h:14
static const wchar_t ZZ_TRANS_PACKED_0[]
Definition: StandardTokenizerImpl.h:60
ReaderPtr zzReader
The input device.
Definition: StandardTokenizerImpl.h:86
static const int32_t ZZ_ROWMAP_LENGTH
Definition: StandardTokenizerImpl.h:51
int32_t zzPushbackPos
The text position at the last state to be included in yytext.
Definition: StandardTokenizerImpl.h:101
void yypushback(int32_t number)
Pushes the specified amount of characters back into the input stream.
int32_t zzState
The current state of the DFA.
Definition: StandardTokenizerImpl.h:89
bool zzAtEOF
zzAtEOF == true if the scanner is at the EOF
Definition: StandardTokenizerImpl.h:125
static const int32_t ZZ_BUFFERSIZE
Initial size of the lookahead buffer.
Definition: StandardTokenizerImpl.h:22
static const int32_t ZZ_TRANS_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:62
static const int32_t * ZZ_ATTRIBUTE()
int32_t yycolumn
The number of characters from the last newline up to the start of the matched text.
Definition: StandardTokenizerImpl.h:119
static const int32_t ZZ_ROWMAP_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:52
void zzScanError(int32_t errorCode)
Reports an error that occurred while scanning.
String yytext()
Returns the text matched by the current regular expression.
static void ZZ_ATTRIBUTE_INIT()
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
void getText(const TokenPtr &t)
Fills Lucene token with the current token text.
static const int32_t ZZ_ATTRIBUTE_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:79
int32_t _yychar
The number of characters up to the start of the matched text.
Definition: StandardTokenizerImpl.h:116
static const int32_t ZZ_ACTION_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:42
static IntArray _ZZ_ATTRIBUTE
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
Definition: StandardTokenizerImpl.h:76
static const wchar_t ZZ_ROWMAP_PACKED_0[]
Definition: StandardTokenizerImpl.h:50
int32_t zzCurrentPos
The current text position in the buffer.
Definition: StandardTokenizerImpl.h:104
static const wchar_t ZZ_CMAP_PACKED[]
Definition: StandardTokenizerImpl.h:30
static const wchar_t ZZ_ACTION_PACKED_0[]
Definition: StandardTokenizerImpl.h:40
static IntArray _ZZ_ACTION
Translates DFA states to action switch labels.
Definition: StandardTokenizerImpl.h:39
int32_t zzEndRead
EndRead marks the last character in the buffer, that has been read from input.
Definition: StandardTokenizerImpl.h:110
void yyclose()
Closes the input stream.
static const wchar_t ZZ_ATTRIBUTE_PACKED_0[]
Definition: StandardTokenizerImpl.h:77