blob: d329aa58bdbc73dea56fa8d38f57950413c5ed81 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
|
/* ANTLR Translator Generator
* Project led by Terence Parr at http://www.jGuru.com
* Software rights: http://www.antlr.org/license.html
*
* $Id$
*/
#include "antlr/TokenBuffer.h"
#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif
/**A Stream of Token objects fed to the parser from a TokenStream that can
* be rewound via mark()/rewind() methods.
* <p>
* A dynamic array is used to buffer up all the input tokens. Normally,
* "k" tokens are stored in the buffer. More tokens may be stored during
* guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
* Consumption of tokens is deferred. In other words, reading the next
* token is not done by conume(), but deferred until needed by LA or LT.
* <p>
*
* @see antlr.Token
* @see antlr.TokenStream
* @see antlr.TokenQueue
*/
/** Create a token buffer */
TokenBuffer::TokenBuffer( TokenStream& inp )
: input(inp)
, nMarkers(0)
, markerOffset(0)
, numToConsume(0)
{
}
TokenBuffer::~TokenBuffer( void )
{
}
/** Ensure that the token buffer is sufficiently full */
void TokenBuffer::fill(unsigned int amount)
{
syncConsume();
// Fill the buffer sufficiently to hold needed tokens
while (queue.entries() < (amount + markerOffset))
{
// Append the next token
queue.append(input.nextToken());
}
}
/** Get a lookahead token value */
int TokenBuffer::LA(unsigned int i)
{
fill(i);
return queue.elementAt(markerOffset+i-1)->getType();
}
/** Get a lookahead token */
RefToken TokenBuffer::LT(unsigned int i)
{
fill(i);
return queue.elementAt(markerOffset+i-1);
}
/** Return an integer marker that can be used to rewind the buffer to
* its current state.
*/
unsigned int TokenBuffer::mark()
{
syncConsume();
nMarkers++;
return markerOffset;
}
/**Rewind the token buffer to a marker.
* @param mark Marker returned previously from mark()
*/
void TokenBuffer::rewind(unsigned int mark)
{
syncConsume();
markerOffset=mark;
nMarkers--;
}
/// Get number of non-consumed tokens
unsigned int TokenBuffer::entries() const
{
return queue.entries() - markerOffset;
}
#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif
|
