com.metaweb.lessen.tokenizers
Class TokenizerBase

java.lang.Object
  com.metaweb.lessen.tokenizers.TokenizerBase

All Implemented Interfaces:

Tokenizer

Direct Known Subclasses:

ReaderTokenizer, StringTokenizer

public abstract class TokenizerBase
extends Object
implements Tokenizer

Field Summary

protected Token

_token

Constructor Summary

TokenizerBase()

Method Summary

protected abstract void	advance()
protected void	advance(int by)
protected abstract void	advance(int by, boolean flush)
protected abstract void	flush(int upToIndex)
protected abstract char	getCharRelative(int offset)
protected abstract char	getCurrentChar()
protected abstract int	getCurrentOffset()
protected abstract String	getText(int from, int to)
Token	getToken()
protected abstract boolean	hasMoreChar()
protected abstract boolean	hasMoreChar(int offset)
protected static boolean	isHexDigit(char c)
protected static boolean	isNameChar(char c)
protected static boolean	isNameStartChar(char c)
protected static boolean	isWhitespace(char c)
protected int	lookOverIdentifier(int offset)
protected int	lookOverName(int offset)
protected int	lookOverNameChar(int offset)
protected int	lookOverNameStartChar(int offset)
protected int	lookOverUnicodeChar(int offset)
protected int	lookOverUnicodeCode(int offset)
protected int	lookOverWhitespace(int index)
void	next()
protected void	parseNumber(int tokenStart, int signum)
protected void	parseRestOfURI(int tokenStart, String prefix)
protected String	parseString(char delimiter)
protected void	swallowWhitespace(boolean flush)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

_token

protected Token _token

Constructor Detail

TokenizerBase

public TokenizerBase()

Method Detail

hasMoreChar

protected abstract boolean hasMoreChar()

hasMoreChar

protected abstract boolean hasMoreChar(int offset)

getCurrentChar

protected abstract char getCurrentChar()

getCharRelative

protected abstract char getCharRelative(int offset)

getCurrentOffset

protected abstract int getCurrentOffset()

advance

protected abstract void advance()

advance

protected abstract void advance(int by,
                                boolean flush)

advance

protected void advance(int by)

getText

protected abstract String getText(int from,
                                  int to)

flush

protected abstract void flush(int upToIndex)

getToken

public Token getToken()

Specified by:

getToken in interface Tokenizer

next

public void next()

Specified by:

next in interface Tokenizer

parseNumber

protected void parseNumber(int tokenStart,
                           int signum)

parseRestOfURI

protected void parseRestOfURI(int tokenStart,
                              String prefix)

parseString

protected String parseString(char delimiter)

swallowWhitespace

protected void swallowWhitespace(boolean flush)

lookOverWhitespace

protected int lookOverWhitespace(int index)

lookOverIdentifier

protected int lookOverIdentifier(int offset)

lookOverName

protected int lookOverName(int offset)

lookOverNameStartChar

protected int lookOverNameStartChar(int offset)

lookOverNameChar

protected int lookOverNameChar(int offset)

lookOverUnicodeChar

protected int lookOverUnicodeChar(int offset)

lookOverUnicodeCode

protected int lookOverUnicodeCode(int offset)

isWhitespace

protected static boolean isWhitespace(char c)

isHexDigit

protected static boolean isHexDigit(char c)

isNameChar

protected static boolean isNameChar(char c)

isNameStartChar

protected static boolean isNameStartChar(char c)