|
Home / Open source / Terimber 2.0
tokenizer.cpp File Reference#include "tokenizer/tokenizer.h"
#include "base/list.hpp"
#include "base/map.hpp"
#include "base/string.hpp"
#include "base/common.hpp"
#include "base/memory.hpp"
#include "xml/defxml.hpp"
Go to the source code of this file.
Function Documentation
| static token_type detect_token_type |
( |
char |
symbol |
) |
[inline, static] |
Definition at line 52 of file tokenizer.cpp.
References ch_0, ch_1, ch_2, ch_3, ch_4, ch_5, ch_6, ch_7, ch_8, ch_9, ch_ampersand, ch_asterisk, ch_at, ch_back_slash, ch_bang, ch_caret, ch_close_angle, ch_close_curly, ch_close_paren, ch_close_square, ch_colon, ch_comma, ch_dash, ch_dollar_sign, ch_double_quote, ch_equal, ch_ff, ch_forward_slash, ch_grave, ch_hor_tab, ch_lf, ch_open_angle, ch_open_curly, ch_open_paren, ch_open_square, ch_percent, ch_period, ch_pipe, ch_plus, ch_pound, ch_question, ch_semicolon, ch_single_quote, ch_space, ch_tilde, ch_underscore, ch_ver_tab, TT_ALPHABETIC, TT_DIGIT, TT_DOT, TT_PUNCTUATION, TT_SYMBOL, and TT_WHITESPACE.
Referenced by tokenizer::tokenize().
| static BEGIN_TERIMBER_NAMESPACE size_t do_hash_lowercase |
( |
const char * |
x, |
|
|
size_t |
len | |
|
) |
| | [inline, static] |
|
|
