Thread portable class Gate portable class Timer class Pool class Terimber 2.0 About C++

Downloads Products & Services Support Clients Open Source About Home / Open source / Terimber 2.0 tokenizer.h Go to the documentation of this file. /* * The Software License * ================================================================================= * Copyright (c) 2003-.The Terimber Corporation. All rights reserved. * ================================================================================= * Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * The end-user documentation included with the redistribution, if any, * must include the following acknowledgment: * "This product includes software developed by the Terimber Corporation." * ================================================================================= * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESSED OR IMPLIED WARRANTIES, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE TERIMBER CORPORATION OR ITS CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * ================================================================================ */ #ifndef _terimber_tokenizer_h_ #define _terimber_tokenizer_h_ #include "allinc.h" #include "base/list.h" #include "base/map.h" #include "base/string.h" #include "pcre/pcre.h" BEGIN_TERIMBER_NAMESPACE #pragma pack(4) const size_t T_REGEX = 0x00000001; const size_t T_ABBR = 0x00000002; const size_t T_HYPHEN = 0x00000004; const size_t T_ALL = T_REGEX | T_ABBR | T_HYPHEN; enum token_type { TT_UNKNOWN = 0, TT_REGEX, TT_ABBR, TT_COMPOSE, TT_ALPHABETIC, TT_DIGIT, TT_WHITESPACE, TT_SYMBOL, TT_PUNCTUATION, TT_DOT }; class token_info { public: token_info( token_type type, size_t len, size_t key ) : _type(type), _len(len), _key(key) { } token_type _type; size_t _len; size_t _key; }; class pcre_key { public: pcre_key( size_t min, size_t max ) : _min(min), _max(max) { } size_t _min; size_t _max; inline bool operator<(const pcre_key& x) const { return _max < x._max; } }; class pcre_entry { public: pcre_entry( pcre* th, pcre_extra* ex, size_t key ) : _this(th), _extra(ex), _key(key) {} pcre* _this; pcre_extra* _extra; size_t _key; }; typedef _list< token_info > tokenizer_output_sequence_t; typedef map< size_t, string_t, less< size_t >, true > abbreviation_map_t; typedef map< pcre_key, pcre_entry, less< pcre_key >, true > regex_map_t; class tokenizer { public: tokenizer(); ~tokenizer(); bool add_regex( const char* regex, size_t key, size_t min, size_t max ); bool load_regex( const char* file_name ); void clear_regex(); bool add_abbreviation(const char* abbr ); bool load_abbr( const char* file_name ); void clear_abbr(); bool tokenize( const char* str, tokenizer_output_sequence_t& out, byte_allocator& all, size_t flags = T_ALL ) const; const string_t& get_last_error() const { return _error; } private: void clear(); void do_regex( const char* phrase, tokenizer_output_sequence_t& tokens ) const; void do_abbr( const char* phrase, tokenizer_output_sequence_t& tokens ) const; void do_hyphen( const char* phrase, tokenizer_output_sequence_t& tokens ) const; size_t match( const char* x, size_t len, size_t tokens, size_t& key ) const; private: string_t _error; abbreviation_map_t _abbr_map; regex_map_t _regex_map; }; #pragma pack() END_TERIMBER_NAMESPACE #endif

© Copyright Terimber 2003-.