mirror of
https://github.com/boostorg/tokenizer.git
synced 2026-01-23 06:02:15 +00:00
205 lines
5.4 KiB
HTML
205 lines
5.4 KiB
HTML
<html>
|
|
|
|
<head>
|
|
<meta http-equiv="Content-Type"
|
|
content="text/html; charset=iso-8859-1">
|
|
<meta name="GENERATOR" content="Microsoft FrontPage Express 2.0">
|
|
<title>Boost Tokenizer Class</title>
|
|
<!--
|
|
-- Copyright © John Bandela 2001
|
|
--
|
|
-- Permission to use, copy, modify, distribute and sell this software
|
|
-- and its documentation for any purpose is hereby granted without fee,
|
|
-- provided that the above copyright notice appears in all copies and
|
|
-- that both that copyright notice and this permission notice appear
|
|
-- in supporting documentation. Jeremy Siek makes no
|
|
-- representations about the suitability of this software for any
|
|
-- purpose. It is provided "as is" without express or implied warranty.
|
|
-->
|
|
</head>
|
|
|
|
<body bgcolor="#FFFFFF" text="#000000" link="#0000EE"
|
|
vlink="#551A8B" alink="#FF0000">
|
|
|
|
<p><img src="../../c++boost.gif" alt="C++ Boost" width="277"
|
|
height="86"> <br>
|
|
</p>
|
|
|
|
<h1 align="center">Tokenizer Class</h1>
|
|
|
|
<pre> template <
|
|
class TokenizerFunc = char_delimiters_separator<char>,
|
|
class Iterator = std::string::const_iterator,
|
|
class Type = std::string
|
|
>
|
|
class tokenizer
|
|
</pre>
|
|
|
|
<p>The tokenizer class provides a container view of a series of
|
|
tokens contained in a sequence. You set the sequence to parse and
|
|
the TokenizerFunction to use to parse the sequence either upon
|
|
construction or using the assign member function. Note: No
|
|
parsing is actually done upon construction. Parsing is done on
|
|
demand as the tokens are accessed via the iterator provided by
|
|
begin.</p>
|
|
|
|
<h2>Example</h2>
|
|
|
|
<pre>// simple_example_1.cpp
|
|
#include<iostream>
|
|
#include<boost/tokenizer.hpp>
|
|
#include<string>
|
|
|
|
int main(){
|
|
using namespace std;
|
|
using namespace boost;
|
|
string s = "This is, a test";
|
|
tokenizer<> tok(s);
|
|
for(tokenizer<>::iterator beg=tok.begin(); beg!=tok.end();++beg){
|
|
cout << *beg << "\n";
|
|
}
|
|
}
|
|
</pre>
|
|
|
|
<p> </p>
|
|
|
|
<h3>Template Parameters</h3>
|
|
|
|
<table border="1">
|
|
<tr>
|
|
<th>Parameter</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
<tr>
|
|
<td><tt>TokenizerFunc</tt></td>
|
|
<td>The TokenizerFunction used to parse the sequence.</td>
|
|
</tr>
|
|
<tr>
|
|
<td><tt>Iterator</tt></td>
|
|
<td>The type of the iterator the specifies the sequence.</td>
|
|
</tr>
|
|
<tr>
|
|
<td><tt>Type</tt></td>
|
|
<td>The type of the token, typically string.</td>
|
|
</tr>
|
|
</table>
|
|
|
|
<p> </p>
|
|
|
|
<h2>Related Types</h2>
|
|
|
|
<table border="1">
|
|
<tr>
|
|
<td><p align="center"><strong>Type</strong></p>
|
|
</td>
|
|
<td><p align="center"><strong>Remarks</strong></p>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td>iterator</td>
|
|
<td>The type returned by begin and end. Note: the
|
|
category of iterator will be at most ForwardIterator. It
|
|
will be InputIterator if the Iterator template parameter
|
|
is an InputIterator. For any other category, it will be
|
|
ForwardIterator.</td>
|
|
</tr>
|
|
<tr>
|
|
<td>const_iterator</td>
|
|
<td>Same type as iterator.</td>
|
|
</tr>
|
|
<tr>
|
|
<td>value_type</td>
|
|
<td>Same type as the template parameter Type</td>
|
|
</tr>
|
|
<tr>
|
|
<td>reference</td>
|
|
<td>Same type as value_type&</td>
|
|
</tr>
|
|
<tr>
|
|
<td>const_reference</td>
|
|
<td>Same type as const reference</td>
|
|
</tr>
|
|
<tr>
|
|
<td>pointer</td>
|
|
<td>Same type as value_type*</td>
|
|
</tr>
|
|
<tr>
|
|
<td>const_pointer</td>
|
|
<td>Same type as const pointer</td>
|
|
</tr>
|
|
<tr>
|
|
<td>size_type</td>
|
|
<td>void</td>
|
|
</tr>
|
|
<tr>
|
|
<td>difference_type</td>
|
|
<td>void</td>
|
|
</tr>
|
|
</table>
|
|
|
|
<p> </p>
|
|
|
|
<h2>Construction and Member Functions</h2>
|
|
|
|
<pre>tokenizer(Iterator first, Iterator last,const TokenizerFunc& f = TokenizerFunc())
|
|
|
|
template<class Container>
|
|
tokenizer(const Container& c,const TokenizerFunc& f = TokenizerFunc())
|
|
|
|
void assign(Iterator first, Iterator last)
|
|
|
|
void assign(Iterator first, Iterator last, const TokenizerFunc& f)
|
|
|
|
template<class Container>
|
|
void assign(const Container& c)
|
|
|
|
template<class Container>
|
|
void assign(const Container& c, const TokenizerFunc& f)
|
|
|
|
iterator begin() const
|
|
|
|
iterator end() const
|
|
</pre>
|
|
|
|
<table border="1">
|
|
<tr>
|
|
<td><p align="center"><strong>Parameter</strong></p>
|
|
</td>
|
|
<td><p align="center"><strong>Description</strong></p>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td>c</td>
|
|
<td>A container that contains the sequence to parse. Note:
|
|
c.begin() and c.end() must be convertible to the template
|
|
parameter Iterator.</td>
|
|
</tr>
|
|
<tr>
|
|
<td>f</td>
|
|
<td>A functor that is a model of TokenizerFunction that
|
|
will be used to parse the sequence.</td>
|
|
</tr>
|
|
<tr>
|
|
<td>first</td>
|
|
<td>The iterator that represents the beginning position
|
|
in the sequence to be parsed.</td>
|
|
</tr>
|
|
<tr>
|
|
<td>last</td>
|
|
<td>The iterator that represents the past the end
|
|
position in the sequence to be parsed.</td>
|
|
</tr>
|
|
</table>
|
|
|
|
<p> </p>
|
|
|
|
<hr>
|
|
|
|
<p>© Copyright John R. Bandela 2001. Permission to copy, use,
|
|
modify, sell and distribute this document is granted provided
|
|
this copyright notice appears in all copies. This document is
|
|
provided "as is" without express or implied warranty,
|
|
and with no claim as to its suitability for any purpose.</p>
|
|
</body>
|
|
</html>
|