26 #include "../tokenizer/reverseTokenizer.h"
36 const std::string tChars,
37 const std::string bChars,
38 const std::string cChars,
41 separatorChars (tChars),
42 blankspaceChars(bChars),
43 controlChars (cChars),
44 lowercase_mode (lowercase)
103 if (prev_context.empty()) {
104 if (curr_context.empty()) {
117 std::string::size_type ctx_idx = curr_context.rfind(prev_context);
119 if (ctx_idx == std::string::npos) {
132 std::string remainder = curr_context.substr(ctx_idx + prev_context.size());
134 std::string::size_type idx = remainder.find_last_of(
wordChars);
135 if (idx == std::string::npos) {
136 if (remainder.empty()) {
139 char last_char = curr_context[ctx_idx + prev_context.size() - 1];
141 if (idx == std::string::npos) {
148 if (idx == remainder.size() - 1) {
206 const std::string& curr_context = past_stream;
211 result = past_stream;
216 std::string::size_type ctx_idx = curr_context.rfind(prev_context);
218 if (ctx_idx == std::string::npos) {
221 result = past_stream;
231 result = curr_context.substr(ctx_idx + prev_context.size());
248 std::stringstream sliding_window_stream;
254 std::string first_token = rTok.
nextToken();
255 if (!first_token.empty()) {
256 result = first_token + result;