tokenizer.h

#pragma once
 
#include <vector>
 
#include "token.h"
 
#include "lib_begin.h"
 
namespace cgv {
    namespace utils {
 
class CGV_API tokenizer : public token
{
protected:
    std::string separators;
    bool merge_separators;
    std::string begin_skip;
    std::string end_skip;
    std::string escape_skip;
    std::string whitespaces;
    void init();
    bool handle_skip(token& result);
    bool handle_separators(token& result,bool check_skip=true);
    bool reverse_handle_skip(token& result);
    bool reverse_handle_separators(token& result,bool check_skip=true);
public:
    tokenizer();
    tokenizer(const token&);
    tokenizer(const char*);
    tokenizer(const std::string&);
    tokenizer& set_ws(const std::string& ws);
    tokenizer& set_skip(const std::string& open, const std::string& close);
    tokenizer& set_skip(const std::string& open, const std::string& close, const std::string& escape);
    tokenizer& set_sep(const std::string& sep, bool merge);
    tokenizer& set_sep(const std::string& sep);
    tokenizer& set_sep_merge(bool merge);
    token bite();
    token reverse_bite();
    void reverse_skip_whitespaces();
    void skip_whitespaces();
    bool skip_ws_check_empty() { skip_whitespaces(); return empty(); }
    bool reverse_skip_ws_check_empty() { reverse_skip_whitespaces(); return empty(); }
    bool balanced_bite(token& result, const std::string& open_parenthesis, const std::string& close_parenthesis, bool wait_for_sep = false);
    void bite_all(std::vector<token>& result);
};
 
inline void bite_all(tokenizer& t, std::vector<token>& result) { while(!t.skip_ws_check_empty())result.push_back(t.bite()); }
 
    }
}
 
#include <cgv/config/lib_end.h>