HOWTO 파서를 여러 번 사용 하시겠습니까?

Question

SQLParser.h :

class SQLParser{ 
/*____Variables____*/ 
private: 
    std::string _vendor; 
    antlr4::CommonTokenStream* _tokenStream; 
    antlr4::Parser* _parser; 
    antlr4::Lexer* _lexer; 

/*____Functions____*/ 
public: 
    SQLParser(const std::string& Vendor); 
    ~SQLParser(); 
    antlr4::CommonTokenStream* get_tokens(const std::string& text); 
    std::vector<std::string> get_lexems(const std::string& text); 
    antlr4::ParserRuleContext* parse(const std::string& text); 
    bool check_syntax(const std::string& text); 
    void print_string_tree(const std::string& text); // parse and print in LISP format 
}; 

SQLParser.cpp :

... 
CommonTokenStream* SQLParser::get_tokens(const std::string& text){ 
    (dynamic_cast<ANTLRInputStream*>(_lexer->getInputStream()))->load(text); 
    _tokenStream->reset(); 
    _tokenStream->fill(); 
    return _tokenStream; 
} 

std::vector<std::string> SQLParser::get_lexems(const std::string& text){ 
    get_tokens(text); 

    std::vector<std::string> lexems; 
    for(auto token : _tokenStream->getTokens()) { 
     lexems.push_back(token->getText()); 
    } 
    return lexems; 
} 

ParserRuleContext* SQLParser::parse(const std::string& text){ 
    get_tokens(text); 

    _parser->setInputStream(_tokenStream); 
    ParserRuleContext* tree; 
    try{ 
     if(_vendor == "tsql"){ 
      tree = (dynamic_cast<tsqlParser*>(_parser))->root(); 
     } 
     if(_vendor == "mysql"){ 
      tree = (dynamic_cast<mysqlParser*>(_parser))->root(); 
     } 
    } 
    catch(std::_Nested_exception<ParseCancellationException>& e){ 
     return nullptr; 
    } 
    return tree; 
} 

목적 SQLParser 각 콘크리트 벤더 생성된다. 이 개체를 사용하여 여러 입력 텍스트을 구문 분석하고 싶습니다. 하지만 TokenStream 크기에 문제가 있습니다. 나는 그 크기가 동적으로 변할 것이라고 예상했다. 이 같은 예를 들어 , 주요 : MAIN.CPP이 같은

#include <iostream> 
#include <string> 

#include <antlr4-runtime.h> 
#include "SQLParser.h" 

using namespace antlr4; 

int main(){ 
    SQLParser parser("tsql"); 
    std::cout << "'select 1;': "; 
    parser.print_string_tree("select 1;"); 
    std::cout << "\n\n'select 1,2,3;': "; 
    parser.print_string_tree("select 1,2;"); 
    std::cout << "\n"; 
    return 0; 
} 

주고 출력 :

'select 1;': (root (sql_clauses (sql_clause (dml_clause (select_statement (query_expression (query_specification select (select_list (select_list_elem (expression (constant 1)))))) ;)))) <EOF>) 

'select 1,2,3;': (root (sql_clauses (sql_clause (dml_clause (select_statement (query_expression (query_specification select (select_list (select_list_elem (expression (constant 1)))))) ,))))) 

어떻게 TokenStream이 오류를 방지하기 위해 사용해야합니까?

Answer 1

비슷한 설정이 나와 있습니다. 컨텍스트 클래스는 전체적으로 동작하는 lexer + parser + listener 등을 함께 유지합니다. 새 입력으로 구문 분석을 다시 시작하려면 토큰 스트림에서 모든 토큰을 다시로드해야합니다. 내 상황에 맞는 클래스에서 나는 그래서 그것을 :

struct MySQLParserContextImpl : public MySQLParserContext { 
    ANTLRInputStream input; 
    MySQLLexer lexer; 
    CommonTokenStream tokens; 
    MySQLParser parser; 
    ContextErrorListener errorListener; 

    bool caseSensitive; 
    std::vector<ParserErrorInfo> errors; 

    ... 

    ParseTree *parse(const std::string &text, MySQLParseUnit unit) { 
    input.load(text); 
    return startParsing(false, unit); 
    } 

    bool errorCheck(const std::string &text, MySQLParseUnit unit) { 
    parser.removeParseListeners(); 
    input.load(text); 
    startParsing(true, unit); 
    return errors.empty(); 
    } 

private: 
    ParseTree *parseUnit(MySQLParseUnit unit) { 
    switch (unit) { 
     case MySQLParseUnit::PuCreateSchema: 
     return parser.createDatabase(); 
     case MySQLParseUnit::PuCreateTable: 
     return parser.createTable(); 
     case MySQLParseUnit::PuCreateTrigger: 
     return parser.createTrigger(); 
     case MySQLParseUnit::PuCreateView: 
     return parser.createView(); 
     case MySQLParseUnit::PuCreateFunction: 
     return parser.createFunction(); 
     case MySQLParseUnit::PuCreateProcedure: 
     return parser.createProcedure(); 
     case MySQLParseUnit::PuCreateUdf: 
     return parser.createUdf(); 
     case MySQLParseUnit::PuCreateRoutine: 
     return parser.createRoutine(); 
     case MySQLParseUnit::PuCreateEvent: 
     return parser.createEvent(); 
     case MySQLParseUnit::PuCreateIndex: 
     return parser.createIndex(); 
     case MySQLParseUnit::PuGrant: 
     return parser.grant(); 
     case MySQLParseUnit::PuDataType: 
     return parser.dataTypeDefinition(); 
     case MySQLParseUnit::PuCreateLogfileGroup: 
     return parser.createLogfileGroup(); 
     case MySQLParseUnit::PuCreateServer: 
     return parser.createServer(); 
     case MySQLParseUnit::PuCreateTablespace: 
     return parser.createTablespace(); 
     default: 
     return parser.query(); 
    } 
    } 

    ParseTree *startParsing(bool fast, MySQLParseUnit unit) { 
    errors.clear(); 
    lexer.reset(); 
    lexer.setInputStream(&input); // Not just reset(), which only rewinds the current position. 
    tokens.setTokenSource(&lexer); 

    parser.reset(); 
    parser.setBuildParseTree(!fast); 

    // First parse with the bail error strategy to get quick feedback for correct queries. 
    parser.setErrorHandler(std::make_shared<BailErrorStrategy>()); 
    parser.getInterpreter<ParserATNSimulator>()->setPredictionMode(PredictionMode::SLL); 

    ParseTree *tree; 
    try { 
     tree = parseUnit(unit); 
    } catch (ParseCancellationException &) { 
     if (fast) 
     tree = nullptr; 
     else { 
     // If parsing was cancelled we either really have a syntax error or we need to do a second step, 
     // now with the default strategy and LL parsing. 
     tokens.reset(); 
     parser.reset(); 
     parser.setErrorHandler(std::make_shared<DefaultErrorStrategy>()); 
     parser.getInterpreter<ParserATNSimulator>()->setPredictionMode(PredictionMode::LL); 
     tree = parseUnit(unit); 
     } 
    } 

    if (errors.empty() && !lexer.hitEOF) { 
     // There is more input than needed for the given parse unit. Make this a fail as we don't allow 
     // extra input after the specific rule. 
     // This part is only needed if the grammar has no explicit EOF token at the end of the parsed rule. 
     Token *token = tokens.LT(1); 
     ParserErrorInfo info = {"extraneous input found, expecting end of input", 
           token->getType(), 
           token->getStartIndex(), 
           token->getLine(), 
           token->getCharPositionInLine(), 
           token->getStopIndex() - token->getStartIndex() + 1}; 
     errors.push_back(info); 
    } 
    return tree; 
    } 
    ...