From 0985a623e4cbd14da77fddf1bb1a9ca3384b28ea Mon Sep 17 00:00:00 2001 From: Adrian Kummerlaender Date: Wed, 10 Jun 2015 20:32:13 +0200 Subject: Changed token determination regarding identifier values * all non operator and digit tokens are now identifier tokens as this seems more intuitive than generating number token lists containing non-digit values * renamed lexer-local `level` variable to `nesting` as to avoid confusion between nesting state and predecence levels --- src/nodes.cc | 8 +++--- src/tree.cc | 24 +++++++--------- src/utils.cc | 94 +++++++++++++++++++++++++++++------------------------------- 3 files changed, 60 insertions(+), 66 deletions(-) diff --git a/src/nodes.cc b/src/nodes.cc index dd003c4..5432318 100644 --- a/src/nodes.cc +++ b/src/nodes.cc @@ -86,13 +86,13 @@ std::string OperatorNode::print() const { } bool OperatorNode::hasChildren() const { - return this->right_ != nullptr && - this->left_ != nullptr; + return this->right_ != nullptr + && this->left_ != nullptr; } bool OperatorNode::isParentOf(Node*const node) const { - return this->right_ == node || - this->left_ == node; + return this->right_ == node + || this->left_ == node; } TokenType OperatorNode::token() const { diff --git a/src/tree.cc b/src/tree.cc index f8c17ae..8e29255 100644 --- a/src/tree.cc +++ b/src/tree.cc @@ -12,7 +12,7 @@ namespace { using SimpleParser::Node; - inline boost::optional top(const std::stack stack) { + boost::optional top(const std::stack stack) { if ( !stack.empty() ) { return boost::make_optional( stack.top() @@ -22,7 +22,7 @@ namespace { } } - inline boost::optional pop(std::stack& stack) { + boost::optional pop(std::stack& stack) { if ( boost::optional node{ top(stack) } ) { stack.pop(); @@ -63,7 +63,7 @@ std::string Tree::print() const { for ( auto&& node : this->node_collection_ ) { out << "node" - << nodeIndex + << nodeIndex << " [ label = \"" << node->print() << "\"]; "; @@ -74,9 +74,9 @@ std::string Tree::print() const { for ( auto&& child : this->node_collection_ ) { if ( node->isParentOf(child.get()) ) { out << "\"node" - << nodeIndex + << nodeIndex << "\" -> \"node" - << childIndex + << childIndex << "\"; "; } @@ -134,10 +134,10 @@ Node* Tree::buildTree(const std::string& term) { OperatorNode*const lastOperator{ static_cast(*lastNode) }; - + if ( precedence(elementToken) > precedence(lastOperator->token()) ) { - operators.emplace( + operators.emplace( this->addNode(elementToken) ); } else { @@ -145,9 +145,7 @@ Node* Tree::buildTree(const std::string& term) { boost::optional rightChild { pop(operands) }; boost::optional leftChild { pop(operands) }; - if ( currentOperator && - rightChild && - leftChild ) { + if ( currentOperator && rightChild && leftChild ) { static_cast( *currentOperator )->setChildren( @@ -172,7 +170,7 @@ Node* Tree::buildTree(const std::string& term) { }; if ( subElements.size() == 1 ) { - switch ( determineToken(subElements.front()) ) { + switch ( determineToken(subElements.front().front()) ) { case TokenType::VALUE_NUMBER: case TokenType::OPERATOR_MINUS: { operands.emplace( @@ -210,9 +208,7 @@ Node* Tree::buildTree(const std::string& term) { boost::optional rightChild { pop(operands) }; boost::optional leftChild { pop(operands) }; - if ( currentOperator && - rightChild && - leftChild ) { + if ( currentOperator && rightChild && leftChild ) { static_cast( *currentOperator )->setChildren( diff --git a/src/utils.cc b/src/utils.cc index 46617d8..b994b10 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -7,35 +7,33 @@ namespace SimpleParser { -TokenType determineToken(const char tmp) { - if ( std::isalpha(tmp) ) { - return TokenType::VALUE_IDENTIFIER; - } else { - switch ( tmp ) { - case '-': - return TokenType::OPERATOR_MINUS; - case '+': - return TokenType::OPERATOR_PLUS; - case '/': - return TokenType::OPERATOR_DIVIDE; - case '*': - return TokenType::OPERATOR_MULTIPLY; - case '^': - return TokenType::OPERATOR_POWER; - case '(': - return TokenType::PARENTHESES_OPEN; - case ')': - return TokenType::PARENTHESES_CLOSE; - case ',': - return TokenType::VALUE_NUMBER; - default: - return TokenType::VALUE_NUMBER; - } +TokenType determineToken(const char token) { + if ( std::isdigit(token) ) { + return TokenType::VALUE_NUMBER; } -} -TokenType determineToken(const std::string& tmp) { - return determineToken(tmp.front()); + switch ( token ) { + case '-': + return TokenType::OPERATOR_MINUS; + case '+': + return TokenType::OPERATOR_PLUS; + case '/': + return TokenType::OPERATOR_DIVIDE; + case '*': + return TokenType::OPERATOR_MULTIPLY; + case '^': + return TokenType::OPERATOR_POWER; + case '(': + return TokenType::PARENTHESES_OPEN; + case ')': + return TokenType::PARENTHESES_CLOSE; + case '.': + return TokenType::VALUE_NUMBER; + case ',': + return TokenType::VALUE_NUMBER; + default: + return TokenType::VALUE_IDENTIFIER; + } } PrecedenceLevel precedence(const TokenType token) { @@ -68,12 +66,12 @@ PrecedenceLevel precedence(const TokenType token) { std::vector lexer(const std::string& term) { std::vector resultBuffer{}; - std::string levelBuffer{}; + std::string nestingBuffer{}; std::string numberBuffer{}; std::string identifierBuffer{}; TokenType previousToken{}; - uint32_t level{0}; + uint8_t nesting{0}; for ( auto&& termIter = term.begin(); termIter < term.end(); @@ -84,8 +82,8 @@ std::vector lexer(const std::string& term) { token == TokenType::VALUE_IDENTIFIER || ( token == TokenType::OPERATOR_MINUS && termIter == term.begin() ) ) { - if ( level > 0 ) { - levelBuffer += *termIter; + if ( nesting ) { + nestingBuffer += *termIter; } else { switch ( token ) { case TokenType::VALUE_NUMBER: @@ -105,7 +103,7 @@ std::vector lexer(const std::string& term) { } } } else { - if ( level == 0 ) { + if ( !nesting ) { switch ( previousToken ) { case TokenType::VALUE_NUMBER: { resultBuffer.push_back(numberBuffer); @@ -127,33 +125,33 @@ std::vector lexer(const std::string& term) { switch ( token ) { case TokenType::PARENTHESES_OPEN: { - if ( level > 0 ) { - levelBuffer += *termIter; + if ( nesting ) { + nestingBuffer += *termIter; } - ++level; + ++nesting; break; } case TokenType::PARENTHESES_CLOSE: { - --level; + --nesting; - if ( level == 0 ) { - resultBuffer.push_back(levelBuffer); - levelBuffer.clear(); + if ( nesting ) { + nestingBuffer += *termIter; } else { - levelBuffer += *termIter; + resultBuffer.push_back(nestingBuffer); + nestingBuffer.clear(); } break; } default: { - if ( level == 0 ) { + if ( nesting ) { + nestingBuffer += *termIter; + } else { const std::string helper{ *termIter }; resultBuffer.push_back(helper); - } else { - levelBuffer += *termIter; } break; @@ -183,16 +181,16 @@ std::vector lexer(const std::string& term) { } } - if ( level != 0 ) { + if ( nesting ) { throw parenthese_exception(); } - if ( previousToken == TokenType::PARENTHESES_CLOSE && + if ( previousToken == TokenType::PARENTHESES_CLOSE && resultBuffer.size() == 1 ) { - resultBuffer = lexer(resultBuffer.front()); + return lexer(resultBuffer.front()); + } else { + return resultBuffer; } - - return resultBuffer; } double stringToDouble(const std::string& str) { -- cgit v1.2.3