Match token blob

samples/test.brc

@@ -1,4 +1,4 @@
-User pack
+User blob
 ;
 
 main fun -> u32

src/Lexer/Lexer.cpp

@@ -208,14 +208,6 @@ shared_ptr<Token> Lexer::nextToken() {
         return token;
 
     // keywords
-    token = match(TokenKind::IF, "if", true);
-    if (token != nullptr)
-        return token;
-
-    token = match(TokenKind::ELSE, "else", true);
-    if (token != nullptr)
-        return token;
-
     token = match(TokenKind::FUNCTION, "fun", true);
     if (token != nullptr)
         return token;
@@ -225,12 +217,24 @@ shared_ptr<Token> Lexer::nextToken() {
         foundRawSourceStart = true;
         return token;
     }
+
+    token = match(TokenKind::BLOB, "blob", true);
+    if (token != nullptr)
+        return token;
     
     token = match(TokenKind::RETURN, "ret", true);
     if (token != nullptr)
         return token;
 
     token = match(TokenKind::REPEAT, "rep", true);
+    if (token != nullptr)
+        return token;
+
+     token = match(TokenKind::IF, "if", true);
+    if (token != nullptr)
+        return token;
+
+    token = match(TokenKind::ELSE, "else", true);
     if (token != nullptr)
         return token;
     
@@ -313,6 +317,31 @@ shared_ptr<Token> Lexer::match(TokenKind kind, string lexme, bool needsSeparator
     return token;
 }
 
+shared_ptr<Token> Lexer::matchReal() {
+    int nextIndex = currentIndex;
+
+    while (nextIndex < source.length() && isDecDigit(nextIndex))
+        nextIndex++;
+    
+    if (nextIndex >= source.length() || source.at(nextIndex) != '.')
+        return nullptr;
+    else
+        nextIndex++;
+    
+    while (nextIndex < source.length() && isDecDigit(nextIndex))
+        nextIndex++;
+
+    if (!isSeparator(nextIndex)) {
+        markError();
+        return nullptr;
+    }
+
+    string lexme = source.substr(currentIndex, nextIndex - currentIndex);
+    shared_ptr<Token> token = make_shared<Token>(TokenKind::REAL, lexme, currentLine, currentColumn);
+    advanceWithToken(token);
+    return token;
+}
+
 shared_ptr<Token> Lexer::matchIntegerDec() {
     int nextIndex = currentIndex;
 
@@ -399,31 +428,6 @@ shared_ptr<Token> Lexer::matchIntegerChar() {
     return token;
 }
 
-shared_ptr<Token> Lexer::matchReal() {
-    int nextIndex = currentIndex;
-
-    while (nextIndex < source.length() && isDecDigit(nextIndex))
-        nextIndex++;
-    
-    if (nextIndex >= source.length() || source.at(nextIndex) != '.')
-        return nullptr;
-    else
-        nextIndex++;
-    
-    while (nextIndex < source.length() && isDecDigit(nextIndex))
-        nextIndex++;
-
-    if (!isSeparator(nextIndex)) {
-        markError();
-        return nullptr;
-    }
-
-    string lexme = source.substr(currentIndex, nextIndex - currentIndex);
-    shared_ptr<Token> token = make_shared<Token>(TokenKind::REAL, lexme, currentLine, currentColumn);
-    advanceWithToken(token);
-    return token;
-}
-
 shared_ptr<Token> Lexer::matchString() {
     int nextIndex = currentIndex;
 

src/Lexer/Lexer.h

@@ -22,11 +22,11 @@ private:
 
     shared_ptr<Token> nextToken();
     shared_ptr<Token> match(TokenKind kind, string lexme, bool needsSeparator);
+    shared_ptr<Token> matchReal();
     shared_ptr<Token> matchIntegerDec();
     shared_ptr<Token> matchIntegerHex();
     shared_ptr<Token> matchIntegerBin();
     shared_ptr<Token> matchIntegerChar();
-    shared_ptr<Token> matchReal();
     shared_ptr<Token> matchString();
     shared_ptr<Token> matchType();
     shared_ptr<Token> matchIdentifier();

src/Lexer/Token.cpp

@@ -47,11 +47,11 @@ vector<TokenKind> Token::tokensBinary = {
 
 vector<TokenKind> Token::tokensLiteral = {
     TokenKind::BOOL,
+    TokenKind::REAL,
     TokenKind::INTEGER_DEC,
     TokenKind::INTEGER_HEX,
     TokenKind::INTEGER_BIN,
     TokenKind::INTEGER_CHAR,
-    TokenKind::REAL,
     TokenKind::STRING
 };
 

src/Lexer/Token.h

@@ -32,20 +32,21 @@ enum class TokenKind {
     FUNCTION,
     RAW_FUNCTION,
     RAW_SOURCE_LINE,
+    BLOB,
     RETURN,
     REPEAT,
     IF,
     ELSE,
 
     BOOL,
+    REAL,
     INTEGER_DEC,
     INTEGER_HEX,
     INTEGER_BIN,
     INTEGER_CHAR,
-    REAL,
     STRING,
-    IDENTIFIER,
     TYPE,
+    IDENTIFIER,
 
     M_EXTERN,
 

src/Logger.cpp

@@ -94,20 +94,22 @@ string Logger::toString(shared_ptr<Token> token) {
         case TokenKind::TYPE:
             return "TYPE(" + token->getLexme() + ")";
 
-        case TokenKind::IF:
-            return "IF";
-        case TokenKind::ELSE:
-            return "ELSE";
         case TokenKind::FUNCTION:
             return "FUN";
         case TokenKind::RAW_FUNCTION:
             return "RAW";
         case TokenKind::RAW_SOURCE_LINE:
             return format("RAW_SOURCE_LINE({})", token->getLexme());
+        case TokenKind::BLOB:
+            return "BLOB";
         case TokenKind::RETURN:
             return "RET";
         case TokenKind::REPEAT:
             return "REP";
+        case TokenKind::IF:
+            return "IF";
+        case TokenKind::ELSE:
+            return "ELSE";
 
         case TokenKind::M_EXTERN:
             return "@EXTERN";