directory organize and lexer now ignores comments

ExampleCodes/1-Variables.fig

@@ -0,0 +1,13 @@
+var any_var;  // type is Any
+var number = 10; // type is Int
+number = "123";  // valid
+
+var number2 := 10; // specific type is Int
+var number3: Int = 10; // both is ok
+/* 
+number2 = 3.14;
+invalid!
+*/
+
+const Pi := 3.14; // recommended, auto detect type
+// equal -> const Pi: Double = 3.14;

src/lexer.cpp

@@ -409,31 +409,39 @@ namespace Fig
         // entry: when iterator current char is '/' and peek is '/' or '*'
         // current char is '/'
         FString comment;
-        if (it.peek() == U'/')
+
+        if (it.peek() == U'/') // single-line comment
         {
-            next();
-            next();
+            next(); // skip first '/'
+            next(); // skip second '/'
+
             UTF8Char c = *it;
             while (c != U'\n' and hasNext())
             {
                 comment += c.getString();
                 next();
+                c = *it;
+            }
+
+            if (hasNext() && c == U'\n')
+            {
+                next();
             }
-            next();
         }
-        else
+        else // multi-line comment
         {
-            next();
-            next();
+            next(); // skip '/'
+            next(); // skip '*'
+
             UTF8Char c = *it;
             bool terminated = false;
+
             while (hasNext())
             {
                 if (c == U'*' and hasNext() and it.peek() == U'/')
                 {
                     next(); // skip '*'
                     next(); // skip '/'
-                    next(); // to next char
                     terminated = true;
                     break;
                 }
@@ -441,8 +449,10 @@ namespace Fig
                 {
                     comment += c.getString();
                     next();
+                    c = *it;
                 }
             }
+
             if (!terminated)
             {
                 error = SyntaxError(FStringView(u8"Unterminated multiline comment"), this->line, it.column());
@@ -450,6 +460,7 @@ namespace Fig
                 return IllegalTok;
             }
         }
+
         return Token(comment, TokenType::Comments);
     }
     Token Lexer::nextToken()
@@ -470,6 +481,24 @@ namespace Fig
         }
         last_line = getCurrentLine();
         last_column = getCurrentColumn();
+        if (ch == U'/')
+        {
+            UTF8Char c{u8""};
+            if (!hasNext())
+            {
+                next();
+                // return Token(u8"/", this->symbol_map.at(u8"/")).setPos(last_line, last_column);
+            }
+            c = it.peek();
+            if (c != U'/' and c != U'*')
+            {
+                next();
+                // return Token(u8"/", this->symbol_map.at(u8"/")).setPos(last_line, last_column);
+            }
+            scanComments().setPos(last_line, last_column);
+            return nextToken();
+            // now we ignore comments to avoid some stupid bugs
+        }
         if (ch == U'r' and hasNext() and it.peek() == U'"')
         {
             // r""
@@ -491,22 +520,6 @@ namespace Fig
         {
             return scanNumber().setPos(last_line, last_column);
         }
-        else if (ch == U'/')
-        {
-            UTF8Char c{u8""};
-            if (!hasNext())
-            {
-                next();
-                return Token(u8"/", this->symbol_map.at(u8"/")).setPos(last_line, last_column);
-            }
-            c = it.peek();
-            if (c != U'/' and c != U'*')
-            {
-                next();
-                return Token(u8"/", this->symbol_map.at(u8"/")).setPos(last_line, last_column);
-            }
-            return scanComments().setPos(last_line, last_column);
-        }
         else if (ch.isPunct())
         {
             return scanSymbol().setPos(last_line, last_column);

src/main.cpp

@@ -109,6 +109,13 @@ int main(int argc, char **argv)
     file.close();
 
     Fig::Lexer lexer((Fig::FString(source)));
+
+    // Token tok;
+    // while ((tok = lexer.nextToken()).getType() != TokenType::EndOfFile)
+    // {
+    //     std::println("{}", tok.toString().toBasicString());
+    // }
+
     Fig::Parser parser(lexer);
     std::vector<Fig::Ast::AstBase> ast;
 
@@ -136,12 +143,6 @@ int main(int argc, char **argv)
         return 1;
     }
 
-    // Token tok;
-    // while ((tok = lexer.nextToken()).getType() != TokenType::EndOfFile)
-    // {
-    //     std::println("{}", tok.toString().toBasicString());
-    // }
-
     // AstPrinter printer;
     // std::print("<Debug> AST:\n");
     // for (const auto &node : ast)