Introduce multiline comments v1.3

2 files changed, 27 insertions, 6 deletions

diff --git a/ccl.7 b/ccl.7
index 83a58b0..7152ddc 100644
--- a/ccl.7
+++ b/ccl.7
@@ -79,10 +79,21 @@ an uncatchable error
 if stack is too small.
 .
 .Pp
-Comments are done using
+They're 2 types of comments in
+.Nm .
+The first is a one line one:
+It starts with a
 .Ic /
-character.
-They end on the end of line.
+character,
+and ends with a newline.
+The second is the multiline comment:
+Text inside
+.Ic \e
+characters is not parsed.
+These comments can't overlap,
+and you can safely put
+characters meaning one comment type
+into another.
 .
 .Pp
 The instructions are as follows:
diff --git a/tokenizer.c b/tokenizer.c
index e843ef1..a22e0db 100644
--- a/tokenizer.c
+++ b/tokenizer.c
@@ -9,7 +9,7 @@
 size_t cccl_tokenize(const char *code, size_t size, struct cccl_Token tokens[], size_t tokens_length)
 {
     size_t i = 0, tokeni = 0;
-    int comment = 0;
+    int comment = 0, mlcomment = 0;
 
     do
     {
@@ -17,7 +17,7 @@ size_t cccl_tokenize(const char *code, size_t size, struct cccl_Token tokens[],
         {
 #define X(name)                                 \
     {                                           \
-        if (comment)                            \
+        if (comment || mlcomment)               \
             break;                              \
         tokens[tokeni++] = (struct cccl_Token)  \
         {                                       \
@@ -27,12 +27,22 @@ size_t cccl_tokenize(const char *code, size_t size, struct cccl_Token tokens[],
     } break
         case '/':
         {
+            if (mlcomment)
+                break;
             comment = 1;
         } break;
         case '\n':
         {
+            if (mlcomment)
+                break;
             comment = 0;
         } break;
+        case '\\':
+        {
+            if (comment)
+                break;
+            mlcomment = !mlcomment;
+        } break;
         case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': 
         case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 
         case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z':
@@ -54,7 +64,7 @@ size_t cccl_tokenize(const char *code, size_t size, struct cccl_Token tokens[],
         { } break;
         default:
         {
-            if (comment)
+            if (comment || mlcomment)
                 break;
             errx(1, "Illegal symbol in a code at byte %lu: [%d] %c", i, code[i], code[i]);
         } break;