1 files changed, 14 insertions, 57 deletions

diff --git a/src/tokenizer.zig b/src/tokenizer.zig
index dc700c8..91cae39 100644
--- a/src/tokenizer.zig
+++ b/src/tokenizer.zig
@@ -3,7 +3,6 @@ const std = @import("std");
 pub const Token_enum = enum {
     RARROW,                     // ->
     LARROW,                     // <-
-    BACKTICK,                   // `
     PERIOD,                     // .
     COMMA,                      // ,
     QMARK,                      // ?
@@ -19,7 +18,6 @@ pub const Token_enum = enum {
 pub const Token = union(Token_enum) {
     RARROW,                     // ->
     LARROW,                     // <-
-    BACKTICK,                   // `
     PERIOD,                     // .
     COMMA,                      // ,
     QMARK,                      // ?
@@ -35,7 +33,6 @@ pub const Token = union(Token_enum) {
         return switch (self) {
             .RARROW => "->",
             .LARROW => "<-",
-            .BACKTICK => "`",
             .PERIOD => ".",
             .COMMA => ",",
             .QMARK => "?",
@@ -92,16 +89,17 @@ pub fn Iterator(comptime typ: type) type {
 
         pub fn consumeuntil(self: *SelfType, alloc: std.mem.Allocator, delims: []const typ) !?[]typ {
             var arr = try std.ArrayList(typ).initCapacity(alloc, 128);
-            while (self.peek()) |item| {
+            out: while (self.peek()) |item| {
                 for (delims) |d|
-                    if (std.meta.eql(item, d)) break;
+                    if (std.meta.eql(item, d)) break :out;
                 self.skip();
                 try arr.append(alloc, item);
             }
             return try arr.toOwnedSlice(alloc);
         }
 
-        pub fn consumeuntilescape(self: *SelfType, alloc: std.mem.Allocator, delims: []const typ, escape: typ) !?[]typ { var arr = try std.ArrayList(typ).initCapacity(alloc, 128);
+        pub fn consumeuntilescape(self: *SelfType, alloc: std.mem.Allocator, delims: []const typ, escape: typ) !?[]typ {
+            var arr = try std.ArrayList(typ).initCapacity(alloc, 128);
             var previous: typ = undefined;
             while (self.peek()) |item| {
                 for (delims) |d|
@@ -152,16 +150,22 @@ pub fn tokenize(allocator: std.mem.Allocator, input: []const u8) ![]Token {
     };
 
     while (src.peek()) |char| {
+        std.debug.print("{}\n", .{char});
         switch (char) {
-            '`' => {
+            ',' => {
                 src.skip();
                 try internals.clearbuff(allocator, &toks, &buff);
-                try toks.append(allocator, .BACKTICK);
+                try toks.append(allocator, .COMMA);
             },
-            ',' => {
+            '?' => {
                 src.skip();
+                if (src.peek().? != '?') {
+                    try toks.append(allocator, .QMARK);
+                    continue;
+                }
                 try internals.clearbuff(allocator, &toks, &buff);
-                try toks.append(allocator, .COMMA);
+                const comment = try src.consumeuntil(allocator, "\n\r");
+                allocator.free(comment.?);
             },
             '.' => {
                 src.skip();
@@ -241,50 +245,3 @@ pub fn tokenize(allocator: std.mem.Allocator, input: []const u8) ![]Token {
     }
     return toks.toOwnedSlice(allocator);
 }
-
-// pub fn tokenize(allocator: std.mem.Allocator, input: []const u8) ![]Token {
-//     var arr = try std.ArrayList(Token).initCapacity(allocator, 1024);
-//     defer arr.deinit(allocator);
-//     var iterator = Iterator(u8).init(input);
-//     return parse: switch (iterator.next().?) {
-//         '-' => {
-//             if (iterator.maybe('>')) |_| try arr.append(allocator, .RARROW);
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         '<' =>  {
-//             if (iterator.maybe('-')) |_| try arr.append(allocator, .LARROW);
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         '.' => {
-//             try arr.append(allocator, .PERIOD);
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         ',' => {
-//             try arr.append(allocator, .COMMA);
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         '`' => {
-//             try arr.append(allocator, .QMARK);
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         '?' => {
-//             try arr.append(allocator, .QMARK);
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         '!' => {
-//             const name = try iterator.consumeuntil(allocator, &std.ascii.whitespace);
-//             try arr.append(allocator, Token{.BUILTIN = name.?});
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         '"' => {
-//             const name = try iterator.consumeuntilescape(allocator, "\"", '\\');
-//             try arr.append(allocator, Token{.STRING = name.?});
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         'A'...'Z' => {
-//             const name = try iterator.consumeuntil(allocator, &std.ascii.whitespace);
-//             try arr.append(allocator, Token{.TYP = name.?});
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         '0'...'9' => {
-//             const name = try iterator.consumewhile(allocator, "0123456789");
-//             try arr.append(allocator, Token{.FUNC = name.?});
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//         else => {
-//             const name = try iterator.consumeuntil(allocator, @constCast(&std.ascii.whitespace));
-//             try arr.append(allocator, Token{.FUNC = name.?});
-//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
-//     };
-
-// }