Initial commit with tokenizerHEADmain

8 files changed, 471 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..dca1103
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+zig-out/
+.zig-cache/
diff --git a/build.zig b/build.zig
new file mode 100644
index 0000000..4d9fb09
--- /dev/null
+++ b/build.zig
@@ -0,0 +1,26 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const exe = b.addExecutable(.{
+        .name = "gren",
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("src/main.zig"),
+            .target = target,
+            .optimize = optimize,
+        }),
+    });
+
+    b.installArtifact(exe);
+    const run_cmd = b.addRunArtifact(exe);
+    run_cmd.step.dependOn(b.getInstallStep());
+
+    if (b.args) |args| {
+        run_cmd.addArgs(args);
+    }
+
+    const run_step = b.step("run", "Run the compiler");
+    run_step.dependOn(&run_cmd.step);
+}
diff --git a/build.zig.zon b/build.zig.zon
new file mode 100644
index 0000000..ba2b122
--- /dev/null
+++ b/build.zig.zon
@@ -0,0 +1,7 @@
+.{
+    .name = .gren,
+    .version = "0.0.1",
+    .minimum_zig_version = "0.15.1",
+    .paths = .{""},
+    .fingerprint = 0x62407f25438ecb36,
+}
diff --git a/examples/example.gn b/examples/example.gn
new file mode 100644
index 0000000..35e2ce0
--- /dev/null
+++ b/examples/example.gn
@@ -0,0 +1,94 @@
+import fn puts(str: [u8]) -> i32;
+
+fn main() -> i32 {
+    return puts("Hello World!");
+}
+
+!use defaults->+
+
+(add <- int <- int) -> int
+add x -> y -> x + y
+
+--------------------------
+
+
+!use builtin -> universal -> Natural.
+!use builtin -> universal -> Integer.
+!use builtin -> universal -> Real.
+!use builtin -> universal ->.
+!use builtin -> x86 -> i32.
+!use builtin -> x86_64 -> i128.
+!use builtin -> x86 -> u32.
+!use defaults -> +.
+!use defaults -> println.
+!use defaults -> map.
+!entrypoint <- main.
+
+
+(Vec3 <- a <- a <- Natural) -> (a, a, Natural).
+Vec3 a b c -> (a, b, c).
+
+(I <- ) ->.
+I a -> a.
+
+(K <- <-) ->.
+K x -> y -> x.
+
+(`add` <- Natural <- Natural) -> Natural.
+`add` x -> y -> x + y.
+
+(`+` <- Float <- Float) -> Float.
+
+
+(doStuff <- Integer) -> Integer.
+doStuff x -> 1.0 + x // Warning
+
+(main <- [String]) -> ? IO.
+main args -> match (x <- len args) (
+    (x = 0) -> println "No Args",
+    (x < 3) -> println (get 0 args),
+      -> map println args
+).
+
+(println <- String) -> ? IO.
+println str -> (print str, print "\n").
+
+!use defaults -> x86 -> linux -> _stdout.
+!use defaults -> write.
+
+(print <- String) -> ? IO.
+print str -> write str _stdout.
+
+(World) -> (IO, MemAction).
+(IO) -> (FileRead, FileWrite, FileOpen, FileClose, FileStat).
+(MemAction) -> (MemRead, MemWrite, MemMap, Alloc, Free).
+
+(write <- String <- File a) -> (x86_64 -> i64) ? (FileAction a).
+write str file -> !syscall_write (len str) str _stdout.
+write str file -> !asm "syscall" (
+                !reg "rax" 1,
+                !reg "rsi" str,
+                !reg "rdx" file,
+                !reg "rdi" (len str))
+                ("rax").
+
+!use defaults -> dlopen.
+(somefunc <- Integer) -> Integer.
+somefunc -> !deref (dlopen "SomeFunc") 0.
+
+!use builtin -> x86_64 -> ptr.
+!use builtin -> x86_64 -> Pointer.
+!setrule embedded. // Disables defaults, enables mutability underneath pointers
+
+(Byte) -> Type.
+(Color) -> Type.
+Byte -> (x86_64 -> u8).
+Color -> Byte.
+(cols) -> Integer.
+cols -> 80.
+
+// (Byte, Byte) is packed
+(framebuffer) -> (Pointer (Byte, Color)).
+framebuffer -> ptr 0x8B000.
+(buffset <- Natural <- Natural <- Byte <- Color) -> ? MemWrite.
+buffset x y char col -> memwrite framebuffer (y * cols + x) (char, col).
diff --git a/examples/test.gn b/examples/test.gn
new file mode 100644
index 0000000..99e7ef6
--- /dev/null
+++ b/examples/test.gn
@@ -0,0 +1,11 @@
+!use defaults -> io -> println.
+!use defaults -> effects.
+!use defaults -> math -> +.
+!use builtin -> block.
+!use builtin -> return.
+!entrypoint <- main.
+(main) -> Int ? (effects -> IO).
+main -> match (x <- println "Hello World") (
+    (x >= 0) -> 0,
+    (x < 0)-> 1,
+)
diff --git a/src/lexer.zig b/src/lexer.zig
new file mode 100644
index 0000000..95a0b68
--- /dev/null
+++ b/src/lexer.zig
@@ -0,0 +1 @@
+const std = @import("std");
diff --git a/src/main.zig b/src/main.zig
new file mode 100644
index 0000000..6095e81
--- /dev/null
+++ b/src/main.zig
@@ -0,0 +1,40 @@
+const std = @import("std");
+const tok = @import("tokenizer.zig");
+
+pub fn main() !void {
+    const inneralloc = std.heap.smp_allocator;
+    var arena = std.heap.ArenaAllocator.init(inneralloc);
+    defer arena.deinit();
+    const alloc = arena.allocator();
+
+    var outbuff: [64]u8 = undefined;
+    var inbuff: [64]u8 = undefined;
+    var stdout_writer = std.fs.File.stdout().writer(&outbuff);
+    var stdin_reader = std.fs.File.stdin().reader(&inbuff);
+    const stdout = &stdout_writer.interface;
+    const stdin = &stdin_reader.interface;
+
+    var sb = try std.ArrayList(u8).initCapacity(alloc, 1024);
+    defer sb.deinit(alloc);
+
+    var line = std.io.Writer.Allocating.init(alloc);
+    defer line.deinit();
+
+    while (true) {
+        _ = stdin.streamDelimiter(&line.writer, '\n') catch |err| {
+            if (err == error.EndOfStream) break else return err;
+        };
+        try sb.appendSlice(alloc, line.written());
+        try sb.appendSlice(alloc, try stdin.take(1));
+        line.clearRetainingCapacity();
+    }
+    if (line.written().len > 0) try sb.appendSlice(alloc, line.written());
+
+    std.debug.print("INPUT\n==========\n{s}\n==========\n", .{sb.items});
+
+    const tokens = try tok.tokenize(alloc, sb.items);
+    std.debug.print("{any}\n", .{tokens});
+    for (tokens) |t|
+        try stdout.print("{s}\n", .{ try t.print(alloc) });
+    try stdout.flush();
+}
diff --git a/src/tokenizer.zig b/src/tokenizer.zig
new file mode 100644
index 0000000..dc700c8
--- /dev/null
+++ b/src/tokenizer.zig
@@ -0,0 +1,290 @@
+const std = @import("std");
+
+pub const Token_enum = enum {
+    RARROW,                     // ->
+    LARROW,                     // <-
+    BACKTICK,                   // `
+    PERIOD,                     // .
+    COMMA,                      // ,
+    QMARK,                      // ?
+    LPAREN,                     // (
+    RPAREN,                     // )
+    STRING,                     // "..."
+    BUILTIN,                    // !word
+    FUNC,                       // all chars
+    TYP,                        // Capital Letters
+    INT,                        // numbers
+};
+
+pub const Token = union(Token_enum) {
+    RARROW,                     // ->
+    LARROW,                     // <-
+    BACKTICK,                   // `
+    PERIOD,                     // .
+    COMMA,                      // ,
+    QMARK,                      // ?
+    LPAREN,                     // (
+    RPAREN,                     // )
+    STRING:     []const u8,     // "..."
+    BUILTIN:    []const u8,     // !word
+    FUNC:       []const u8,     // lowercase letters
+    TYP:        []const u8,     // Capital Letters
+    INT:        i64,            // numbers
+
+    pub fn print(self: Token,alloc: std.mem.Allocator) ![]const u8 {
+        return switch (self) {
+            .RARROW => "->",
+            .LARROW => "<-",
+            .BACKTICK => "`",
+            .PERIOD => ".",
+            .COMMA => ",",
+            .QMARK => "?",
+            .LPAREN => "(",
+            .RPAREN => ")",
+            .STRING => |v| v,
+            .BUILTIN => |v| v,
+            .FUNC => |v| v,
+            .TYP => |v| v,
+            .INT => |v| try std.fmt.allocPrint(alloc, "{d}",.{v}),
+        };
+    }
+};
+
+/// Creates a tokenizer over a slice of typ
+pub fn Iterator(comptime typ: type) type {
+    return struct {
+        items: []const typ,
+        index: usize = 0,
+
+        const SelfType = Iterator(typ);
+        const Error = error{
+            OutOfBounds,
+            ExpectedItem,
+            EndOfItems,
+        };
+
+        /// Initialize tokenizer with a slice
+        pub fn init(items: []const typ) SelfType {
+            return Iterator(typ){ .items = items };
+        }
+
+        /// Get current item
+        pub fn peekAhead(self: *SelfType, ahead: u32) ?typ {
+            if (self.index + ahead >= self.items.len) return null;
+            return self.items[self.index + ahead];
+        }
+
+        pub fn peek(self: *SelfType) ?typ {
+            return self.peekAhead(0);
+        }
+
+        /// Get current item and iterate index
+        pub fn next(self: *SelfType) ?typ {
+            const ret = self.peek();
+            self.skip();
+            return ret;
+        }
+
+        pub fn consume(self: *SelfType, expected: typ) !?typ {
+            if (!std.meta.eql(self.peek().?, expected)) return Error.ExpectedItem;
+            return self.next();
+        }
+
+        pub fn consumeuntil(self: *SelfType, alloc: std.mem.Allocator, delims: []const typ) !?[]typ {
+            var arr = try std.ArrayList(typ).initCapacity(alloc, 128);
+            while (self.peek()) |item| {
+                for (delims) |d|
+                    if (std.meta.eql(item, d)) break;
+                self.skip();
+                try arr.append(alloc, item);
+            }
+            return try arr.toOwnedSlice(alloc);
+        }
+
+        pub fn consumeuntilescape(self: *SelfType, alloc: std.mem.Allocator, delims: []const typ, escape: typ) !?[]typ { var arr = try std.ArrayList(typ).initCapacity(alloc, 128);
+            var previous: typ = undefined;
+            while (self.peek()) |item| {
+                for (delims) |d|
+                    if (std.meta.eql(item, d) and !std.meta.eql(previous, escape)) break;
+                self.skip();
+                try arr.append(alloc, item);
+                previous = item;
+            }
+            return try arr.toOwnedSlice(alloc);
+        }
+
+        pub fn consumewhile(self: *SelfType, alloc: std.mem.Allocator, allowed: []const typ) !?[]typ {
+            var arr = try std.ArrayList(typ).initCapacity(alloc, 128);
+            while (self.peek()) |item| {
+                for (allowed) |d|
+                    if (!std.meta.eql(item, d)) break;
+                self.skip();
+                try arr.append(alloc, item);
+            }
+            return try arr.toOwnedSlice(alloc);
+        }
+
+
+        pub fn maybe(self: *SelfType, expected: typ) ?typ {
+            return self.consume(expected) catch null;
+        }
+
+        /// Skip over current item
+        pub fn skip(self: *Iterator(typ)) void {
+            self.index += 1;
+        }
+    };
+}
+
+pub fn tokenize(allocator: std.mem.Allocator, input: []const u8) ![]Token {
+    var toks = std.ArrayList(Token){};
+    var buff = std.ArrayList(u8){};
+    defer buff.deinit(allocator);
+    var src = Iterator(u8).init(input);
+
+    const internals = struct {
+        fn clearbuff(alloc: std.mem.Allocator, tok: *std.ArrayList(Token), buf: *std.ArrayList(u8)) !void {
+            if (buf.items.len == 0) return;
+            const str = try buf.toOwnedSlice(alloc);
+            try tok.append(alloc, .{ .FUNC = str });
+            buf.clearAndFree(alloc);
+        }
+    };
+
+    while (src.peek()) |char| {
+        switch (char) {
+            '`' => {
+                src.skip();
+                try internals.clearbuff(allocator, &toks, &buff);
+                try toks.append(allocator, .BACKTICK);
+            },
+            ',' => {
+                src.skip();
+                try internals.clearbuff(allocator, &toks, &buff);
+                try toks.append(allocator, .COMMA);
+            },
+            '.' => {
+                src.skip();
+                try internals.clearbuff(allocator, &toks, &buff);
+                try toks.append(allocator, .PERIOD);
+            },
+            '(' => {
+                src.skip();
+                try internals.clearbuff(allocator, &toks, &buff);
+                try toks.append(allocator, .LPAREN);
+            },
+            ')' => {
+                src.skip();
+                try internals.clearbuff(allocator, &toks, &buff);
+                try toks.append(allocator, .RPAREN);
+            },
+            '-' => {
+                src.skip();
+                if (src.peek().? != '>') {
+                    try buff.append(allocator, '-' );
+                    continue;
+                }
+                src.skip();
+                try internals.clearbuff(allocator, &toks, &buff);
+                try toks.append(allocator, .RARROW);
+            },
+            '<' => {
+                src.skip();
+                if (src.peek().? != '-') {
+                    try buff.append(allocator, '<' );
+                    continue;
+                }
+                src.skip();
+                try internals.clearbuff(allocator, &toks, &buff);
+                try toks.append(allocator, .LARROW);
+            },
+            '0'...'9' => {
+                while (std.ascii.isDigit(src.peek().?))
+                    try buff.append(allocator, src.next().?);
+
+                const num: i32 = try std.fmt.parseInt(i32, buff.items, 10);
+                try toks.append(allocator, .{ .INT = num });
+                buff.clearAndFree(allocator);
+            },
+            'A'...'Z' => {
+                while (std.ascii.isAlphabetic(src.peek().?))
+                    try buff.append(allocator, src.next().?);
+                const str = try buff.toOwnedSlice(allocator);
+                try toks.append(allocator, .{ .TYP = str });
+                buff.clearAndFree(allocator);
+            },
+            '!' => {
+                src.skip();
+                while (std.ascii.isAlphanumeric(src.peek().?))
+                    try buff.append(allocator, src.next().?);
+                const str = try buff.toOwnedSlice(allocator);
+                try toks.append(allocator, .{ .BUILTIN = str });
+                buff.clearAndFree(allocator);
+            },
+            '"' => {
+                _ = src.next();
+                while (src.peek().? != '"')
+                    try buff.append(allocator, src.next().?);
+
+                _ = src.next();
+                const token = Token{ .STRING = try buff.toOwnedSlice(allocator) };
+                try toks.append(allocator, token);
+                buff.clearAndFree(allocator);
+            },
+            ' ', '\t', '\n' => {
+                src.skip();
+                if (buff.items.len == 0) continue;
+                try internals.clearbuff(allocator, &toks, &buff);
+            },
+            else => try buff.append(allocator, src.next().?),
+        }
+    }
+    return toks.toOwnedSlice(allocator);
+}
+
+// pub fn tokenize(allocator: std.mem.Allocator, input: []const u8) ![]Token {
+//     var arr = try std.ArrayList(Token).initCapacity(allocator, 1024);
+//     defer arr.deinit(allocator);
+//     var iterator = Iterator(u8).init(input);
+//     return parse: switch (iterator.next().?) {
+//         '-' => {
+//             if (iterator.maybe('>')) |_| try arr.append(allocator, .RARROW);
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         '<' =>  {
+//             if (iterator.maybe('-')) |_| try arr.append(allocator, .LARROW);
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         '.' => {
+//             try arr.append(allocator, .PERIOD);
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         ',' => {
+//             try arr.append(allocator, .COMMA);
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         '`' => {
+//             try arr.append(allocator, .QMARK);
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         '?' => {
+//             try arr.append(allocator, .QMARK);
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         '!' => {
+//             const name = try iterator.consumeuntil(allocator, &std.ascii.whitespace);
+//             try arr.append(allocator, Token{.BUILTIN = name.?});
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         '"' => {
+//             const name = try iterator.consumeuntilescape(allocator, "\"", '\\');
+//             try arr.append(allocator, Token{.STRING = name.?});
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         'A'...'Z' => {
+//             const name = try iterator.consumeuntil(allocator, &std.ascii.whitespace);
+//             try arr.append(allocator, Token{.TYP = name.?});
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         '0'...'9' => {
+//             const name = try iterator.consumewhile(allocator, "0123456789");
+//             try arr.append(allocator, Token{.FUNC = name.?});
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//         else => {
+//             const name = try iterator.consumeuntil(allocator, @constCast(&std.ascii.whitespace));
+//             try arr.append(allocator, Token{.FUNC = name.?});
+//             if (iterator.peek()) |pk| continue :parse pk else break :parse arr.items; },
+//     };
+
+// }