From c658b86a0770ba99c51bdf4e2df9544363468253 Mon Sep 17 00:00:00 2001 From: Nic Gaffney Date: Thu, 23 Oct 2025 16:54:54 -0500 Subject: Updated to zig 0.15.1 -- BREAKING --- src/codegen.zig | 20 ++++++-- src/main.zig | 24 ++++----- src/parser.zig | 146 +++++++++++++++++++++++++++++++++++-------------------- src/symtable.zig | 6 +-- src/tokenize.zig | 64 +++++++++++++++--------- src/utils.zig | 20 ++++++++ 6 files changed, 184 insertions(+), 96 deletions(-) create mode 100644 src/utils.zig (limited to 'src') diff --git a/src/codegen.zig b/src/codegen.zig index c19dfbc..252cf9c 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -1,4 +1,6 @@ +// TODO: Fork and fix this LLVM lib. const std = @import("std"); +const util = @import("utils.zig"); const parse = @import("parser.zig"); const symb = @import("symtable.zig"); const llvm = @import("llvm"); @@ -102,6 +104,7 @@ pub const Generator = struct { const symbol = table.getValue(nodeVar.ident.ident).?; // const ptr = try self.genAlloc(toLLVMtype(nodeVar.expr.typ.?, table), nodeVar.ident.ident); const ptr = try self.genAlloc(toLLVMtype(nodeVar.expr.typ orelse try nodeVar.expr.inferType(self.allocator, table), table, nodeVar.expr), nodeVar.ident.ident); + std.debug.print("{any}\n", .{nodeVar.expr}); const value = try self.genExpr(nodeVar.expr); _ = core.LLVMBuildStore(self.builder, value, ptr); // std.debug.print("\t\t\tGenerated Value {s}\n", .{nodeVar.ident.ident}); @@ -156,7 +159,7 @@ pub const Generator = struct { block = fun.block.?; codeSlice = block.kind.block; } - const funcName: [*:0]const u8 = try self.allocator.dupeZ(u8, fun.ident.ident); + const funcName: [*:0]const u8 = "main"; const retType = toLLVMtype(fun.retType.?, table, null); var params = std.ArrayList(types.LLVMTypeRef).init(self.allocator); @@ -255,7 +258,7 @@ pub const Generator = struct { fn genExpr(self: *Generator, expr: parse.NodeExpr) !types.LLVMValueRef { return switch (expr.kind) { .ident => |id| blk: { - // std.debug.print("getValue({s})\n", .{id.ident}); + std.debug.print("getValue({s})\n", .{id.ident}); const table = expr.symtable; // std.debug.print("\n\nEXPERTABLE\n\n", .{}); @@ -322,11 +325,20 @@ pub const Generator = struct { const function = core.LLVMGetNamedFunction(self.module, ident); var args = std.ArrayList(types.LLVMValueRef).init(self.allocator); for (call.args.items, functype.input) |arg, intype| { - if (!std.meta.eql(expr.symtable.getType(arg.typ.?).?, intype)) return CodegenError.IncorrectType; + if (!std.meta.eql(expr.symtable.getType(arg.typ.?).?, intype)) return { + try util.comptimeError(.{ + .line = expr.line, + .err = CodegenError.IncorrectType, + .got = try intype.toString(self.allocator), + .exp = arg.typ.?.ident, + }); + break :blk CodegenError.IncorrectType; + }; try args.append(try self.genExpr(arg)); } + std.debug.print("FUNCTYPE: {s}\n", .{call.ident.ident}); + std.debug.print("{any}\n", .{function}); const funcType = core.LLVMGlobalGetValueType(function); - // std.debug.print("FUNCTYPE: {s}\n", .{call.ident.ident}); const llvmCall = core.LLVMBuildCall2( self.builder, diff --git a/src/main.zig b/src/main.zig index be9a2d8..cd4fde9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3,6 +3,7 @@ const tok = @import("tokenize.zig"); const parse = @import("parser.zig"); const gen = @import("codegen.zig"); const symb = @import("symtable.zig"); +pub var publicFileName: []const u8 = undefined; pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; @@ -22,6 +23,7 @@ pub fn main() !void { var args = std.process.args(); _ = args.skip(); const inputFileName = args.next(); + publicFileName = inputFileName.?; var out_name: []const u8 = "out"; if (std.os.argv.len == 3) out_name = args.next().?; @@ -35,9 +37,9 @@ pub fn main() !void { // Setup native code writer const outFileName = try getFileName(allocator, out_name, "ll"); defer allocator.free(outFileName); - const outfile = try std.fs.cwd().createFile(outFileName, .{}); - const outWriter = outfile.writer(); - defer outfile.close(); + // const outfile = try std.fs.cwd().createFile(outFileName, .{}); + // const outWriter = outfile.writer(); + // defer outfile.close(); // Turn the input file into a string const all = try inputFile.readToEndAlloc(allocator, 2048); @@ -66,18 +68,18 @@ pub fn main() !void { // Codegen const fname = try allocator.dupeZ(u8, inputFileName.?); defer allocator.free(fname); - var generator = gen.Generator.init(arena.allocator(), tree, @ptrCast(fname)); - defer generator.deinit(); - const code = try generator.generate(); + // var generator = gen.Generator.init(arena.allocator(), tree, @ptrCast(fname)); + // defer generator.deinit(); + // const code = try generator.generate(); // std.debug.print("{s}\n", .{code}); - try outWriter.writeAll(code); + // try outWriter.writeAll(); const binFile = try getFileName(allocator, out_name, ""); defer allocator.free(binFile); - const ldargv = [_][]const u8{ "clang", "-o", binFile, outFileName }; - const ldproc = try std.process.Child.run(.{ .argv = &ldargv, .allocator = allocator }); - defer allocator.free(ldproc.stdout); - defer allocator.free(ldproc.stderr); + // const ldargv = [_][]const u8{ "clang", "-o", binFile, outFileName }; + // const ldproc = try std.process.Child.run(.{ .argv = &ldargv, .allocator = allocator }); + // defer allocator.free(ldproc.stdout); + // defer allocator.free(ldproc.stderr); } /// Get file extension based on filename diff --git a/src/parser.zig b/src/parser.zig index bd4627d..4763e85 100644 --- a/src/parser.zig +++ b/src/parser.zig @@ -1,6 +1,7 @@ const std = @import("std"); const tok = @import("tokenize.zig"); const symb = @import("symtable.zig"); +const util = @import("utils.zig"); const Iterator = tok.Iterator; const Token = tok.Token; @@ -34,18 +35,19 @@ pub const Node = union(enum) { Expr: NodeExpr, Stmt: NodeStmt, pub fn children(self: Node, allocator: std.mem.Allocator) ![]Node { - var childrenArray = std.ArrayList(Node).init(allocator); - defer childrenArray.deinit(); + var childrenArray = std.ArrayList(Node){}; + defer childrenArray.deinit(allocator); switch (self) { - .Expr => |expr| try childrenArray.appendSlice(try expr.children(allocator)), - .Stmt => |stmt| try childrenArray.appendSlice(try stmt.children(allocator)), + .Expr => |expr| try childrenArray.appendSlice(allocator, try expr.children(allocator)), + .Stmt => |stmt| try childrenArray.appendSlice(allocator, try stmt.children(allocator)), } - return try childrenArray.toOwnedSlice(); + return try childrenArray.toOwnedSlice(allocator); } }; pub const NodeExpr = struct { id: u32, + line: u32, kind: ExprKind, symtable: *symb.SymbolTable, typ: ?TypeIdent, @@ -56,13 +58,14 @@ pub const NodeExpr = struct { } pub fn children(self: NodeExpr, allocator: std.mem.Allocator) ![]Node { - var childrenArray = std.ArrayList(Node).init(allocator); - defer childrenArray.deinit(); + var childrenArray = std.ArrayList(Node){}; + defer childrenArray.deinit(allocator); switch (self.kind) { else => {}, } - return try childrenArray.toOwnedSlice(); + return try childrenArray.toOwnedSlice(allocator); } + pub fn inferType(self: NodeExpr, allocator: std.mem.Allocator, table: *symb.SymbolTable) TypeError!TypeIdent { const expectedType = try switch (self.kind) { .call => |call| if (table.getValue(call.ident.ident)) |symbol| try symbol.typ.Function.output.toTypeIdent(allocator) else TypeError.UnknownIdentifier, @@ -72,23 +75,22 @@ pub const NodeExpr = struct { .stringLit => TypeIdent{ .ident = "[u8]", .list = true }, }; if (self.typ) |typ| { - return if (std.mem.eql(u8, typ.ident, expectedType.ident)) expectedType else TypeError.IncorrectType; + return if (std.mem.eql(u8, typ.ident, expectedType.ident)) expectedType else blk: { + util.comptimeError(.{ + .line = self.line, + .err = TypeError.IncorrectType, + .exp = expectedType.ident, + .got = typ.ident, + }); + break :blk TypeError.IncorrectType; + }; } else return expectedType; } }; -pub fn map(comptime T: type, comptime F: type, slice: []const F, func: fn (F) T) []const T { - var list: [64]T = undefined; - var max: usize = 0; - for (slice, 0..) |item, i| { - list[i] = func(item); - max = i + 1; - } - return list[0..max]; -} - pub const NodeStmt = struct { id: u32, + line: u32, kind: StmtKind, symtable: *symb.SymbolTable, @@ -97,23 +99,23 @@ pub const NodeStmt = struct { } pub fn children(self: NodeStmt, allocator: std.mem.Allocator) ![]Node { - var childrenArray = std.ArrayList(Node).init(allocator); - defer childrenArray.deinit(); + var childrenArray = std.ArrayList(Node){}; + defer childrenArray.deinit(allocator); switch (self.kind) { - .exit => |exit| try childrenArray.append(exit.asNode()), - .defValue => |value| try childrenArray.append(value.expr.asNode()), - .defVar => |variable| try childrenArray.append(variable.expr.asNode()), - .assignVar => |assign| try childrenArray.append(assign.expr.asNode()), + .exit => |exit| try childrenArray.append(allocator, exit.asNode()), + .defValue => |value| try childrenArray.append(allocator, value.expr.asNode()), + .defVar => |variable| try childrenArray.append(allocator, variable.expr.asNode()), + .assignVar => |assign| try childrenArray.append(allocator, assign.expr.asNode()), .block => |block| { - const blockChildren = map(Node, NodeStmt, block, NodeStmt.asNode); - for (blockChildren) |child| try childrenArray.append(child); + const blockChildren = util.map(Node, NodeStmt, block, NodeStmt.asNode); + for (blockChildren) |child| try childrenArray.append(allocator, child); }, - .ifstmt => |ifstmt| try childrenArray.append(ifstmt.body.*.asNode()), - .whileStmt => |whilestmt| try childrenArray.append(whilestmt.body.*.asNode()), - .function => |fun| if (fun.block == null) {} else try childrenArray.append(fun.block.?.asNode()), - .expr => |expr| try childrenArray.append(expr.asNode()), + .ifstmt => |ifstmt| try childrenArray.append(allocator, ifstmt.body.*.asNode()), + .whileStmt => |whilestmt| try childrenArray.append(allocator, whilestmt.body.*.asNode()), + .function => |fun| if (fun.block == null) {} else try childrenArray.append(allocator, fun.block.?.asNode()), + .expr => |expr| try childrenArray.append(allocator, expr.asNode()), } - return try childrenArray.toOwnedSlice(); + return try childrenArray.toOwnedSlice(allocator); } }; @@ -124,6 +126,7 @@ pub const Parser = struct { allocator: std.mem.Allocator, nodes: std.ArrayList(NodeStmt), nextId: u32 = 1, + line: u32 = 1, fn reserveId(self: *Parser) u32 { defer self.nextId += 1; @@ -135,7 +138,7 @@ pub const Parser = struct { .top = symbolTable, .allocator = allocator, .tokens = Iterator(Token).init(tokens), - .nodes = std.ArrayList(NodeStmt).init(allocator), + .nodes = std.ArrayList(NodeStmt){}, .id = 0, }; } @@ -173,9 +176,9 @@ pub const Parser = struct { fn parseCall(self: *Parser, typ: ?TypeIdent, expr: NodeExpr) ParsingError!NodeExpr { if (self.tokens.peek().? != .openParen) return expr; _ = try self.tokens.consume(.openParen); - var args = std.ArrayList(NodeExpr).init(self.allocator); + var args = std.ArrayList(NodeExpr){}; while (self.tokens.peek().? != .closeParen) { - try args.append(try self.parseExpr()); + try args.append(self.allocator, try self.parseExpr()); if (self.tokens.peek().? != .comma) break; } _ = try self.tokens.consume(.closeParen); @@ -184,6 +187,7 @@ pub const Parser = struct { return NodeExpr{ .kind = kind, .isConst = false, + .line = self.tokens.line, .typ = typ, .id = self.reserveId(), .symtable = self.top, @@ -212,15 +216,15 @@ pub const Parser = struct { .call = .{ .ident = ident, .args = innerblk: { - var argExprs = std.ArrayList(NodeExpr).init(self.allocator); + var argExprs = std.ArrayList(NodeExpr){}; while (!tok.checkType(self.tokens.peek().?, .closeParen)) { - try argExprs.append(try self.parseExpr()); + try argExprs.append(self.allocator, try self.parseExpr()); if (tok.checkType(self.tokens.peek().?, .closeParen)) break; _ = try self.tokens.consume(.comma); } _ = try self.tokens.consume(.closeParen); - break :innerblk try argExprs.clone(); + break :innerblk try argExprs.clone(self.allocator); }, }, }; @@ -243,6 +247,7 @@ pub const Parser = struct { const lhs = NodeExpr{ .id = self.reserveId(), + .line = self.tokens.line, .isConst = false, .kind = kind, .symtable = self.top, @@ -282,6 +287,7 @@ pub const Parser = struct { }; return NodeExpr{ .id = self.reserveId(), + .line = self.tokens.line, .isConst = false, .kind = kind, .symtable = self.top, @@ -303,7 +309,7 @@ pub const Parser = struct { break :blk try self.parseAssign(); break :blk try self.parseExprStmt(); }, - .openBrace => self.parseBlock(), + .openBrace => self.parseBlock(false), .fun => try self.parseFunc(false), .import => try self.parseFunc(true), else => self.parseExprStmt(), @@ -315,6 +321,7 @@ pub const Parser = struct { _ = try self.tokens.consume(.semiCol); return NodeStmt{ .id = self.reserveId(), + .line = self.tokens.line, .symtable = self.top, .kind = kind, }; @@ -337,6 +344,7 @@ pub const Parser = struct { }, }; return NodeStmt{ + .line = self.tokens.line, .id = self.reserveId(), .kind = kind, .symtable = self.top, @@ -360,6 +368,7 @@ pub const Parser = struct { }, }; return NodeStmt{ + .line = self.tokens.line, .id = self.reserveId(), .kind = kind, .symtable = self.top, @@ -368,12 +377,13 @@ pub const Parser = struct { fn parseFunc(self: *Parser, external: bool) ParsingError!NodeStmt { if (external) _ = try self.tokens.consume(.import); + var returns = false; var typ: ?TypeIdent = null; _ = try self.tokens.consume(.fun); const ident = (try self.tokens.consume(.ident)).?; _ = try self.tokens.consume(.openParen); - var args = std.ArrayList(FunctionArg).init(self.allocator); - defer args.deinit(); + var args = std.ArrayList(FunctionArg){}; + defer args.deinit(self.allocator); while (!tok.checkType(self.tokens.peek().?, .closeParen)) { const argIdent: Token = (try self.tokens.consume(.ident)).?; _ = try self.tokens.consume(.colon); @@ -382,12 +392,13 @@ pub const Parser = struct { .ident = argIdent.ident, .typ = argTypIdent, }; - try args.append(funcArg); + try args.append(self.allocator, funcArg); if (!tok.checkType(self.tokens.peek().?, .comma)) break; _ = try self.tokens.consume(.comma); } _ = try self.tokens.consume(.closeParen); if (tok.checkType(self.tokens.peek().?, .arrow)) { + returns = true; self.tokens.skip(); typ = try self.parseType(); } @@ -395,11 +406,12 @@ pub const Parser = struct { if (external) { _ = try self.tokens.consume(.semiCol); return NodeStmt{ + .line = self.tokens.line, .id = self.reserveId(), .kind = StmtKind{ .function = .{ .ident = ident, - .args = try args.toOwnedSlice(), + .args = try args.toOwnedSlice(self.allocator), .retType = typ, .block = null, }, @@ -409,12 +421,12 @@ pub const Parser = struct { } const block = try self.allocator.create(NodeStmt); - block.* = try self.parseBlock(); + block.* = try self.parseBlock(returns); const kind = StmtKind{ .function = .{ .ident = ident, - .args = try args.toOwnedSlice(), + .args = try args.toOwnedSlice(self.allocator), .retType = typ, .block = block, }, @@ -422,25 +434,38 @@ pub const Parser = struct { return NodeStmt{ .id = self.reserveId(), + .line = self.tokens.line, .kind = kind, .symtable = self.top, }; } - fn parseBlock(self: *Parser) ParsingError!NodeStmt { + fn parseBlock(self: *Parser, returns: bool) ParsingError!NodeStmt { _ = try self.tokens.consume(.openBrace); - var stmtArr = std.ArrayList(NodeStmt).init(self.allocator); + var stmtArr = std.ArrayList(NodeStmt){}; const child = try self.top.makeChild(); self.top = child; while (!tok.checkType(self.tokens.peek().?, .closeBrace)) - try stmtArr.append(try self.parseStmt()); + try stmtArr.append(self.allocator, try self.parseStmt()); self.top = self.top.parent().?; _ = try self.tokens.consume(.closeBrace); + if (returns) { + if (blk: { + for (stmtArr.items) |it| { + if (switch (it.kind) { + .exit => true, + else => false, + }) break :blk true; + } + break :blk false; + }) {} else return ParsingError.ExpectedExit; + } const kind = StmtKind{ - .block = try stmtArr.toOwnedSlice(), + .block = try stmtArr.toOwnedSlice(self.allocator), }; return NodeStmt{ + .line = self.tokens.line, .id = self.reserveId(), .kind = kind, .symtable = child, @@ -459,6 +484,7 @@ pub const Parser = struct { }, }; return NodeStmt{ + .line = self.tokens.line, .id = self.reserveId(), .kind = kind, .symtable = self.top, @@ -471,6 +497,7 @@ pub const Parser = struct { _ = try self.tokens.consume(.semiCol); const kind = StmtKind{ .exit = expr }; return NodeStmt{ + .line = self.tokens.line, .symtable = self.top, .kind = kind, .id = self.reserveId(), @@ -495,6 +522,7 @@ pub const Parser = struct { .defVar = NodeVar{ .ident = ident, .expr = NodeExpr{ + .line = self.tokens.line, .typ = typ, .id = expr.id, .kind = expr.kind, @@ -504,6 +532,7 @@ pub const Parser = struct { }, }; return NodeStmt{ + .line = self.tokens.line, .id = self.reserveId(), .kind = kind, .symtable = self.top, @@ -511,7 +540,7 @@ pub const Parser = struct { } fn parseType(self: *Parser) ParsingError!TypeIdent { - const list = tok.checkType(self.tokens.peek().?, .openBracket); + const list = tok.checkType(self.tokens.peekAhead(0).?, .openBracket); if (list) { _ = try self.tokens.consume(.openBracket); const typ = (try self.parseType()).ident; @@ -530,11 +559,15 @@ pub const Parser = struct { fn parseConstant(self: *Parser) ParsingError!NodeStmt { _ = try self.tokens.consume(.constant); var typ: ?TypeIdent = null; - _ = if (self.tokens.consume(.colon)) |_| { + std.debug.print("{any}\n", .{self.tokens.peek().?}); + _ = if (tok.checkType(self.tokens.peek().?, .colon)) { + _ = try self.tokens.consume(.colon); typ = try self.parseType(); - } else |err| { - if (err != tok.TokenizeError.ExpectedToken) return err; }; + // else { + // // return; + // // if (err != tok.TokenizeError.ExpectedToken) return err; + // }; const ident = (try self.tokens.consume(.ident)).?; _ = try self.tokens.consume(.equal); @@ -544,6 +577,7 @@ pub const Parser = struct { .defValue = NodeValue{ .ident = ident, .expr = NodeExpr{ + .line = self.tokens.line, .typ = typ orelse expr.typ, .id = expr.id, .kind = expr.kind, @@ -553,6 +587,7 @@ pub const Parser = struct { }, }; return NodeStmt{ + .line = self.tokens.line, .id = self.reserveId(), .kind = kind, .symtable = self.top, @@ -561,9 +596,10 @@ pub const Parser = struct { pub fn parse(self: *Parser) !NodeStmt { while (self.tokens.peek()) |_| - try self.nodes.append(try self.parseStmt()); + try self.nodes.append(self.allocator, try self.parseStmt()); return NodeStmt{ + .line = self.tokens.line, .id = self.reserveId(), .kind = StmtKind{ .block = self.nodes.items }, .symtable = self.top, @@ -679,8 +715,10 @@ test "Parser" { .Stmt = NodeStmt{ .id = 2, .symtable = symbTable, + .line = 0, .kind = StmtKind{ .exit = NodeExpr{ + .line = 0, .id = 1, .kind = ExprKind{ .intLit = Token{ .intLit = 120 }, diff --git a/src/symtable.zig b/src/symtable.zig index f426ef8..65fedc4 100644 --- a/src/symtable.zig +++ b/src/symtable.zig @@ -254,14 +254,14 @@ pub const Populator = struct { args: []const pars.FunctionArg, retType: ?pars.TypeIdent, ) !Symbol { - var inputArr = std.ArrayList(SymbType).init(self.allocator); + var inputArr = std.ArrayList(SymbType){}; for (args) |arg| { // std.debug.print("{s}: {s}\n", .{ arg.ident, arg.typ.ident }); const argSymb = try self.buildValueSymb(table, arg.typ, false); if (!try table.insert(arg.ident, argSymb)) return error.FailedToInsert; - try inputArr.append(table.getType(arg.typ) orelse SymbType.Void); + try inputArr.append(self.allocator, table.getType(arg.typ) orelse SymbType.Void); } - const input = try inputArr.toOwnedSlice(); + const input = try inputArr.toOwnedSlice(self.allocator); const output = try self.allocator.create(SymbType); output.* = if (retType) |typ| table.getType(typ).? else SymbType.Void; diff --git a/src/tokenize.zig b/src/tokenize.zig index 93d1f87..290823b 100644 --- a/src/tokenize.zig +++ b/src/tokenize.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const u = @import("utils.zig"); pub const TokenizeError = error{ UnknownToken, @@ -41,6 +42,7 @@ pub const TokenType = enum { arrow, colon, comma, + newLine, }; pub const Token = union(TokenType) { @@ -77,6 +79,7 @@ pub const Token = union(TokenType) { arrow, colon, comma, + newLine, pub fn fromChar(char: u8) !Token { return switch (char) { @@ -96,6 +99,7 @@ pub const Token = union(TokenType) { ':' => .colon, '[' => .openBracket, ']' => .closeBracket, + '\n' => .newLine, else => errblk: { std.debug.print("{c}: ", .{char}); break :errblk TokenizeError.UnknownToken; @@ -146,6 +150,7 @@ pub const Token = union(TokenType) { .arrow => "ARROW", .colon => "COLON", .comma => "COMMA", + .newLine => "\n", }; } }; @@ -162,6 +167,7 @@ pub fn Iterator(comptime typ: type) type { return struct { items: []const typ, index: usize = 0, + line: u32 = 1, /// Initialize tokenizer with a slice pub fn init(items: []const typ) Iterator(typ) { @@ -169,13 +175,20 @@ pub fn Iterator(comptime typ: type) type { } /// Get current item - pub fn peekAhead(self: Iterator(typ), ahead: u32) ?typ { + pub fn peekAhead(self: *Iterator(typ), ahead: u32) ?typ { if (self.index + ahead >= self.items.len) return null; + if (typ == Token) { + if (self.items[self.index + ahead] == .newLine) { + self.line += 1; + self.skip(); + return self.peekAhead(ahead); + } + } return self.items[self.index + ahead]; } - pub fn peek(self: Iterator(typ)) ?typ { - return peekAhead(self, 0); + pub fn peek(self: *Iterator(typ)) ?typ { + return self.peekAhead(0); } /// Get current item and iterate index @@ -188,6 +201,8 @@ pub fn Iterator(comptime typ: type) type { pub fn consume(self: *Iterator(typ), comptime expected: TokenType) error{ ExpectedToken, TokenIteratorOnly }!?typ { if (typ != Token) return TokenizeError.TokenIteratorOnly; if (!checkType(self.peek().?, expected)) { + // std.debug.print("Expected {}, got {}\n", .{ expected, self.peek().? }); + u.comptimeError(.{ .line = 0, .err = TokenizeError.ExpectedToken, .exp = expected, .got = self.peek().?, }); return TokenizeError.ExpectedToken; } return self.next(); @@ -202,6 +217,7 @@ pub fn Iterator(comptime typ: type) type { /// Tokenizes a string of source code pub const Tokenizer = struct { + line: u32 = 1, src: Iterator(u8), allocator: std.mem.Allocator, toks: std.ArrayList(Token), @@ -212,7 +228,7 @@ pub const Tokenizer = struct { return Tokenizer{ .src = Iterator(u8).init(src), .allocator = allocator, - .toks = std.ArrayList(Token).init(allocator), + .toks = std.ArrayList(Token){}, }; } @@ -224,64 +240,64 @@ pub const Tokenizer = struct { if (checkType(token, .stringLit)) self.allocator.free(token.stringLit); } - self.toks.deinit(); + self.toks.deinit(self.allocator); } /// Returns an ArrayList of tokens pub fn tokenize(self: *Tokenizer) ![]Token { - var buff = std.ArrayList(u8).init(self.allocator); - defer buff.deinit(); + var buff = std.ArrayList(u8){}; + defer buff.deinit(self.allocator); while (self.src.peek()) |char| { try switch (char) { '=' => { self.src.skip(); if (self.src.peek().? != '=') { - try self.toks.append(.equal); + try self.toks.append(self.allocator, .equal); continue; } self.src.skip(); - try self.toks.append(.eqleql); + try self.toks.append(self.allocator, .eqleql); }, '-' => { self.src.skip(); if (self.src.peek().? != '>') { - try self.toks.append(.minus); + try self.toks.append(self.allocator, .minus); continue; } self.src.skip(); - try self.toks.append(.arrow); + try self.toks.append(self.allocator, .arrow); }, - ' ', '\n', '\t' => self.src.skip(), + ' ', '\t' => self.src.skip(), '0'...'9' => { while (std.ascii.isDigit(self.src.peek().?)) - try buff.append(self.src.next().?); + try buff.append(self.allocator, self.src.next().?); const num: i32 = try std.fmt.parseInt(i32, buff.items, 10); - try self.toks.append(.{ .intLit = num }); - buff.clearAndFree(); + try self.toks.append(self.allocator, .{ .intLit = num }); + buff.clearAndFree(self.allocator); }, 'a'...'z', 'A'...'Z' => { while (std.ascii.isAlphanumeric(self.src.peek().?)) - try buff.append(self.src.next().?); - const str = try buff.toOwnedSlice(); + try buff.append(self.allocator, self.src.next().?); + const str = try buff.toOwnedSlice(self.allocator); const token = Token.fromStr(str); - try self.toks.append(token); + try self.toks.append(self.allocator, token); if (!checkType(token, TokenType.ident)) self.allocator.free(str); - buff.clearAndFree(); + buff.clearAndFree(self.allocator); }, '"' => { _ = self.src.next(); while (self.src.peek().? != '"') - try buff.append(self.src.next().?); + try buff.append(self.allocator, self.src.next().?); _ = self.src.next(); // std.debug.print("{c}\n", .{self.src.peek().?}); - const token = Token{ .stringLit = try buff.toOwnedSlice() }; - try self.toks.append(token); - buff.clearAndFree(); + const token = Token{ .stringLit = try buff.toOwnedSlice(self.allocator) }; + try self.toks.append(self.allocator, token); + buff.clearAndFree(self.allocator); }, - else => self.toks.append(try Token.fromChar(self.src.next().?)), + else => self.toks.append(self.allocator, try Token.fromChar(self.src.next().?)), }; } return self.toks.items; diff --git a/src/utils.zig b/src/utils.zig new file mode 100644 index 0000000..03ab8ef --- /dev/null +++ b/src/utils.zig @@ -0,0 +1,20 @@ +const std = @import("std"); +const m = @import("main.zig"); + +pub fn comptimeError(err: anytype) void { + // const stderr = std.fs.File.stderr(); + // var buff: [256]u8 = undefined; + // const errWriter = stderr.writer(&buff); + // var errWriterInterface = errWriter.interface; + std.debug.print("{s}:{d}: {any}: Expected '{any}' got '{any}'\n", .{ m.publicFileName, err.line, err.err, err.exp, err.got }); +} + +pub fn map(comptime T: type, comptime F: type, slice: []const F, func: fn (F) T) []const T { + var list: [64]T = undefined; + var max: usize = 0; + for (slice, 0..) |item, i| { + list[i] = func(item); + max = i + 1; + } + return list[0..max]; +} -- cgit v1.2.3