Lots of refactoring and implementing a good macro calling system, next: macro definitions

examples/testing/src/main.zk

@@ -1,30 +1,23 @@
 #llvm(
     #"@int.fmt = private unnamed_addr constant [14 x i8] c"Integer: %ld\0A\00""#
 )
-
 #llvm(
     #"@float.fmt = private unnamed_addr constant [12 x i8] c"Float: %lf\0A\00""#
 )
 
-#llvm(
-    "declare i32 @printf(ptr, ...)"
-)
-
-fn test() -> i64 {
-    return 12 + 2
-}
+#llvm("declare i32 @printf(ptr, ...)")
 
 fn main() -> void {
-    val test: i64 = 123 * 2
-    val test2: f64 = 5.0 * 0.5 + 2.0
+    val test: i64 = 123
+    val test2: f64 = 12.3 + 5.6 * 3.2
 
     #llvm(
-        "%call = call i32 (ptr, ...) @printf(ptr @int.fmt, i64 %v_42_test)"
+        "%call.{:id} = call i32 (ptr, ...) @printf(ptr @int.fmt, {0:ty} {0})",
+        test
     )
-
     #llvm(
-        "%call.2 = call i32 (ptr, ...) @printf(ptr @float.fmt, double %v_52_test2)"
+        "%call.{:id} = call i32 (ptr, ...) @printf(ptr @float.fmt, {0:ty} {0})",
+        test2
     )
-    
     return
 }

src/zelkelbt/__main__.py

@@ -91,17 +91,4 @@ if __name__ == "__main__":
         build_package("debug", config, args.debug_info)
         subprocess.run([f"./target/debug/{config['package']['name']}"], check=True)
     else:
-        parser.print_help()
-
-
-# tokens = lex(source)
-# print(tokens)
-# parser = Parser(tokens, 0)
-# ast = parser.parse()
-# rich.print(ast)
-
-# Validator(ast)
-
-# codegen = Codegen(ast)
-# ir = codegen.generate()
-# print(ir)
+        parser.print_help()

src/zelkelbt/build.py

@@ -1,3 +1,4 @@
+import itertools
 import subprocess
 from pathlib import Path
 
@@ -5,10 +6,12 @@ import rich
 
 from src.zelkelc.ast.parser import Parser
 from src.zelkelc.codegen.codegen import Codegen
+from src.zelkelc.expander.expander import Expander
 from src.zelkelc.lexer.tokenizer import lex
 from src.zelkelc.validator.validator import Validator
 
 PROJECT_ROOT = Path(__file__).resolve().parents[2]
+ids = itertools.count()
 
 
 def _resolve_project_root() -> Path:
@@ -41,11 +44,12 @@ def build_package(build_mode: str, config: dict[str, any], debug_info: bool):
     tokens = lex(source)
     if debug_info:
         print(tokens)
-    parser = Parser(tokens, 0)
+    parser = Parser(tokens, 0, ids)
     ast = parser.parse()
     if debug_info:
         rich.print(ast)
 
+    ast = Expander(ids).run(ast)
     Validator(ast)
 
     codegen = Codegen(ast)

src/zelkelbt/config.py

@@ -6,12 +6,13 @@ from pathlib import Path
 def has_config():
     return Path("zelkel.toml").is_file()
 
+
 def get_config():
     if not has_config():
         print("Not in a Zelkel project: No zelkel.toml found")
         sys.exit(1)
-    
+
     with open("zelkel.toml", "rb") as f:
         data = tomllib.load(f)
-    
-    return data
+
+    return data

src/zelkelc/ast/nodes.py

@@ -1,74 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-import src.zelkelc.ast.values as values
-
-
-@dataclass
-class ClassDeclaration:
-    name: str
-    members: list[ValueDeclaration]
-    methods: list[FunctionDeclaration]
-    irname: str
-
-
-@dataclass
-class StructDeclaration:
-    name: str
-    members: list[MemberDeclaration]
-    irname: str
-
-
-@dataclass
-class MemberDeclaration:
-    name: str
-    mutable: bool
-    typ: values.Value
-    real_idx: int
-
-
-@dataclass
-class ValueDeclaration:
-    name: str
-    mutable: bool
-    typ: values.Value
-    expr: Expression
-    irname: str
-
-
-@dataclass
-class FunctionDeclaration:
-    name: str
-    typ: values.Value
-    args: dict[str, values.Value]
-    body: list[Node]
-    irname: str
-
-
-@dataclass
-class ReturnStatement:
-    expr: Expression | None
-
-
-Node = ClassDeclaration | ValueDeclaration | FunctionDeclaration | StructDeclaration
-
-
-@dataclass
-class BinaryExpression:
-    left: Expression
-    right: Expression
-    op: str
-
-
-@dataclass
-class UnaryExpression:
-    expr: Expression
-    sign: str
-
-
-@dataclass
-class PrimaryExpression:
-    value: values.Value
-
-
-Expression = BinaryExpression | UnaryExpression | PrimaryExpression

src/zelkelc/ast/parser.py

@@ -1,36 +1,44 @@
-import sys
-from typing import Optional
+from itertools import count
 
 from src.zelkelc.ast import types
 from src.zelkelc.lexer import token
 
 
+class ParseError(Exception):
+    pass
+
+
 class Parser:
     tokens: list[token.Token]
     cursor: int
+    ids: count
 
-    def __init__(self, tokens: list[token.Token], cursor: int):
+    def __init__(self, tokens: list[token.Token], cursor: int, ids: count):
         self.tokens = tokens
         self.cursor = cursor
+        self.ids = ids
 
     def expect(self, tok: token.Token):
         if self.tokens[self.cursor] != tok:
-            print(f"Expected {tok}, but found {self.tokens[self.cursor]}")
-            sys.exit(1)
+            raise ParseError(f"Expected {tok}, but found {self.tokens[self.cursor]}")
         self.cursor += 1
 
     def expect_type(self, tok_type: token.Token) -> token.Token:
         if type(self.tokens[self.cursor]) is not tok_type:
-            print(f"Expected {tok_type}, but found {type(self.tokens[self.cursor])}")
-            sys.exit(1)
+            raise ParseError(
+                f"Expected {tok_type}, but found {type(self.tokens[self.cursor])}"
+            )
         self.cursor += 1
         return self.tokens[self.cursor - 1]
 
-    def current(self, tok: token.Token):
-        return self.tokens[self.cursor] == tok
+    def current(self, tok) -> bool:
+        return self.cursor < len(self.tokens) and self.tokens[self.cursor] == tok
 
-    def current_type(self, tok_type: token.Token) -> bool:
-        return type(self.tokens[self.cursor]) is tok_type
+    def current_type(self, tok_type) -> bool:
+        return (
+            self.cursor < len(self.tokens)
+            and type(self.tokens[self.cursor]) is tok_type
+        )
 
     @staticmethod
     def type_to_str(typ: types.Value, is_return_type: bool) -> types.Value:
@@ -44,8 +52,7 @@ class Parser:
             case types.Void if is_return_type:
                 return "void"
             case _:
-                print(f"Invalid type found in type_to_str: {typ}")
-                sys.exit(1)
+                raise ParseError(f"Invalid type found in type_to_str: {typ}")
 
     def parse_type(self, is_return_type: bool) -> types.Value:
         identifier = self.expect_type(token.Identifier)
@@ -59,53 +66,69 @@ class Parser:
             case "void" if is_return_type:
                 return types.Void
             case _:
-                print(f"Invalid type found in parse_type: {identifier.value}")
-                sys.exit(1)
+                raise ParseError(
+                    f"Invalid type found in parse_type: {identifier.value}"
+                )
 
-    def parse(self):
-        ast: list[types.FunctionDeclaration | types.ClassDeclaration] = []
+    def parse_item(self, path: str = "_main.zk_") -> types.Node:
+        t = self.tokens[self.cursor]
+
+        match t:
+            case token.Identifier():
+                match t.value:
+                    case "fn":
+                        return self.parse_function_declaration(path)
+                    case "struct":
+                        return self.parse_struct_declaration(path)
+                    case "class":
+                        return self.parse_class_declaration(path)
+                    case _:
+                        raise ParseError(f"Invalid keyword in global {t.value}")
+            case token.Pound():
+                return self.parse_pound()
+            case _:
+                raise ParseError(f"Invalid token found while parsing global {t}")
 
+    def parse_items(self) -> list[types.Node]:
+        items = []
         while self.cursor < len(self.tokens):
-            t = self.tokens[self.cursor]
+            items.append(self.parse_item())
+        return items
 
-            match t:
-                case token.Identifier():
-                    match t.value:
-                        case "fn":
-                            ast.append(
-                                self.parse_function_declaration("_main.zk_")
-                            )  # TODO: replace with actual filepath to the file
-                        case "struct":
-                            ast.append(self.parse_struct_declaration("_main.zk_"))
-                        case "class":
-                            ast.append(self.parse_class_declaration("_main.zk_"))
-                        case _:
-                            print(f"Invalid keyword in global {t.value}")
-                            sys.exit(1)
-                case token.Pound():  # TODO: hard coded to that all macro calls are treated as #llvm() macro calls
-                    self.expect(token.Pound())
-                    ident = self.expect_type(token.Identifier)
+    def parse(self):
+        return self.parse_items()
 
-                    match ident.value:
-                        case "fn":
-                            print("Macro declaration is not implemented")
-                            sys.exit(1)
-                        case _:
-                            ast.append(self.parse_macro_call(ident))
-                case _:
-                    print(f"Invalid token found while parsing global {t}")
-                    sys.exit(1)
+    def take_delimited(self) -> list[token.Token]:
+        if self.current(token.LParen()):
+            open_t, _close_t = token.LParen(), token.RParen()
+        elif self.current(token.LBrace()):
+            open_t, _close_t = token.LBrace(), token.RBrace()
+        else:
+            raise ParseError("Expected '(' or '{' to start delimited token tree")
+        self.expect(open_t)
+        depth, start = 1, self.cursor
+        while depth:
+            if self.cursor >= len(self.tokens):
+                raise ParseError("unclosed delimited group")
+            t = self.tokens[self.cursor]
+            if isinstance(t, (token.LParen, token.LBrace)):
+                depth += 1
+            elif isinstance(t, (token.RParen, token.RBrace)):
+                depth -= 1
+            self.cursor += 1
+        return self.tokens[start : self.cursor - 1]
 
-        return ast
+    def parse_pound(self) -> types.Node:
+        self.expect(token.Pound())
+        ident = self.expect_type(token.Identifier)
+        if ident.value == "fn":
+            # return self.parse_macro_definition()
+            raise ParseError("Implemented parse_macro_definition")
+        return self.parse_macro_call(ident)
 
     def parse_macro_call(self, ident: token.Identifier) -> types.MacroCall:
-        # Hard core builtin #llvm macro block
-
-        self.expect(token.LParen())
-        body = self.expect_type(token.String)
-        self.expect(token.RParen())
-
-        return types.MacroCall(body.value)
+        args = self.take_delimited()
+        return types.MacroCall(name=ident.value, args=args)
 
     def parse_struct_declaration(self, path: str) -> types.StructDeclaration:
         self.expect(token.Identifier("struct"))
@@ -132,11 +155,13 @@ class Parser:
                                 self.parse_member_declaration(True, len(members))
                             )
                         case _:
-                            print(f"Invalid keyword in struct body {t.value}")
-                            sys.exit(1)
+                            raise ParseError(
+                                f"Invalid keyword in struct body {t.value}"
+                            )
                 case _:
-                    print(f"Invalid token found while parsing class declaration: {t}")
-                    sys.exit(1)
+                    raise ParseError(
+                        f"Invalid token found while parsing class declaration: {t}"
+                    )
 
         self.expect(token.RBrace())
 
@@ -187,16 +212,46 @@ class Parser:
                                 self.parse_member_declaration(True, len(members))
                             )
                         case _:
-                            print(f"Invalid keyword in class body {t.value}")
-                            sys.exit(1)
+                            raise ParseError(f"Invalid keyword in class body {t.value}")
                 case _:
-                    print(f"Invalid token found while parsing class declaration: {t}")
-                    sys.exit(1)
+                    raise ParseError(
+                        f"Invalid token found while parsing class declaration: {t}"
+                    )
 
         self.expect(token.RBrace())
 
         return types.ClassDeclaration(name, members, methods, irname=f"{path}.c_{name}")
 
+    def parse_statement(self, path: str) -> types.Node:
+        t = self.tokens[self.cursor]
+
+        match t:
+            case token.Identifier():
+                match t.value:
+                    case "fn":
+                        return self.parse_function_declaration(path)
+                    case "val":
+                        return self.parse_value_declaration(False)
+                    case "var":
+                        return self.parse_value_declaration(True)
+                    case "return":
+                        return self.parse_return()
+                    case _:
+                        raise ParseError(f"Invalid keyword in function body {t.value}")
+            case token.Pound():
+                return self.parse_pound()
+            case _:
+                # TODO: expression statement
+                raise ParseError(f"implement expression statement: {t}")
+
+    def parse_statements(self, path: str, stop_at_rbrace: bool) -> list[types.Node]:
+        body = []
+        while self.cursor < len(self.tokens):
+            if stop_at_rbrace and self.current(token.RBrace()):
+                break
+            body.append(self.parse_statement(path))
+        return body
+
     def parse_function_declaration(self, path: str) -> types.FunctionDeclaration:
         self.expect(token.Identifier("fn"))
         name = self.expect_type(token.Identifier).value
@@ -221,46 +276,7 @@ class Parser:
         typ = self.parse_type(True)
 
         self.expect(token.LBrace())
-
-        body: list[types.Node] = []
-
-        while (
-            self.cursor < len(self.tokens)
-            and self.tokens[self.cursor] != token.RBrace()
-        ):
-            t = self.tokens[self.cursor]
-
-            match t:
-                case token.Identifier():
-                    match t.value:
-                        case "fn":
-                            body.append(
-                                self.parse_function_declaration(f"{path}.f_{name}")
-                            )
-                        case "val":
-                            body.append(self.parse_value_declaration(False))
-                        case "var":
-                            body.append(self.parse_value_declaration(True))
-                        case "return":
-                            body.append(self.parse_return())
-                        case _:
-                            print(f"Invalid keyword in function body {t.value}")
-                            sys.exit(1)
-                case token.Pound():  # TODO: hard coded to that all macro calls are treated as #llvm() macro calls
-                    self.expect(token.Pound())
-                    ident = self.expect_type(token.Identifier)
-
-                    match ident.value:
-                        case "fn":
-                            print("Macro declaration is not implemented")
-                            sys.exit(1)
-                        case _:
-                            body.append(self.parse_macro_call(ident))
-                case _:
-                    # TODO: expression statement
-                    print(f"implement expression statement: {t}")
-                    sys.exit(1)
-
+        body = self.parse_statements(f"{path}.f_{name}", stop_at_rbrace=True)
         self.expect(token.RBrace())
 
         irname = f"{path}.f_{name}_{self.type_to_str(typ, True)}"
@@ -296,7 +312,7 @@ class Parser:
         expr = self.parse_expression()
 
         return types.ValueDeclaration(
-            name, mutable, typ, expr, irname=f"v_{self.cursor}_{name}"
+            name, mutable, typ, expr, irname=f"v_{next(self.ids)}_{name}"
         )
 
     def parse_expression(self) -> types.Expression:
@@ -326,14 +342,13 @@ class Parser:
             op = self.expect_type(token.Operator).value
 
             if op not in ("-", "+"):
-                print(f"Invalid unary operator: {op}")
-                sys.exit(1)
+                raise ParseError(f"Invalid unary operator: {op}")
             operand = self.parse_unary_expression()
             return types.UnaryExpression(operand, op, operand.typ)
 
         return self.parse_primary_expression()
 
-    def parse_primary_expression(self) -> Optional[types.Expression]:
+    def parse_primary_expression(self) -> types.Expression | None:
         t = self.tokens[self.cursor]
 
         match t:
@@ -380,6 +395,10 @@ class Parser:
                 self.expect(token.RParen())
                 return expr
 
+            case token.Interpolated():
+                self.cursor += 1
+                return t.node
+
             case _:
                 return None
 

src/zelkelc/ast/types.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
-from dataclasses import dataclass
 import sys
-from typing import Optional
+from dataclasses import dataclass
+
+from src.zelkelc.lexer import token
 
 
 @dataclass
@@ -48,12 +49,27 @@ class FunctionDeclaration:
 
 @dataclass
 class ReturnStatement:
-    expr: Optional[Expression]
+    expr: Expression | None
+
+
+@dataclass
+class MacroDefinition:
+    name: str
+    params: list[tuple[str, str]]
+    body: list[token.Token]
 
 
 @dataclass
 class MacroCall:
-    body: str
+    name: str
+    args: list[token.Token]
+
+
+@dataclass
+class InlineIR:
+    template: str
+    args: list[Expression]
+    expansion_id: int
 
 
 Node = (
@@ -61,7 +77,9 @@ Node = (
     | ValueDeclaration
     | FunctionDeclaration
     | StructDeclaration
+    | MacroDefinition
     | MacroCall
+    | InlineIR
 )
 
 

src/zelkelc/codegen/codegen.py

@@ -1,6 +1,7 @@
 import sys
 
 from src.zelkelc.ast import types
+from src.zelkelc.expander.expander import render_llvm
 
 
 class IRTable:
@@ -37,8 +38,8 @@ class Codegen:
                     self.gen_class_declaration(node)
                 case types.FunctionDeclaration():
                     self.ir.global_functions += self.gen_function_declaration(node)
-                case types.MacroCall():
-                    self.ir.top = rf"{node.body}" + "\n" + self.ir.top
+                case types.InlineIR():
+                    self.ir.top = self.gen_inline_ir(node, is_top=True) + "\n" + self.ir.top
                 case _:
                     print(f"Invalid node found during codegen {type(node)}")
                     sys.exit(1)
@@ -63,6 +64,37 @@ class Codegen:
         self.ir.bottom += "call void @zkmain()\n"
         self.ir.bottom += "ret i32 0\n"
         self.ir.bottom += "}\n"
+        
+    def gen_inline_ir(self, node: types.InlineIR, is_top: bool) -> str:
+        vals: list[str] = []
+        tys: list[str] = []
+        prefix = ""
+        for arg in node.args:
+            reg, expr_ir = self.gen_expression(arg)
+            prefix += expr_ir
+            vals.append(f"%{reg}" if not str(reg).startswith("%") else str(reg))
+
+            typ = arg.typ
+            if (typ is types.VariableRef or typ is types.FunctionCall) and (
+                isinstance(arg, types.PrimaryExpression)
+                and getattr(arg.value, "typ", None) is not None
+            ):
+                typ = arg.value.typ
+            if typ is types.Integer64 or isinstance(typ, types.Integer64):
+                tys.append("i64")
+            elif typ is types.Float64 or isinstance(typ, types.Float64):
+                tys.append("double")
+            elif typ is types.Void or isinstance(typ, types.Void):
+                tys.append("void")
+            else:
+                tys.append("i64")
+
+        rendered = render_llvm(node.template, vals, tys, node.expansion_id)
+        if is_top:
+            return rendered
+        return (
+            prefix + ("\n" if prefix and not prefix.endswith("\n") else "") + rendered
+        )
 
     def gen_struct_declaration(self, node: types.StructDeclaration):
         members_str = ""
@@ -93,8 +125,8 @@ class Codegen:
             match child_node:
                 case types.ValueDeclaration():
                     ir += self.gen_value_declaration(child_node)
-                case types.MacroCall():
-                    ir += "\n" + rf"{child_node.body}"
+                case types.InlineIR():
+                    ir += "\n" + self.gen_inline_ir(child_node, is_top=False)
                 case types.ReturnStatement():
                     ir += self.gen_return_statement(child_node)
                 case _:
@@ -129,20 +161,20 @@ class Codegen:
                 sys.exit(1)
 
         return ir
-    
+
     def gen_return_statement(self, node: types.ReturnStatement) -> str:
         print(node)
         print(node.expr)
-        
+
         ir = ""
-        
+
         if node.expr:
             count, expr = self.gen_expression(node.expr)
             ir += expr
             ir += f"\nret {node.expr.typ.to_llvm()} %{count}"
         else:
             ir += "\nret void"
-            
+
         return ir
 
     def make_i64_op(self, sign: str) -> str:

src/zelkelc/lexer/token.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from typing import Any
 
 
 @dataclass
@@ -31,6 +32,16 @@ class LParen:
     pass
 
 
+@dataclass
+class Dollar:
+    pass
+
+
+@dataclass
+class Interpolated:
+    node: Any
+
+
 @dataclass
 class RParen:
     pass
@@ -91,4 +102,6 @@ Token = (
     | Colon
     | Equals
     | Operator
+    | Dollar
+    | Interpolated
 )

src/zelkelc/lexer/tokenizer.py

@@ -137,6 +137,10 @@ def lex(source: str) -> list[token.Token]:
                     i += 1
                 case "%":
                     tokens.append(token.Operator("%"))
+                    i += 1
+                case "$":
+                    tokens.append(token.Dollar())
+                    i += 1
                 case _:
                     print(f"Unknown character found during lexing: '{c}' at {i}")
                     break

src/zelkelc/validator/scope.py

@@ -153,8 +153,11 @@ class ScopeChecker:
                         self._check_function(method, self.global_scope)
                 case types.StructDeclaration():
                     continue
+                case types.InlineIR():
+                    for arg in top_level.args:
+                        self._infer_expression_type(arg, self.global_scope)
                 case types.MacroCall():
-                    continue
+                    self._error(f"unexpanded macro call #{top_level.name}")
                 case _:
                     self._error(f"Unsupported top-level node: {type(top_level)}")
 
@@ -263,7 +266,7 @@ class ScopeChecker:
                         )
                     )
 
-                case types.MacroCall():
+                case types.MacroCall() | types.InlineIR():
                     continue
 
                 case _:
@@ -363,8 +366,9 @@ class ScopeChecker:
                         )
                     self._check_function(child, function_scope)
 
-                case types.MacroCall():
-                    continue
+                case types.InlineIR():
+                    for arg in child.args:
+                        self._infer_expression_type(arg, function_scope)
 
                 case _:
                     self._error(