Initial work on `core:text/template`

core/text/template/execute.odin

@@ -0,0 +1,670 @@
+package text_template
+
+import "core:io"
+import "core:reflect"
+import "core:strings"
+import "core:mem"
+import "core:mem/virtual"
+import "core:fmt"
+
+import parse "parse"
+
+Template :: struct {
+	tree: ^parse.Tree,
+
+	using config: Config,
+}
+
+Config :: struct {
+	flags:       Flags,
+	allocator:   mem.Allocator,
+	left_delim:  string,
+	right_delim: string,
+}
+
+Flags :: distinct bit_set[Flag; u64]
+Flag :: enum u64 {
+	Emit_Comments,
+}
+
+Parse_Error :: parse.Error
+Execution_Error :: enum {
+	Accumulated_Errors,
+
+	Invalid_Node,
+	Invalid_Command,
+	Invalid_Value,
+
+	Undeclared_Variable,
+	Undefined_Function,
+
+	Invalid_Break,
+	Invalid_Continue,
+
+	Invalid_Argument_Count,
+	Invalid_Argument_Type,
+	Out_Of_Bounds_Access,
+}
+Error :: union {
+	io.Error,
+	Execution_Error,
+}
+
+create_from_string :: proc(input: string, config: Maybe(Config) = nil) -> (t: ^Template, err: Parse_Error) {
+	cfg, _ := config.?
+	if cfg.allocator.procedure == nil {
+		cfg.allocator = context.allocator
+	}
+
+	t = new(Template, cfg.allocator)
+	t.config = cfg
+	t.tree = nil
+	t.tree, err = parse.parse(input, t.left_delim, t.right_delim, .Emit_Comments in t.flags, t.allocator)
+	return
+}
+
+must :: proc(t: ^Template, err: Parse_Error) -> ^Template {
+	assert(t != nil)
+	assert(err == nil)
+	return t
+}
+
+destroy :: proc(t: ^Template) {
+	allocator := t.allocator
+	parse.destroy_tree(t.tree)
+	free(t, allocator)
+}
+
+execute :: proc(t: ^Template, w: io.Writer, data: any) -> Error {
+	s := State{
+		tmpl = t,
+		w    = w,
+	}
+	context.allocator = virtual.arena_allocator(&s.arena)
+	defer free_all(context.allocator)
+
+	err := walk(&s, data,t.tree.root)
+	if err == nil && s.error_count > 0 {
+		err = .Accumulated_Errors
+	}
+	return err
+}
+
+
+State :: struct {
+	arena: virtual.Growing_Arena,
+
+	tmpl:  ^Template,
+	w:     io.Writer,
+	at:    ^parse.Node,
+	vars:  [dynamic]Variable,
+	depth: int,
+	error_count: int,
+}
+Variable :: struct {
+	name:  string,
+	value: any,
+}
+
+walk :: proc(s: ^State, dot: any, node: ^parse.Node) -> Error {
+	s.at = node
+
+	switch n in node.variant {
+	case ^parse.Node_Comment:
+		// ignore
+	case ^parse.Node_Text:
+		io.write_string(s.w, n.text) or_return
+		return nil
+	case ^parse.Node_List:
+		for elem in n.nodes {
+			walk(s, dot, elem) or_return
+		}
+		return nil
+	case ^parse.Node_Action:
+		val := eval_pipeline(s, dot, n.pipe) or_return
+		if len(n.pipe.decl) == 0 {
+			print_value(s, n, val)
+		}
+		return nil
+	case ^parse.Node_Pipeline:
+		print_value(s, n, eval_pipeline(s, dot, n) or_return)
+		return nil
+
+	case ^parse.Node_Import:
+		panic("TODO Node_Import")
+
+	case ^parse.Node_If:
+		return walk_if_or_with(s, .If, dot, n.pipe, n.list, n.else_list)
+	case ^parse.Node_With:
+		return walk_if_or_with(s, .With, dot, n.pipe, n.list, n.else_list)
+	case ^parse.Node_For:
+		return walk_for(s, dot, n)
+
+	case ^parse.Node_Break:
+		return .Invalid_Break
+	case ^parse.Node_Continue:
+		return .Invalid_Continue
+
+	case ^parse.Node_Else,
+	     ^parse.Node_End,
+	     ^parse.Node_Nil,
+	     ^parse.Node_Bool,
+	     ^parse.Node_Number,
+	     ^parse.Node_String,
+	     ^parse.Node_Variable,
+	     ^parse.Node_Identifier,
+	     ^parse.Node_Operator,
+	     ^parse.Node_Dot,
+	     ^parse.Node_Field:
+		return .Invalid_Node
+
+	case ^parse.Node_Command:
+		return .Invalid_Node
+	case ^parse.Node_Chain:
+		return .Invalid_Node
+	}
+
+	return .Invalid_Node
+}
+
+mark_vars :: proc(s: ^State) -> int {
+	return len(s.vars)
+}
+pop_vars :: proc(s: ^State, n: int) {
+	resize(&s.vars, n)
+}
+
+@(private, deferred_in_out=pop_vars)
+SCOPE :: proc(s: ^State) -> int {
+	return mark_vars(s)
+}
+walk_if_or_with :: proc(s: ^State, kind: enum{If, With}, dot: any,
+                        pipe: ^parse.Node_Pipeline,
+                        list: ^parse.Node_List,
+                        else_list: ^parse.Node_List) -> Error {
+	SCOPE(s)
+
+	val := eval_pipeline(s, dot, pipe) or_return
+	truth, ok := is_true(val)
+	if !ok {
+		error(s, "if/with cannot use %v", val)
+	}
+	if truth {
+		if kind == .With {
+			return walk(s, val, list)
+		} else {
+			return walk(s, dot, list)
+		}
+	} else if else_list != nil {
+		return walk(s, dot, else_list)
+	}
+	return nil
+}
+
+walk_for :: proc(s: ^State, dot: any, f: ^parse.Node_For) -> Error {
+	s.at = f
+	SCOPE(s)
+
+	val, _ := indirect(eval_pipeline(s, dot, f.pipe) or_return)
+
+	mark := mark_vars(s)
+
+	the_body :: proc(s: ^State, f: ^parse.Node_For, elem, index: any, mark: int) -> Error {
+		if len(f.pipe.decl) > 0 {
+			set_top_var(s, 1, index) or_return
+		}
+		if len(f.pipe.decl) > 1 {
+			set_top_var(s, 2, elem) or_return
+		}
+		defer pop_vars(s, mark)
+		return walk(s, elem, f.list)
+	}
+
+	original_id := val.id
+	ti := reflect.type_info_base(type_info_of(val.id))
+	val.id = ti.id
+	#partial switch info in ti.variant {
+	case reflect.Type_Info_Array,
+	     reflect.Type_Info_Slice,
+	     reflect.Type_Info_Dynamic_Array:
+		n := reflect.length(val)
+		if n == 0 {
+			break
+		}
+		for i in 0..<n {
+			the_body(s, f, reflect.index(val, i), i, mark) or_return
+		}
+		return nil
+	case reflect.Type_Info_Map:
+		if reflect.length(val) == 0 {
+			break
+		}
+
+		gs := reflect.type_info_base(info.generated_struct).variant.(reflect.Type_Info_Struct)
+		ed := reflect.type_info_base(gs.types[1]).variant.(reflect.Type_Info_Dynamic_Array)
+		entry_type := ed.elem.variant.(reflect.Type_Info_Struct)
+		key_offset :=  entry_type.offsets[2]
+		value_offset :=  entry_type.offsets[3]
+		entry_size := uintptr(ed.elem_size)
+		key_type := entry_type.types[2]
+		value_type := entry_type.types[3]
+
+		rm := (^mem.Raw_Map)(val.data)
+
+		data := uintptr(rm.entries.data)
+		for i in 0..<rm.entries.len {
+			key := any{rawptr(data + key_offset), key_type.id}
+			value := any{rawptr(data + value_offset), value_type.id}
+
+			the_body(s, f, key, value, mark) or_return
+
+			data += entry_size
+		}
+		return nil
+
+		// TODO
+	case:
+		error(s, "for cannot iterate over %v", original_id)
+	}
+
+	if f.else_list != nil {
+		return walk(s, dot, f.else_list)
+	}
+	return nil
+}
+
+indirect :: proc(val: any) -> (v: any, is_nil: bool) {
+	v = val
+	for v != nil {
+		ti := reflect.type_info_base(type_info_of(v.id))
+		info, ok := ti.variant.(reflect.Type_Info_Pointer)
+		if !ok {
+			break
+		}
+
+		ptr := (^rawptr)(v.data)^
+		if ptr == nil {
+			return v, true
+		}
+		v = any{ptr, info.elem.id}
+	}
+	return v, false
+}
+
+
+error :: proc(s: ^State, format: string, args: ..any) {
+	s.error_count += 1
+
+	assert(s.at != nil)
+
+	// NOTE(bill): the line and column are recalculated
+	// each time here because errors are usually an early
+	// out for this execution system
+
+	pos := int(s.at.pos)
+	text := s.tmpl.tree.input[:pos]
+	col := strings.last_index(text, "\n")
+	if col < 0 {
+		col = pos
+	} else {
+		col += 1
+		col = pos - col
+	}
+
+	line := 1 + strings.count(text, "\n")
+
+	name := s.tmpl.tree.name
+	if name == "" {
+		name = "<input>"
+	}
+	fmt.eprintf("%s:%d:%d: ", name, line, col)
+	fmt.eprintf(format, ..args)
+	fmt.eprintln()
+}
+
+is_true :: proc(val: any) -> (truth, ok: bool) {
+	check_trivial :: proc(v: any) -> (bool, bool) {
+		data := reflect.as_bytes(v)
+		for v in data {
+			if v != 0 {
+				return true, true
+			}
+		}
+		return false, true
+	}
+
+	if val == nil {
+		return false, true
+	}
+
+	ti := reflect.type_info_base(type_info_of(val.id))
+	switch v in ti.variant {
+	case reflect.Type_Info_Named:
+		unreachable()
+	case reflect.Type_Info_Integer,
+	     reflect.Type_Info_Rune,
+	     reflect.Type_Info_Float,
+	     reflect.Type_Info_Complex,
+	     reflect.Type_Info_Quaternion,
+	     reflect.Type_Info_Boolean,
+	     reflect.Type_Info_Bit_Set,
+	     reflect.Type_Info_Enum:
+	     return check_trivial(val)
+
+	case reflect.Type_Info_String:
+		if v.is_cstring {
+			cstr := (^cstring)(val.data)^
+			if cstr == nil {
+				return false, true
+			}
+			return ([^]u8)(cstr)[0] != 0, true
+		}
+		str := (^string)(val.data)^
+		return len(str) > 0, true
+
+	case reflect.Type_Info_Any:
+		return false, false
+	case reflect.Type_Info_Type_Id:
+		return (^typeid)(val.data)^ != nil, true
+	case reflect.Type_Info_Pointer,
+	     reflect.Type_Info_Multi_Pointer:
+		return (^rawptr)(val.data)^ != nil, true
+
+	case reflect.Type_Info_Procedure:
+		return
+
+	case reflect.Type_Info_Array:
+		return v.count > 0, true
+	case reflect.Type_Info_Enumerated_Array:
+		return v.count > 0, true
+	case reflect.Type_Info_Dynamic_Array:
+		a := (^mem.Raw_Dynamic_Array)(val.data)
+		return a.len > 0, true
+	case reflect.Type_Info_Slice:
+		a := (^mem.Raw_Slice)(val.data)
+		return a.len > 0, true
+	case reflect.Type_Info_Tuple:
+		return
+	case reflect.Type_Info_Struct:
+		// All structs are always non nil
+		return true, true
+	case reflect.Type_Info_Union:
+	     return reflect.union_variant_typeid(val) != nil, true
+	case reflect.Type_Info_Map:
+		m := (^mem.Raw_Map)(val.data)
+		return m.entries.len > 0, true
+	case reflect.Type_Info_Simd_Vector:
+		return v.count > 0, true
+	case reflect.Type_Info_Relative_Pointer:
+		return check_trivial(val)
+	case reflect.Type_Info_Relative_Slice:
+		return check_trivial(val)
+	case reflect.Type_Info_Matrix:
+		return check_trivial(val)
+	}
+	return
+}
+
+
+eval_pipeline :: proc(s: ^State, dot: any, pipe: ^parse.Node_Pipeline) -> (value: any, err: Error) {
+	if pipe == nil {
+		return
+	}
+	s.at = pipe
+	value = nil
+	for cmd in pipe.cmds {
+		value = eval_command(s, dot, cmd, value) or_return
+	}
+	for var in pipe.decl {
+		if pipe.is_assign {
+			set_var(s, var.name, value) or_return
+		} else {
+			push_var(s, var.name, value)
+		}
+	}
+	return
+}
+
+set_var :: proc(s: ^State, name: string, value: any) -> Error {
+	for i := mark_vars(s)-1; i >= 0; i -= 1 {
+		if s.vars[i].name == name {
+			s.vars[i].value = value
+			return nil
+		}
+	}
+	return .Undeclared_Variable
+}
+
+set_top_var :: proc(s: ^State, n: int, value: any) -> Error {
+	if len(s.vars) > 0 {
+		s.vars[len(s.vars)-n].value = value
+		return nil
+	}
+	return .Undeclared_Variable
+}
+
+push_var :: proc(s: ^State, name: string, value: any) {
+	append(&s.vars, Variable{name, value})
+}
+get_var :: proc(s: ^State, name: string) -> (value: any, err: Error) {
+	for i := mark_vars(s)-1; i >= 0; i -= 1 {
+		if s.vars[i].name == name {
+			return s.vars[i].value, nil
+		}
+	}
+	error(s, "undeclared variable $%s", name)
+	return nil, .Undeclared_Variable
+
+}
+
+eval_command :: proc(s: ^State, dot: any, cmd: ^parse.Node_Command, final: any) -> (value: any, err: Error) {
+	first_word := cmd.args[0]
+	#partial switch n in first_word.variant {
+	case ^parse.Node_Field:
+		s.at = n
+		return eval_fields(s, dot, n.idents)
+	case ^parse.Node_Chain:
+		return eval_chain(s, dot, n)
+	case ^parse.Node_Identifier:
+		return eval_function(s, dot, n.ident, cmd, final)
+	case ^parse.Node_Operator:
+		return eval_function(s, dot, n.value, cmd, final)
+	case ^parse.Node_Pipeline:
+		return eval_pipeline(s, dot, n)
+	case ^parse.Node_Variable:
+		s.at = n
+		return get_var(s, n.name)
+	}
+	s.at = first_word
+	#partial switch n in first_word.variant {
+	case ^parse.Node_Bool:
+		return new_any(n.ok), nil
+	case ^parse.Node_Dot:
+		return dot, nil
+	case ^parse.Node_Nil:
+		return nil, nil
+	case ^parse.Node_Number:
+		if i, ok := n.i.?; ok {
+			return new_any(i), nil
+		}
+		if u, ok := n.u.?; ok {
+			return new_any(u), nil
+		}
+		if f, ok := n.f.?; ok {
+			return new_any(f), nil
+		}
+		return eval_function(s, dot, n.text, cmd, final)
+	case ^parse.Node_String:
+		return new_any(n.text), nil
+	}
+
+	error(s, "cannot evaluate command %v", first_word.variant)
+	return nil, .Invalid_Command
+}
+
+eval_chain :: proc(s: ^State, dot: any, chain: ^parse.Node_Chain) -> (value: any, err: Error) {
+	s.at = chain
+	return eval_fields(s, eval_arg(s, dot, chain.node) or_return, chain.fields[:])
+}
+
+eval_fields :: proc(s: ^State, dot: any, idents: []string) -> (value: any, err: Error) {
+	value = dot
+	for ident in idents {
+		value = eval_field(s, value, ident) or_return
+	}
+	return
+}
+
+eval_field :: proc(s: ^State, dot: any, ident: string) -> (value: any, err: Error) {
+	if dot == nil {
+		return nil, nil
+	}
+
+	ti := reflect.type_info_base(type_info_of(dot.id))
+	#partial switch info in ti.variant {
+	case reflect.Type_Info_Struct:
+		value = reflect.struct_field_value_by_name(dot, ident, true)
+		if value != nil {
+			return
+		}
+	case reflect.Type_Info_Pointer:
+		if dot.data != nil {
+			deref := (^rawptr)(dot.data)^
+			return eval_field(s, {deref, info.elem.id}, ident)
+		}
+	case reflect.Type_Info_Map:
+		key_type := reflect.type_info_base(info.key)
+		switch key_type.id {
+		case typeid_of(string), typeid_of(cstring):
+			gs := reflect.type_info_base(info.generated_struct).variant.(reflect.Type_Info_Struct)
+			ed := reflect.type_info_base(gs.types[1]).variant.(reflect.Type_Info_Dynamic_Array)
+			entry_type := ed.elem.variant.(reflect.Type_Info_Struct)
+			key_offset :=  entry_type.offsets[2]
+			value_offset :=  entry_type.offsets[3]
+			entry_size := uintptr(ed.elem_size)
+
+			rm := (^mem.Raw_Map)(dot.data)
+
+			data := uintptr(rm.entries.data)
+			for i in 0..<rm.entries.len {
+				key: string
+				switch key_type.id {
+				case typeid_of(string):
+					key = (^string)(data + key_offset)^
+				case typeid_of(cstring):
+					key = string((^cstring)(data + key_offset)^)
+				}
+
+				if key == ident {
+					ptr := rawptr(data + value_offset)
+					return any{ptr, entry_type.types[3].id}, nil
+				}
+
+				data += entry_size
+			}
+			return nil, nil
+		}
+	}
+
+
+	error(s, "cannot evaluate field %s in type %v", ident, dot.id)
+	return nil, .Invalid_Value
+}
+
+eval_function :: proc(s: ^State, dot: any, name: string, cmd: ^parse.Node_Command, final: any) -> (value: any, err: Error) {
+	cmd_args := cmd.args[1:]
+
+	switch name {
+	case "+", "-":
+		if len(cmd_args) < 1 {
+			error(s, "%q expects at least 1 argument", name)
+			return nil, .Invalid_Argument_Count
+		}
+		// TODO
+
+		return nil, nil
+	case "*":
+		if len(cmd_args) < 1 {
+			error(s, "%q expects at least 2 arguments, got %d", name, len(cmd_args))
+			return nil, .Invalid_Argument_Count
+		}
+		// TODO
+		return nil, nil
+	}
+
+	function, ok := builtin_funcs[name]
+	if !ok {
+		error(s, "%q is not a defined function", name)
+		err = .Undefined_Function
+		return
+	}
+
+	if function == nil {
+		switch name {
+		case "and":
+			// TODO
+		case "or":
+			// TODO
+		case:
+			panic("unhandled built-in procedure")
+		}
+	}
+
+	n := len(cmd_args)
+	if final != nil {
+		n += 1
+	}
+	args_to_call := make([dynamic]any, 0, n)
+	for arg in cmd_args {
+		append(&args_to_call, eval_arg(s, dot, arg) or_return)
+	}
+	if final != nil {
+		append(&args_to_call, final)
+	}
+
+	return function(args_to_call[:])
+}
+
+eval_arg :: proc(s: ^State, dot: any, arg: ^parse.Node) -> (value: any, err: Error) {
+	s.at = arg
+	#partial switch n in arg.variant {
+	case ^parse.Node_Dot:
+		return dot, nil
+	case ^parse.Node_Nil:
+		return nil, nil
+	case ^parse.Node_Bool:
+		return new_any(n.ok), nil
+	case ^parse.Node_Number:
+		if i, ok := n.i.?; ok {
+			return new_any(i), nil
+		}
+		if u, ok := n.u.?; ok {
+			return new_any(u), nil
+		}
+		if f, ok := n.f.?; ok {
+			return new_any(f), nil
+		}
+	case ^parse.Node_String:
+		return new_any(n.text), nil
+
+	case ^parse.Node_Field:
+		return eval_fields(s, dot, n.idents)
+	case ^parse.Node_Variable:
+		return get_var(s, n.name)
+	case ^parse.Node_Pipeline:
+		return eval_pipeline(s, dot, n)
+	case ^parse.Node_Chain:
+		return eval_chain(s, dot, n)
+	}
+	return nil, .Invalid_Node
+}
+
+
+print_value :: proc(s: ^State, n: ^parse.Node, val: any) {
+	s.at = n
+	if val == nil {
+		io.write_string(s.w, "nil")
+	} else {
+		fmt.wprint(s.w, val)
+	}
+}

core/text/template/function.odin

@@ -0,0 +1,91 @@
+package text_template
+
+import "core:mem"
+import "core:fmt"
+import "core:reflect"
+
+Function :: #type proc(args: []any) -> (value: any, err: Error)
+
+
+@(private)
+new_any :: proc(x: $T) -> any {
+	ptr := new_clone(x)
+	return any{ptr, typeid_of(T)}
+}
+
+builtin_funcs: map[string]Function
+
+@(private, init)
+init_builtin_funcs :: proc() {
+	builtin_funcs["and"] = nil // requires shortcircuiting behaviour so implemented internally
+	builtin_funcs["or"]  = nil // requires shortcircuiting behaviour so implemented internally
+	builtin_funcs["not"] = proc(args: []any) -> (value: any, err: Error) {
+		if len(args) != 1 {
+			err = .Invalid_Argument_Count
+			return
+		}
+		t, _ := is_true(args[0])
+		return new_any(t), nil
+	}
+
+	builtin_funcs["index"] = proc(args: []any) -> (value: any, err: Error) {
+		if len(args) < 2 {
+			err = .Invalid_Argument_Count
+			return
+		}
+
+		arg := args[0]
+		for idx in args[1:] {
+			i, ok := reflect.as_int(idx)
+			if !ok {
+				err = .Invalid_Argument_Type
+				return
+			}
+			if reflect.length(arg) < i {
+				return nil, .Out_Of_Bounds_Access
+			}
+			arg = reflect.index(arg, i)
+		}
+		return arg, nil
+	}
+
+	builtin_funcs["len"] = proc(args: []any) -> (value: any, err: Error) {
+		if len(args) != 1 {
+			err = .Invalid_Argument_Count
+			return
+		}
+
+		n := reflect.length(args[0])
+		return new_any(n), nil
+	}
+
+
+	builtin_funcs["print"] = proc(args: []any) -> (value: any, err: Error) {
+		return new_any(fmt.aprint(..args)), nil
+	}
+	builtin_funcs["println"] = proc(args: []any) -> (value: any, err: Error) {
+		return new_any(fmt.aprintln(..args)), nil
+	}
+	builtin_funcs["printf"] = proc(args: []any) -> (value: any, err: Error) {
+		if len(args) < 1 {
+			err = .Invalid_Argument_Count
+			return
+		}
+		format_any := args[0]
+		format_any.id = reflect.typeid_base(format_any.id)
+		format: string
+		switch v in format_any {
+		case string:
+			format = v
+		case cstring:
+			format = string(v)
+		case:
+			err = .Invalid_Argument_Type
+			return
+		}
+
+		other_args := args[1:]
+		return new_any(fmt.aprintf(format, ..other_args)), nil
+	}
+}
+

core/text/template/parse/ast.odin

@@ -0,0 +1,162 @@
+package text_template_parse
+
+import "../scan"
+import "core:strings"
+
+Pos :: scan.Pos
+Token :: scan.Token
+Token_Kind :: scan.Token_Kind
+
+new_node :: proc($T: typeid, pos: Pos = 0, allocator := context.allocator) -> ^T {
+	n := new(T, allocator)
+	n.pos = pos
+	n.variant = n
+	return n
+}
+
+Node :: struct {
+	pos: Pos,
+	variant: union{
+		^Node_Text,
+		^Node_Comment,
+		^Node_Action,
+		^Node_Pipeline,
+		^Node_Chain,
+		^Node_Command,
+		^Node_Import,
+		^Node_Dot,
+		^Node_Field,
+		^Node_Identifier,
+		^Node_Operator,
+		^Node_If,
+		^Node_For,
+		^Node_List,
+		^Node_Nil,
+		^Node_Bool,
+		^Node_Number,
+		^Node_String,
+		^Node_Variable,
+		^Node_With,
+		^Node_Break,
+		^Node_Continue,
+
+		// Dummy nodes
+		^Node_Else,
+		^Node_End,
+	},
+}
+
+Node_Branch :: struct{
+	using base: Node,
+	pipe: ^Node_Pipeline,
+	list: ^Node_List,
+	else_list: ^Node_List,
+}
+
+Node_Text :: struct{
+	using base: Node,
+	text: string,
+}
+Node_Action :: struct{
+	using base: Node,
+	pipe: ^Node_Pipeline,
+}
+Node_Bool :: struct{
+	using base: Node,
+	ok: bool,
+}
+Node_Chain :: struct{
+	using base: Node,
+	node: ^Node,
+	fields: [dynamic]string,
+}
+Node_Command :: struct{
+	using base: Node,
+	args: [dynamic]^Node,
+}
+Node_Dot :: struct{
+	using base: Node,
+}
+Node_Field :: struct{
+	using base: Node,
+	idents: []string,
+}
+Node_Identifier :: struct{
+	using base: Node,
+	ident: string,
+}
+Node_Operator :: struct{
+	using base: Node,
+	value: string,
+}
+
+
+Node_If   :: distinct Node_Branch
+Node_For  :: distinct Node_Branch
+Node_With :: distinct Node_Branch
+
+Node_List :: struct{
+	using base: Node,
+	nodes: [dynamic]^Node,
+}
+Node_Nil :: struct{
+	using base: Node,
+}
+Node_Number :: struct{
+	using base: Node,
+	text: string,
+	i: Maybe(i64),
+	u: Maybe(u64),
+	f: Maybe(f64),
+}
+Node_Pipeline :: struct{
+	using base: Node,
+	is_assign: bool,
+	decl: [dynamic]^Node_Variable,
+	cmds: [dynamic]^Node_Command,
+}
+Node_String :: struct{
+	using base: Node,
+	quoted: string,
+	text: string, // after processing
+}
+Node_Import :: struct{
+	using base: Node,
+	name: string, // unquoted
+	pipe: ^Node_Pipeline,
+}
+Node_Variable :: struct{
+	using base: Node,
+	name: string,
+}
+Node_Comment :: struct{
+	using base: Node,
+	text: string,
+}
+Node_Break :: struct{
+	using base: Node,
+}
+Node_Continue :: struct{
+	using base: Node,
+}
+
+Node_Else :: struct {
+	using base: Node,
+}
+
+Node_End :: struct {
+	using base: Node,
+}
+
+chain_add :: proc(c: ^Node_Chain, field: string) {
+	field := field
+	if len(field) == 0 || field[0] != '.' {
+		panic("not a .field")
+	}
+	field = field[1:]
+	if field == "" {
+		panic("empty field")
+	}
+	append(&c.fields, field)
+}
+

core/text/template/parse/parse.odin

@@ -0,0 +1,528 @@
+package text_template_parse
+
+import "core:io"
+import "core:fmt"
+import "core:mem"
+import "core:mem/virtual"
+import "core:strconv"
+import "../scan"
+
+Error :: enum {
+	None,
+
+	Unexpected_Token,
+	Unexpected_EOF,
+
+	Expected_End,
+
+	Invalid_Node,
+
+	Invalid_Character,
+	Invalid_Number,
+	Invalid_String,
+
+	Empty_Command,
+	Missing_Value,
+	Non_Executable_Command,
+	Undefined_Variable,
+	Unexpected_Operand,
+	Invalid_For_Initialization,
+	Too_Many_Declarations,
+}
+Tree :: struct {
+	general_allocator: mem.Allocator,
+
+	arena: virtual.Growing_Arena,
+	name: string,
+
+	tokens: []Token, // general_allocator
+
+	root: ^Node_List,
+	input: string,
+	offset: uint,
+
+	for_loop_depth: uint,
+
+	vars: [dynamic]string,
+}
+
+@(require_results)
+errorf :: proc(t: ^Tree, err: Error, format: string, args: ..any) -> Error {
+	if err != nil {
+		fmt.eprintf(format, ..args)
+		fmt.eprintln()
+	}
+	return err
+}
+
+@(require_results)
+unexpected_token :: proc(t: ^Tree, token: Token) -> Error {
+	return errorf(t, .Unexpected_Token, "unexpected token: %s", token.value)
+}
+
+
+peek :: proc(t: ^Tree, n: uint = 0) -> Token {
+	if t.offset+n < len(t.tokens) {
+		return t.tokens[t.offset+n]
+	}
+	return Token{.EOF, "", Pos(len(t.input)), 0}
+}
+next :: proc(t: ^Tree) -> (token: Token) {
+	if t.offset < len(t.tokens) {
+		token = t.tokens[t.offset]
+		t.offset += 1
+		return
+	}
+	return Token{.EOF, "", Pos(len(t.input)), 0}
+}
+backup :: proc(t: ^Tree, n: uint = 1) {
+	if n > t.offset {
+		t.offset = 0
+	} else {
+		t.offset -= n
+	}
+}
+
+next_non_space :: proc(t: ^Tree) -> (token: Token) {
+	for {
+		token = next(t)
+		if token.kind != .Space {
+			break
+		}
+	}
+	return
+}
+peek_non_space :: proc(t: ^Tree, offset: uint = 0) -> (token: Token) {
+	i := offset
+	for {
+		if t.offset+i < len(t.tokens) {
+			token = t.tokens[t.offset+i]
+		} else {
+			token = Token{.EOF, "", Pos(len(t.input)), 0}
+		}
+		if token.kind != .Space {
+			break
+		}
+		i += 1
+	}
+	return
+}
+peek_after_non_space :: proc(t: ^Tree) -> (token: Token) {
+	return peek_non_space(t, 1)
+}
+
+expect :: proc(t: ^Tree, expected: Token_Kind, ctx: string) -> (token: Token, err: Error) {
+	token = next_non_space(t)
+	if token.kind != expected {
+		err = errorf(t, .Unexpected_Token, "unexpected token, expected %s, got %s", expected, token.value)
+	}
+	return
+}
+
+
+parse :: proc(input: string, left_delim, right_delim: string, emit_comments: bool = false, general_allocator := context.allocator) -> (t: ^Tree, err: Error) {
+	t = new(Tree, general_allocator)
+	t.general_allocator = general_allocator
+	t.vars.allocator = general_allocator
+	t.input = input
+
+
+	s := scan.init(&scan.Scanner{}, t.name, input, left_delim, right_delim, emit_comments)
+	s.tokens.allocator = t.general_allocator
+	scan.run(s)
+	t.tokens = s.tokens[:] // general_allocator
+
+	context.allocator = virtual.arena_allocator(&t.arena)
+
+	t.root = new_node(Node_List)
+	for peek(t).kind != .EOF {
+		if peek(t).kind == .Left_Delim && peek_after_non_space(t).kind == .Declare {
+			// TODO
+			continue
+		}
+		node := text_or_action(t) or_return
+		if node != nil {
+			append(&t.root.nodes, node)
+		} else {
+			break
+		}
+	}
+
+	return
+}
+
+destroy_tree :: proc(t: ^Tree) {
+	if t != nil {
+		virtual.arena_destroy(&t.arena)
+
+		ga := t.general_allocator
+		delete(t.tokens, ga)
+		delete(t.vars)
+		free(t, ga)
+	}
+}
+
+
+text_or_action :: proc(t: ^Tree) -> (node: ^Node, err: Error) {
+	#partial switch token := next_non_space(t); token.kind {
+	case .Text:
+		n := new_node(Node_Text, token.pos)
+		n.text = token.value
+		return n, nil
+	case .Left_Delim:
+		return action(t)
+	case .Comment:
+		n := new_node(Node_Comment, token.pos)
+		n.text = token.value
+		return n, nil
+	case:
+		return nil, unexpected_token(t, token)
+	}
+	return nil, nil
+}
+
+parse_list :: proc(t: ^Tree) -> (list: ^Node_List, next: ^Node, err: Error) {
+	list = new_node(Node_List, peek_non_space(t).pos)
+	for peek_non_space(t).kind != .EOF {
+		node := text_or_action(t) or_return
+		#partial switch n in node.variant {
+		case ^Node_Else:
+			next = n
+			return
+		case ^Node_End:
+			next = n
+			return
+		}
+		append(&list.nodes, node)
+	}
+	err = errorf(t, .Unexpected_EOF, "unexpected EOF")
+	return
+}
+
+parse_control :: proc(t: ^Tree, allow_else_if: bool, ctx: string) -> (pipe: ^Node_Pipeline, list, else_list: ^Node_List, err: Error) {
+	pipe = pipeline(t, ctx, .Right_Delim) or_return
+
+	if ctx == "for" {
+		t.for_loop_depth += 1
+	}
+
+	next_node: ^Node
+	list, next_node = parse_list(t) or_return
+
+	if ctx == "for" {
+		t.for_loop_depth -= 1
+	}
+
+	#partial switch n in next_node.variant {
+	case ^Node_End:
+		// We are done
+
+	case ^Node_Else:
+		if allow_else_if && peek(t).kind == .If {
+			// {{if a}}...{{else if b}}...{{end}}
+			// is translated into
+			// {{if a}}...{{else}}{{if b}}...{{end}}{{end}}
+			next(t)
+			else_list = new_node(Node_List, next_node.pos)
+			append(&else_list.nodes, parse_if(t) or_return)
+			break
+		}
+		else_list, next_node = parse_list(t) or_return
+		if _, ok := next_node.variant.(^Node_End); !ok {
+			errorf(t, .Expected_End, "expected end") or_return
+		}
+	}
+	return
+}
+
+// {{if pipeline}} list {{end}}
+// {{if pipeline}} list {{else}} list {{end}}
+// {{if pipeline}} list {{else if pipeline}} list {{end}}
+parse_if :: proc(t: ^Tree) -> (node: ^Node_If, err: Error) {
+	pipe, list, else_list := parse_control(t, true, "if") or_return
+	node = new_node(Node_If, pipe.pos)
+	node.pipe = pipe
+	node.list = list
+	node.else_list = else_list
+	return
+}
+
+// {{for pipeline}} list {{end}}
+// {{for pipeline}} list {{else}} list {{end}}
+parse_for :: proc(t: ^Tree) -> (node: ^Node_For, err: Error) {
+	pipe, list, else_list := parse_control(t, false, "for") or_return
+	node = new_node(Node_For, pipe.pos)
+	node.pipe = pipe
+	node.list = list
+	node.else_list = else_list
+	return
+}
+
+// {{with pipeline}} list {{end}}
+// {{with pipeline}} list {{else}} list {{end}}
+parse_with :: proc(t: ^Tree) -> (node: ^Node_With, err: Error) {
+	pipe, list, else_list := parse_control(t, false, "with") or_return
+	node = new_node(Node_With, pipe.pos)
+	node.pipe = pipe
+	node.list = list
+	node.else_list = else_list
+	return
+}
+
+
+// {{else}}
+parse_else :: proc(t: ^Tree) -> (node: ^Node_Else, err: Error) {
+	p := peek_non_space(t)
+	if p.kind == .If {
+		node = new_node(Node_Else, p.pos)
+		return
+	}
+	token := expect(t, .Right_Delim, "else") or_return
+	node = new_node(Node_Else, token.pos)
+	return
+}
+// {{end}}
+parse_end :: proc(t: ^Tree) -> (node: ^Node_End, err: Error) {
+	token := expect(t, .Right_Delim, "end") or_return
+	node = new_node(Node_End, token.pos)
+	return
+}
+
+
+
+
+action :: proc(t: ^Tree) -> (^Node, Error) {
+	// TODO actions
+	#partial switch token := next_non_space(t); token.kind {
+	case .If:   return parse_if(t)
+	case .For:  return parse_for(t)
+	case .With: return parse_with(t)
+	case .Else: return parse_else(t)
+	case .End:  return parse_end(t)
+
+	case .Block:
+		return nil, .Invalid_Node
+	case .Break:
+		return nil, .Invalid_Node
+	case .Continue:
+		return nil, .Invalid_Node
+	case .Include:
+		return nil, .Invalid_Node
+	}
+	backup(t)
+
+	return pipeline(t, "command", .Right_Delim)
+}
+
+
+pipeline :: proc(t: ^Tree, ctx: string, end: Token_Kind) -> (pipe: ^Node_Pipeline, err: Error) {
+	pipe = new_node(Node_Pipeline, peek_non_space(t).pos)
+
+	decls: for v := peek_non_space(t); v.kind == .Variable; /**/ {
+		next_non_space(t)
+
+		token_after_variable := peek(t) // could be space
+		next := peek_non_space(t)
+		switch {
+		case next.kind == .Assign, next.kind == .Declare:
+			pipe.is_assign = next.kind == .Assign
+			next_non_space(t)
+			append(&t.vars, v.value)
+			append(&pipe.decl, parse_variable(t, v) or_return)
+
+		case next.kind == .Char && next.value == ",":
+			next_non_space(t)
+			append(&t.vars, v.value)
+			append(&pipe.decl, parse_variable(t, v) or_return)
+			if ctx == "for" && len(pipe.decl) < 2 {
+				#partial switch peek_non_space(t).kind {
+				case .Variable, .Right_Delim, .Right_Paren:
+					v = peek_non_space(t)
+					continue decls
+				}
+				errorf(t, .Invalid_For_Initialization, "for can only initialize variables") or_return
+			}
+			errorf(t, .Too_Many_Declarations, "too many declarations in %s", ctx) or_return
+
+		case token_after_variable.kind == .Space:
+			backup(t, 2)
+		case:
+			backup(t, 1)
+		}
+
+		break decls
+	}
+
+	for {
+		#partial switch tok := next_non_space(t); tok.kind {
+		case end:
+			if len(pipe.cmds) == 0 {
+				errorf(t, .Missing_Value, "missing value for %s", ctx) or_return
+			}
+			for c, i in pipe.cmds[1:] {
+				#partial switch n in c.variant {
+				case ^Node_Bool, ^Node_Dot, ^Node_Nil, ^Node_Number, ^Node_String:
+					errorf(t, .Non_Executable_Command, "non executable command in pipeline stage for %d", i+2) or_return
+				}
+			}
+			return
+		case .Bool, .Char, .Dot, .Field, .Identifier, .Operator, .Number, .Nil, .Raw_String, .String, .Variable, .Left_Paren:
+			backup(t)
+			append(&pipe.cmds, command(t) or_return)
+		case:
+			err = unexpected_token(t, tok)
+			return
+		}
+	}
+}
+
+command :: proc(t: ^Tree) -> (cmd: ^Node_Command, err: Error) {
+	cmd = new_node(Node_Command, peek_non_space(t).pos)
+	loop: for {
+		op := operand(t) or_return
+		if op != nil {
+			append(&cmd.args, op)
+		}
+		#partial switch token := next(t); token.kind {
+		case .Space:
+			continue loop
+		case .Right_Delim, .Right_Paren:
+			backup(t)
+		case .Pipe:
+			break loop
+		case:
+			errorf(t, .Unexpected_Operand, "unexpected operand %s", token.value) or_return
+		}
+		break loop
+	}
+	if len(cmd.args) == 0 {
+		err = errorf(t, .Empty_Command, "empty command")
+	}
+	return
+}
+
+operand :: proc(t: ^Tree) -> (node: ^Node, err: Error) {
+	node = term(t) or_return
+	if node == nil {
+		return
+	}
+	if p := peek(t); p.kind == .Field {
+		chain := new_node(Node_Chain, p.pos)
+		chain.node = node
+		for peek(t).kind == .Field {
+			chain_add(chain, next(t).value)
+		}
+
+		#partial switch n in node.variant {
+		case ^Node_Field:
+			f := new_node(Node_Field, chain.pos)
+			resize(&chain.fields, len(chain.fields)+len(n.idents))
+			copy(chain.fields[len(n.idents):], chain.fields[:])
+			copy(chain.fields[:], n.idents)
+			f.idents = chain.fields[:]
+			node = f
+		case:
+			node = chain
+		}
+
+	}
+	return
+}
+
+// literal (number, string, nil, boolean)
+// function (identifier)
+// operator (function-like thing)
+// .
+// .field
+// $
+// $
+// '(' pipeline ')'
+term :: proc(t: ^Tree) -> (^Node, Error) {
+	#partial switch token := next_non_space(t); token.kind {
+	case .Identifier:
+		n := new_node(Node_Identifier, token.pos)
+		n.ident = token.value
+		return n, nil
+	case .Operator:
+		n := new_node(Node_Operator, token.pos)
+		n.value = token.value
+		return n, nil
+	case .Dot: return new_node(Node_Dot, token.pos), nil
+	case .Nil: return new_node(Node_Nil, token.pos), nil
+	case .Variable:
+		return parse_variable(t, token)
+	case .Field:
+		f := new_node(Node_Field, token.pos)
+		f.idents = make([]string, 1)
+		f.idents[0] = token.value[1:]
+		return f, nil
+	case .Bool:
+		b := new_node(Node_Bool, token.pos)
+		b.ok = token.value == "true"
+		return b, nil
+	case .Char, .Number:
+		return parse_number(t, token)
+	case .String, .Raw_String:
+		text, _, ok := strconv.unquote_string(token.value)
+		if !ok {
+			return nil, errorf(t, .Invalid_String, "invalid string literal: %s", token.value)
+		}
+		n := new_node(Node_String, token.pos)
+		n.quoted = token.value
+		n.text = text
+		return n, nil
+	case .Left_Paren:
+		return pipeline(t, "parenthesized pipeline", .Right_Paren)
+	}
+	backup(t)
+	return nil, nil
+}
+
+
+parse_number :: proc(t: ^Tree, token: Token) -> (^Node_Number, Error) {
+	text := token.value
+	n := new_node(Node_Number, token.pos)
+	n.text = text
+	if token.kind == .Char {
+		r, _, tail, ok := strconv.unquote_char(text[:], text[0])
+		if !ok || tail != "" {
+			return nil, errorf(t, .Invalid_Character, "invalid character literal: %s", text)
+		}
+		n.i = i64(r)
+		n.u = u64(r)
+		n.f = f64(r)
+		return n, nil
+	}
+
+
+	if u, ok := strconv.parse_u64(text); ok {
+		n.u = u
+	}
+	if i, ok := strconv.parse_i64(text); ok {
+		n.i = i
+		if i == 0 {
+			n.u = 0
+		}
+	}
+	if n.u == nil && n.i == nil {
+		if f, ok := strconv.parse_f64(text); ok {
+			n.f = f
+		}
+	}
+	if n.u == nil && n.i == nil && n.f == nil {
+		return nil, errorf(t, .Invalid_Number, "invalid number syntax: %q", text)
+	}
+	return n, nil
+
+}
+
+parse_variable :: proc(t: ^Tree, token: Token) -> (^Node_Variable, Error) {
+	v := new_node(Node_Variable, token.pos)
+	v.name = token.value
+	for var in t.vars {
+		if var == v.name {
+			return v, nil
+		}
+	}
+	return nil, errorf(t, .Undefined_Variable, "undefined variable %q", v.name)
+}

core/text/template/scan/scan.odin

@@ -0,0 +1,608 @@
+package text_template_scan
+
+import "core:fmt"
+import "core:unicode"
+import "core:unicode/utf8"
+import "core:strings"
+
+Pos :: distinct int
+
+Token_Kind :: enum {
+	Error,
+	EOF,
+	
+	Comment,
+	Space,
+	
+	Left_Delim,
+	Right_Delim,	
+	
+	Identifier,	
+	Field,
+	Left_Paren,
+	Right_Paren,
+		
+	Bool,
+	Char,
+	Number,
+	Pipe,
+	Raw_String,
+	String,
+	Text,
+	Variable,
+	Operator,
+
+	Declare, // :=
+	Assign,  // -
+	
+	_Keyword,
+	Dot,
+	Block,
+	Break,
+	Continue,
+	Define,
+	Else,
+	End,
+	For,
+	If,
+	Include,
+	Nil,
+	With,
+}
+
+
+keywords := map[string]Token_Kind {
+	"."        = .Dot,
+	"block"    = .Block,
+	"break"    = .Break,
+	"continue" = .Continue,
+	"define"   = .Define,
+	"else"     = .Else,
+	"end"      = .End,
+	"for"      = .For,
+	"if"       = .If,
+	"include"  = .Include,
+	"nil"      = .Nil,
+	"with"     = .With,
+}
+
+Token :: struct {
+	kind:  Token_Kind,
+	value: string,
+	pos:   Pos,
+	line:  int,
+}
+
+token_to_string :: proc(using tok: Token, allocator := context.temp_allocator) -> string {
+	context.allocator = allocator
+	switch {
+	case kind == .EOF:
+		return fmt.tprint("EOF")
+	case kind == .Error:
+		return fmt.tprint(value)
+	case kind > ._Keyword:
+		return fmt.tprintf("<%s>", value)
+	case len(value) > 10:
+		return fmt.tprintf("%.10q...", value)
+	}
+	return fmt.tprintf("%q", value)
+}
+
+Scanner :: struct {
+	name:          string,
+	input:         string,
+	left_delim:    string,
+	right_delim:   string,
+	pos:           Pos,
+	start:         Pos,
+	width:         Pos,
+	tokens:        [dynamic]Token,
+	paren_depth:   int,
+	line:          int,
+	start_line:    int,
+	emit_comments: bool,
+}
+
+next :: proc(s: ^Scanner) -> rune {
+	if int(s.pos) >= len(s.input) {
+		s.width = 0
+		return utf8.RUNE_EOF
+	}
+	r, w := utf8.decode_rune_in_string(s.input[s.pos:])
+	s.width = Pos(w)
+	s.pos += s.width
+	if r == '\n' {
+		s.line += 1
+	}
+	return r
+}
+
+backup :: proc(s: ^Scanner) {
+	s.pos -= s.width
+	if s.width == 1 && s.input[s.pos] == '\n' {
+		s.line -= 1
+	}
+}
+
+peek :: proc(s: ^Scanner) -> rune {
+	r := next(s)
+	backup(s)
+	return r
+}
+
+emit :: proc(s: ^Scanner, kind: Token_Kind) {
+	append(&s.tokens, Token{
+		kind = kind,
+		pos = s.start,
+		line = s.start_line,
+		value = s.input[s.start:s.pos],
+	})
+	s.start = s.pos
+	s.start_line = s.line
+}
+
+ignore :: proc(s: ^Scanner) {
+	s.line += strings.count(s.input[s.start:s.pos], "\n")
+	s.start = s.pos
+	s.start_line = s.line
+}
+
+accept :: proc(s: ^Scanner, valid: string) -> bool {
+	if strings.contains_rune(valid, next(s)) >= 0 {
+		return true
+	}
+	backup(s)
+	return false
+}
+accept_run :: proc(s: ^Scanner, valid: string) {
+	for strings.contains_rune(valid, next(s)) >= 0 {
+		// Okay
+	}
+	backup(s)
+}
+
+
+// State procedures
+
+DEFAULT_LEFT_DELIM :: "{{"
+DEFAULT_RIGHT_DELIM :: "}}"
+
+LEFT_COMMENT  :: "/*"
+RIGHT_COMMENT :: "*/"
+
+
+init :: proc(s: ^Scanner, name, input: string, left: string = "", right: string = "",
+             emit_comments: bool = false) -> ^Scanner {
+	s.name          = name
+	s.input         = input
+	s.left_delim    = left  if left  != "" else DEFAULT_LEFT_DELIM
+	s.right_delim   = right if right != "" else DEFAULT_RIGHT_DELIM
+	s.emit_comments = emit_comments
+	reset(s)
+	return s
+}
+
+destroy :: proc(s: ^Scanner) {
+	delete(s.tokens)
+	s.tokens = {}
+}
+
+
+reset :: proc(s: ^Scanner) {
+	clear(&s.tokens)
+	s.pos   = 0
+	s.start = 0
+	s.width = 0
+	s.paren_depth = 0
+	s.line        = 1
+	s.start_line  = 1
+}
+
+
+// Finite State Machine Scanning
+
+Scan_State :: enum {
+	None,
+
+	Comment,
+	Space,
+
+	Identifier,
+	Field,
+	Left_Delim,
+	Right_Delim,
+
+	Char,
+	Number,
+	Raw_String,
+	String,
+	Text,
+	Variable,
+
+	Inside_Action,
+}
+
+step :: proc(s: ^Scanner, state: Scan_State) -> Scan_State {
+	scan_error :: proc(s: ^Scanner, value: string) -> Scan_State {
+		append(&s.tokens, Token{
+			kind = .Error,
+			pos = s.start,
+			line = s.start_line,
+			value = value,
+		})
+		return nil
+	}
+	scan_variable_or_field :: proc(s: ^Scanner, kind: Token_Kind) -> Scan_State {
+		if at_terminator(s) {
+			if kind == .Variable {
+				emit(s, kind)
+			} else {
+				emit(s, .Dot)
+			}
+			return .Inside_Action
+		}
+		for {
+			r := next(s)
+			if !is_alpha_numeric(r) {
+				backup(s)
+				break
+			}
+		}
+		if !at_terminator(s) {
+			return scan_error(s, "bad character")
+		}
+		emit(s, kind)
+		return .Inside_Action
+	}
+
+	switch state {
+	case .None:
+		return nil
+
+	case .Comment:
+		s.pos += Pos(len(LEFT_COMMENT))
+		i := strings.index(s.input[s.pos:], RIGHT_COMMENT)
+		if i < 0 {
+			return scan_error(s, "unclosed comment")
+		}
+		s.pos += Pos(i + len(RIGHT_COMMENT))
+		delim, trim_space := at_right_delim(s)
+		if !delim {
+			return scan_error(s, "comment ends before closing delimiter")
+		}
+		if s.emit_comments {
+			emit(s, .Comment)
+		}
+		if trim_space {
+			s.pos += TRIM_MARKER_LEN
+		}
+		s.pos += Pos(len(s.right_delim))
+		if trim_space {
+			s.pos += left_trim_length(s.input[s.pos:])
+		}
+		ignore(s)
+		return .Text
+
+	case .Space:
+		space_count: int
+		for {
+			r := peek(s)
+			if !is_space(r) {
+				break
+			}
+			next(s)
+			space_count += 1
+		}
+
+		if has_right_trim_marker(s.input[s.pos-1:]) && strings.has_prefix(s.input[s.pos-1+TRIM_MARKER_LEN:], s.right_delim) {
+			backup(s)
+			if space_count == 1 {
+				return .Right_Delim
+			}
+		}
+		emit(s, .Space)
+		return .Inside_Action
+
+	case .Identifier:
+		identifier_loop: for {
+			r := next(s)
+			if is_alpha_numeric(r) {
+				// Okay
+			} else {
+				backup(s)
+				word := s.input[s.start:s.pos]
+				if !at_terminator(s) {
+					return scan_error(s, "bad character")
+				}
+
+				if kw := keywords[word]; kw > ._Keyword {
+					emit(s, kw)
+				} else if word == "true" || word == "false" {
+					emit(s, .Bool)
+				} else {
+					emit(s, .Identifier)
+				}
+				break identifier_loop
+			}
+		}
+		return .Inside_Action
+	case .Field:
+		return scan_variable_or_field(s, .Field)
+
+	case .Left_Delim:
+		s.pos += Pos(len(s.left_delim))
+		trim_space := has_left_trim_marker(s.input[s.pos:])
+		after_marker := TRIM_MARKER_LEN if trim_space else 0
+		if strings.has_prefix(s.input[s.pos+after_marker:], LEFT_COMMENT) {
+			s.pos += after_marker
+			ignore(s)
+			return .Comment
+		}
+		emit(s, .Left_Delim)
+		s.pos += after_marker
+		ignore(s)
+		s.paren_depth = 0
+		return .Inside_Action
+
+	case .Right_Delim:
+		trim_space := has_right_trim_marker(s.input[s.pos:])
+		if trim_space {
+			s.pos += TRIM_MARKER_LEN
+			ignore(s)
+		}
+		s.pos += Pos(len(s.right_delim))
+		emit(s, .Right_Delim)
+		if trim_space {
+			s.pos += left_trim_length(s.input[s.pos:])
+			ignore(s)
+		}
+		return .Text
+
+	case .Char:
+		char_loop: for {
+			switch next(s) {
+			case '\\':
+				if r := next(s); r != utf8.RUNE_EOF && r != '\n' {
+					break
+				}
+				fallthrough
+			case utf8.RUNE_EOF, '\n':
+				return scan_error(s, "unterminated character constant")
+			case '\'':
+				break char_loop
+			}
+		}
+		emit(s, .Char)
+		return .Inside_Action
+
+	case .Number:
+		accept(s, "+-")
+		digits := "0123456789_"
+		if accept(s, "0") {
+			switch {
+			case accept(s, "bB"):
+				digits = "01_"
+			case accept(s, "oO"):
+				digits = "01234567_"
+			case accept(s, "xX"):
+				digits = "0123456789ABCDEFabcdef_"
+			}
+		}
+		accept_run(s, digits)
+		if accept(s, ".") {
+			accept_run(s, digits)
+		}
+		if len(digits) == 10+1 && accept(s, "eE") {
+			accept(s, "+-")
+			accept_run(s, digits)
+		}
+		if is_alpha_numeric(peek(s)) {
+			next(s)
+			return scan_error(s, "bad number syntax")
+		}
+		emit(s, .Number)
+		return .Inside_Action
+
+	case .Raw_String:
+		raw_string_loop: for {
+			switch next(s) {
+			case utf8.RUNE_EOF:
+				return scan_error(s, "unterminated raw quoted string")
+			case '`':
+				break raw_string_loop
+			}
+		}
+		emit(s, .Raw_String)
+		return .Inside_Action
+
+	case .String:
+		string_loop: for {
+			switch next(s) {
+			case '\\':
+				if r := next(s); r != utf8.RUNE_EOF && r != '\n' {
+					break
+				}
+				fallthrough
+			case utf8.RUNE_EOF, '\n':
+				return scan_error(s, "unterminated quoted string")
+			case '"':
+				break string_loop
+			}
+		}
+		emit(s, .String)
+		return .Inside_Action
+
+	case .Text:
+		s.width = 0
+		if x := strings.index(s.input[s.pos:], s.left_delim); x >= 0 {
+			ldn := Pos(len(s.left_delim))
+			s.pos += Pos(x)
+			trim_length := Pos(0)
+			if has_left_trim_marker(s.input[s.pos+ldn:]) {
+				trim_length = right_trim_length(s.input[s.start:s.pos])
+			}
+			s.pos -= trim_length
+			if s.pos > s.start {
+				s.line += strings.count(s.input[s.start:s.pos], "\n")
+				emit(s, .Text)
+			}
+			s.pos += trim_length
+			ignore(s)
+			return .Left_Delim
+		}
+		s.pos = Pos(len(s.input))
+		// EOF
+		if s.pos > s.start {
+			s.line += strings.count(s.input[s.start:s.pos], "\n")
+			emit(s, .Text)
+		}
+		emit(s, .EOF)
+
+	case .Variable:
+		if at_terminator(s) {
+			emit(s, .Variable)
+			return .Inside_Action
+		}
+		return scan_variable_or_field(s, .Variable)
+
+	case .Inside_Action:
+		if delim, _ := at_right_delim(s); delim {
+			if s.paren_depth == 0 {
+				return .Right_Delim
+			}
+			return scan_error(s, "unclosed left paren")
+		}
+
+		rp := peek(s)
+		switch r := next(s); {
+		case r == utf8.RUNE_EOF:
+			return scan_error(s, "unclosed action")
+		case is_space(r):
+			backup(s) // Just in case of " -}}"
+			return .Space
+		case r == '.':
+			// Look for a '.field'
+			if s.pos < Pos(len(s.input)) {
+				if r := s.input[s.pos]; r < '0' || r > '9' {
+					return .Field
+				}
+			}
+			// it's a number
+			fallthrough
+
+		case (r == '+' || r == '-') && ('0' <= rp && rp <= '9'):
+			fallthrough
+		case '0' <= r && r <= '9':
+			backup(s)
+			return .Number
+		case r == '+', r == '-', r == '*':
+			emit(s, .Operator)
+		case is_alpha_numeric(r):
+			backup(s)
+			return .Identifier
+		case r == '|':
+			emit(s, .Pipe)
+		case r == '"':
+			return .String
+		case r == '`':
+			return .Raw_String
+		case r == '\'':
+			return .Char
+		case r == '(':
+			emit(s, .Left_Paren)
+			s.paren_depth += 1
+		case r == ')':
+			emit(s, .Right_Paren)
+			s.paren_depth -= 1
+			if s.paren_depth < 0 {
+				return scan_error(s, "unexpected right parenthesis ')'")
+			}
+		case r == '$':
+			return .Variable
+		case r == ':':
+			if next(s) != '=' {
+				return scan_error(s, "expected :=")
+			}
+			emit(s, .Declare)
+		case r == '=':
+			emit(s, .Assign)
+		case r <= unicode.MAX_ASCII && unicode.is_print(r):
+			emit(s, .Char)
+		case:
+			return scan_error(s, "unrecognized character in action")
+		}
+
+		return .Inside_Action
+	}
+	return nil
+}
+
+
+
+run :: proc(s: ^Scanner) {
+	state := Scan_State.Text
+	for state != nil {
+		state = step(s, state)
+	}
+}
+
+@private TRIM_MARKER :: '-'
+@private TRIM_MARKER_LEN :: Pos(2) // includes space
+
+is_space :: proc(r: rune) -> bool {
+	switch r {
+	case ' ', '\t', '\r', '\n':
+		return true
+	}
+	return false
+}
+
+is_alpha_numeric :: proc(r: rune) -> bool {
+	return r == '_' || unicode.is_letter(r) || unicode.is_digit(r)
+}
+
+left_trim_length :: proc(s: string) -> Pos {
+	return Pos(len(s) - len(strings.trim_left_proc(s, is_space)))
+}
+right_trim_length :: proc(s: string) -> Pos {
+	return Pos(len(s) - len(strings.trim_right_proc(s, is_space)))
+}
+
+has_left_trim_marker :: proc(s: string) -> bool {
+	return len(s) >= 2 && s[0] == TRIM_MARKER && is_space(rune(s[1]))
+}
+
+has_right_trim_marker :: proc(s: string) -> bool {
+	return len(s) >= 2 && is_space(rune(s[0])) && s[1] == TRIM_MARKER
+}
+
+at_right_delim :: proc(s: ^Scanner) -> (delim, trim_spaces: bool) {
+	if has_right_trim_marker(s.input[s.pos:]) && strings.has_prefix(s.input[s.pos+TRIM_MARKER_LEN:], s.right_delim) {
+		delim = true
+		trim_spaces = true
+		return
+	}
+	if strings.has_prefix(s.input[s.pos:], s.right_delim) {
+		delim = true
+		trim_spaces = false
+		return
+	}
+	delim = false
+	trim_spaces = false
+	return
+}
+
+at_terminator :: proc(s: ^Scanner) -> bool {
+	r := peek(s)
+	if is_space(r) {
+		return true
+	}
+	
+	switch r {
+	case utf8.RUNE_EOF, '.', ',', '(', ')', '|', ':':
+		return true
+	}
+	
+	rd, _ := utf8.decode_rune_in_string(s.right_delim)
+	return rd == r
+}