code-template ::
      name '=' code-exprs ';'
    | CODE_TEMPLATE ';'
    | '<undefined>' ';'
code-exprs ::
      <EMPTY>
    | code-exprs code-expr
code-expr ::
      STRING
    | VARIABLE
    | optional
    | list
optional ::
      '(' CONDITIONAL '?' code-exprs ')'
    | '(' CONDITIONAL '?' code-exprs ':' code-exprs ')'
list ::
      '[' VARIABLE ':' code-exprs ']'
    | '[' VARIABLE '{' NUMBER '}' ':' code-exprs ']'
    | '[' VARIABLE '{' NUMBER ',' NUMBER '}' ':' code-exprs ']'

یک الگوی کد دنباله‌ای از لیترال‌های رشته‌ای، متغیرها، مؤلفه‌های اختیاری و فهرست‌ها، یا ارجاعی به یک الگوی کد دیگر، یا مقدار ویژهٔ <undefined> است. متغیرها جانگهدارنده‌هایی هستند که در مرحلهٔ تولید کد جایگزین می‌شوند. متغیرهای فهرست خاص هستند: هنگام بسط الگوهای فهرست، re2zig عبارت‌های سمت راست دونقطه را چند بار تکرار می‌کند، و هر بار رخدادهای متغیر فهرست را با مقداری مخصوص به همان تکرار جایگزین می‌سازد. فهرست‌ها دارای کران‌های اختیاری هستند (مقادیر منفی از انتها شمرده می‌شوند، برای نمونه -1 به معنای آخرین مؤلفه است). نام‌های شرطی با یک نقطه شروع می‌شوند. هم شرط‌ها و هم متغیرها می‌توانند محلی (مخصوص همان الگوی کد) یا سراسری (مجاز در تمام الگوهای کد) باشند. هنگامی که re2zig فایل نحو را می‌خواند، بررسی می‌کند که هر الگوی کد تنها از متغیرها و شرط‌هایی استفاده کند که در آن مجاز شناخته شده‌اند.

برای نمونه، الگوی کد زیر ساختار if-then-else را برای یک زبان شبه C تعریف می‌کند:

code:if_then_else =
    [branch{0}: topindent "if " cond " {" nl
        indent [stmt: stmt] dedent]
    [branch{1:-1}: topindent "} else" (.cond ? " if " cond) " {" nl
        indent [stmt: stmt] dedent]
    topindent "}" nl;

در اینجا branch یک متغیر فهرست است: branch{0} به شاخهٔ نخست بسط می‌یابد (که خاص است، چرا که بخش else ندارد)، و branch{1:-1} به تمام شاخه‌های باقی‌مانده (در صورت وجود) بسط می‌یابد. stmt نیز یک متغیر فهرست است: [stmt: stmt] یک فهرست تو در تو است که به فهرستی از دستورها در بدنهٔ شاخهٔ فعلی بسط پیدا می‌کند. topindent، indent، dedent و nl متغیرهای سراسری هستند، و .cond یک شرط محلی است (مفهوم آن‌ها در ادامه شرح داده شده است). این الگوی کد می‌تواند کد زیر را تولید کند:

if x {
    // do something
} else if y {
    // do something else
} else {
    // don't do anything
}

در ادامه فهرستی از تمام الگوهای کد پشتیبانی‌شده در re2zig به همراه متغیرها و شرط‌های محلی آن‌ها آورده شده است. توجه داشته باشید که یک تعریف مشخص می‌تواند از متغیرها و شرط‌های محلی استفاده کند، اما الزامی به این کار ندارد. تمام الگوهای کد بدون استفاده باید روی <undefined> تنظیم شوند.

در اینجا فهرستی از تمام متغیرهای سراسری مجاز در فایل‌های نحو (syntax files) آمده است:

در اینجا فهرستی از تمام شرط‌های سراسریِ مجاز در فایل‌های نحو (syntax) آورده شده است:

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
// Expects a null-terminated string.
fn lex(yyinput: [:0]const u8) i32 {
    var yycursor: u32 = 0;
    var count: i32 = 0;
    loop: while (true) {
        %{
            re2c:yyfill:enable = 0;
            *      { return -1; }
            [\x00] { return count; }
            [a-z]+ { count += 1; continue :loop; }
            [ ]+   { continue :loop; }
        %}
    }
}
test {
    try std.testing.expectEqual(lex(""), 0);
    try std.testing.expectEqual(lex("one two three"), 3);
    try std.testing.expectEqual(lex("f0ur"), -1);
}

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
// Expects a null-terminated string.
fn lex(yyinput: [:0]const u8) i32 {
    var yycursor: usize = 0;
    var yymarker: usize = 0;
    const yylimit: usize = yyinput.len; // points at the terminating null
    var count: i32 = 0;
    loop: while (true) {
        %{
            re2c:yyfill:enable = 0;
            re2c:eof = 0;
            str = ['] ([^'\\] | [\\][^])* ['];
            *    { return -1; }
            $    { return count; }
            str  { count += 1; continue :loop; }
            [ ]+ { continue :loop; }
        %}
    }
}
test {
    try std.testing.expectEqual(lex(""), 0);
    try std.testing.expectEqual(lex("'qu\x00tes' 'are' 'fine: \\'' "), 3);
    try std.testing.expectEqual(lex("'unterminated\\'"), -1);
}

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
%{max %}
fn lex(str: []const u8) !i32 {
    // Create a copy of the input string padded with yymaxfill zeroes at the end.
    var yyinput = try std.testing.allocator.alloc(u8, str.len + yymaxfill);
    defer std.testing.allocator.free(yyinput);
    std.mem.copyForwards(u8, yyinput[0..], str);
    std.mem.copyForwards(u8, yyinput[str.len..], &[_]u8{0} ** yymaxfill); // zero padding
    var yycursor: usize = 0;
    const yylimit: usize = yyinput.len;
    var count: i32 = 0;
    loop: while (true) {
        %{
            re2c:YYFILL = "return -1;";
            str = ['] ([^'\\] | [\\][^])* ['];
            [\x00] {
                // Check that it is the sentinel, not some unexpected null.
                return if (yycursor - 1 == str.len) count else -1;
            }
            str  { count += 1; continue :loop; }
            [ ]+ { continue :loop; }
            *    { return -1; }
        %}
    }
}
test {
    try std.testing.expectEqual(lex(""), 0);
    try std.testing.expectEqual(lex("'qu\x00tes' 'are' 'fine: \\'' "), 3);
    try std.testing.expectEqual(lex("'unterminated\\'"), -1);
    try std.testing.expectEqual(lex("'unexpected \x00 null\\'"), -1);
}

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
// Expects a string without terminating null.
fn lex(str: []const u8) i32 {
    var cur: usize = 0;
    var count: i32 = 0;
    loop: while (true) {
        %{
            re2c:api = generic;
            re2c:yyfill:enable = 0;
            // YYPEEK returns "fake" terminating null if cursor has reached limit.
            re2c:YYPEEK = "if (cur >= str.len) 0 else str[cur]";
            re2c:YYSKIP = "cur += 1;";
            *      { return -1; }
            [\x00] { return count; }
            [a-z]+ { count += 1; continue :loop; }
            [ ]+   { continue :loop; }
        %}
    }
}
test {
    try std.testing.expectEqual(lex(""), 0);
    try std.testing.expectEqual(lex("one two three"), 3);
    try std.testing.expectEqual(lex("f0ur"), -1);
}

               <-- shift -->
             >-A------------B---------C-------------D#-----------E->
             buffer       token    marker         limit,
                                                  cursor
>-A------------B---------C-------------D------------E#->
             buffer,  marker        cursor        limit
             token
               <-- shift -->
             >-A------------B---------C-------------D#--E (EOF)
             buffer       token    marker         limit,
                                                  cursor
>-A------------B---------C-------------D---E#........
             buffer,  marker       cursor limit
             token

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
const bufsize = 4095;
const State = struct {
    file: *std.Io.Reader,
    yyinput: [bufsize + 1]u8,
    yycursor: usize,
    yymarker: usize,
    yylimit: usize,
    token: usize,
    eof: bool
};
fn fill(st: *State) i32 {
    if (st.eof) { return -1; } // unexpected EOF
    // Error: lexeme too long. In real life can reallocate a larger buffer.
    if (st.token < 1) { return -2; }
    // Shift buffer contents (discard everything up to the current token).
    std.mem.copyBackwards(
        u8, st.yyinput[0..st.yylimit - st.token], st.yyinput[st.token..st.yylimit]);
    st.yycursor -= st.token;
    st.yymarker = @subWithOverflow(st.yymarker, st.token)[0];
    st.yylimit -= st.token;
    st.token = 0;
    // Fill free space at the end of buffer with new data from file.
    st.yylimit += st.file.readSliceShort(st.yyinput[st.yylimit..bufsize]) catch 0;
    st.yyinput[st.yylimit] = 0; // append sentinel symbol
    // If read less than expected, this is the end of input.
    st.eof = st.yylimit < bufsize;
    return 0;
}
fn lex(yyrecord: *State) i32 {
    var count: i32 = 0;
    loop: while (true) {
        yyrecord.token = yyrecord.yycursor;
        %{
            re2c:api = record;
            re2c:eof = 0;
            re2c:YYFILL = "fill(yyrecord) == 0";
            str = ['] ([^'\\] | [\\][^])* ['];
            *    { return -1; }
            $    { return count; }
            str  { count += 1; continue :loop; }
            [ ]+ { continue :loop; }
        %}
    }
}
test {
    const fname = "input";
    const content = "'qu\x00tes' 'are' 'fine: \\'' " ** bufsize;
    const count = 3 * bufsize; // number of quoted strings written to file
    // Prepare input file: a few times the size of the buffer, containing
    // strings with zeroes and escaped quotes.
    var fw = try std.fs.cwd().createFile(fname, .{});
    try fw.writeAll(content);
    fw.close();
    // Prepare lexer state: all offsets are at the end of buffer.
    // Use unbuffered reader - lexer does its own buffering.
    const zerobuf: [0]u8 = undefined;
    var fr = try std.fs.cwd().openFile(fname, .{.mode = .read_only});
    var reader = fr.reader(&zerobuf);
    var st = State{
        .file = &reader.interface,
        .yyinput = undefined,
        .yycursor = bufsize,
        .yymarker = bufsize,
        .yylimit = bufsize,
        .token = bufsize,
        .eof = false,
    };
    // Sentinel at `yylimit` offset is set to zero, which triggers YYFILL.
    st.yyinput[st.yylimit] = 0;
    // Run the lexer.
    try std.testing.expectEqual(lex(&st), count);
    // Cleanup: remove input file.
    fr.close();
    try std.fs.cwd().deleteFile(fname);
}

               <-- shift -->                 <-- need -->
             >-A------------B---------C-----D-------E---F--------G->
             buffer       token    marker cursor  limit
>-A------------B---------C-----D-------E---F--------G->
             buffer,  marker cursor               limit
             token
               <-- shift -->                 <-- need -->
             >-A------------B---------C-----D-------E-F        (EOF)
             buffer       token    marker cursor  limit
>-A------------B---------C-----D-------E-F###############
             buffer,  marker cursor                   limit
             token                        <- YYMAXFILL ->

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
%{max %}
const bufsize = 4096;
const State = struct {
    file: *std.Io.Reader,
    yyinput: [bufsize + yymaxfill]u8,
    yycursor: usize,
    yymarker: usize,
    yylimit: usize,
    token: usize,
    eof: bool
};
fn fill(st: *State, need: usize) i32 {
    if (st.eof) { return -1; } // unexpected EOF
    // Error: lexeme too long. In real life can reallocate a larger buffer.
    if (st.token < need) { return -2; }
    // Shift buffer contents (discard everything up to the current token).
    std.mem.copyBackwards(
        u8, st.yyinput[0..st.yylimit - st.token], st.yyinput[st.token..st.yylimit]);
    st.yycursor -= st.token;
    st.yymarker = @subWithOverflow(st.yymarker, st.token)[0];
    st.yylimit -= st.token;
    st.token = 0;
    // Fill free space at the end of buffer with new data from file.
    st.yylimit += st.file.readSliceShort(st.yyinput[st.yylimit..bufsize]) catch 0;
    // If read less than expected, this is the end of input.
    if (st.yylimit < bufsize) {
        st.eof = true;
        @memset(st.yyinput[st.yylimit..st.yylimit + yymaxfill], 0);
        st.yylimit += yymaxfill;
    }
    return 0;
}
fn lex(yyrecord: *State) i32 {
    var count: i32 = 0;
    loop: while (true) {
        yyrecord.token = yyrecord.yycursor;
        %{
            re2c:api = record;
            re2c:YYFILL = "{ if (fill(yyrecord, @@) != 0) return -2; }";
            str = ['] ([^'\\] | [\\][^])* ['];
            [\x00] {
                // Check that it is the sentinel, not some unexpected null.
                return if (yyrecord.token == yyrecord.yylimit - yymaxfill) count else -1;
            }
            str  { count += 1; continue :loop; }
            [ ]+ { continue :loop; }
            *    { return -1; }
        %}
    }
}
test {
    const fname = "input";
    const content = "'qu\x00tes' 'are' 'fine: \\'' " ** bufsize;
    const count = 3 * bufsize; // number of quoted strings written to file
    // Prepare input file: a few times the size of the buffer, containing
    // strings with zeroes and escaped quotes.
    var fw = try std.fs.cwd().createFile(fname, .{});
    try fw.writeAll(content);
    fw.close();
    // Prepare lexer state: all offsets are at the end of buffer.
    // This immediately triggers YYFILL, as the YYLESSTHAN condition is true.
    // Use unbuffered reader - lexer does its own buffering.
    const zerobuf: [0]u8 = undefined;
    var fr = try std.fs.cwd().openFile(fname, .{.mode = .read_only});
    var reader = fr.reader(&zerobuf);
    var st = State{
        .file = &reader.interface,
        .yyinput = undefined,
        .yycursor = bufsize,
        .yymarker = bufsize,
        .yylimit = bufsize,
        .token = bufsize,
        .eof = false,
    };
    @memset(st.yyinput[st.yylimit..st.yylimit + yymaxfill], 0); // zero-padding at the end
    // Run the lexer.
    try std.testing.expectEqual(lex(&st), count);
    // Cleanup: remove input file.
    fr.close();
    try std.fs.cwd().deleteFile(fname);
}

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
// Store u32 number in u64 during parsing to simplify overflow handling.
const State = struct {
    yyinput: [:0]const u8,
    yycursor: usize,
    yymarker: usize,
    num: u64,
};
// Common re2c definitions shared between all functions.
%{
    re2c:api = record;
    re2c:yyrecord = st;
    re2c:yyfill:enable = 0;
%}
const ERROR: u64 = @as(u64, std.math.maxInt(u32)) + 1; // overflow
// Add digit with the given base, checking for overflow.
fn add(st: *State, offs: u8, base: u64) void {
    const digit = st.yyinput[st.yycursor - 1] - offs;
    st.num = @min(st.num * base + digit, ERROR);
}
// Convert u64 to optional u32 (null meaning overflow or parse error).
fn maybeU32(num: u64) ?u32 {
    return if (num < ERROR) @intCast(num) else null;
}
fn parse_u32(s: [:0]const u8) ?u32 {
    var st = State {.yyinput = s, .yycursor = 0, .yymarker = 0, .num = 0};
    %{
        '0b' / [01]        { return parse_bin(&st); }
        "0"                { return parse_oct(&st); }
        "" / [1-9]         { return parse_dec(&st); }
        '0x' / [0-9a-fA-F] { return parse_hex(&st); }
        *                  { return null; }
    %}
}
fn parse_bin(st: *State) ?u32 {
    bin: while (true) {%{
        [01] { add(st, 48, 2); continue :bin; }
        *    { return maybeU32(st.num); }
    %}}
}
fn parse_oct(st: *State) ?u32 {
    oct: while (true) {%{
        [0-7] { add(st, 48, 8); continue :oct; }
        *     { return maybeU32(st.num); }
    %}}
}
fn parse_dec(st: *State) ?u32 {
    dec: while (true) {%{
        [0-9] { add(st, 48, 10); continue :dec; }
        *     { return maybeU32(st.num); }
    %}}
}
fn parse_hex(st: *State) ?u32 {
    hex: while (true) {%{
        [0-9] { add(st, 48, 16); continue :hex; }
        [a-f] { add(st, 87, 16); continue :hex; }
        [A-F] { add(st, 55, 16); continue :hex; }
        *     { return maybeU32(st.num); }
    %}}
}
test {
    try std.testing.expectEqual(parse_u32(""), null);
    try std.testing.expectEqual(parse_u32("1234567890"), 1234567890);
    try std.testing.expectEqual(parse_u32("0b1101"), 13);
    try std.testing.expectEqual(parse_u32("0x7Fe"), 2046);
    try std.testing.expectEqual(parse_u32("0644"), 420);
    try std.testing.expectEqual(parse_u32("9999999999"), null);
}

// re2zig $INPUT -o $OUTPUT -c
const std = @import("std");
%{conditions %}
const ERROR: u64 = @as(u64, std.math.maxInt(u32)) + 1; // overflow
// Add digit with the given base, checking for overflow.
fn add(num: *u64, str: [:0]const u8, cur: usize, offs: u8, base: u64) void {
    num.* = @min(num.* * base + (str[cur - 1] - offs), ERROR);
}
fn parse_u32(yyinput: [:0]const u8) ?u32 {
    var yycursor: usize = 0;
    var yymarker: usize = 0;
    var yycond = yycinit;
    var num: u64 = 0; // Store number in u64 to simplify overflow checks.
    loop: while(true) {
    %{
        re2c:yyfill:enable = 0;
        <init> '0b' / [01]        :=> bin
        <init> "0"                :=> oct
        <init> "" / [1-9]         :=> dec
        <init> '0x' / [0-9a-fA-F] :=> hex
        <init> * { return null; }
        <bin> [01]  { add(&num, yyinput, yycursor, 48, 2);  continue :loop; }
        <oct> [0-7] { add(&num, yyinput, yycursor, 48, 8);  continue :loop; }
        <dec> [0-9] { add(&num, yyinput, yycursor, 48, 10); continue :loop; }
        <hex> [0-9] { add(&num, yyinput, yycursor, 48, 16); continue :loop; }
        <hex> [a-f] { add(&num, yyinput, yycursor, 87, 16); continue :loop; }
        <hex> [A-F] { add(&num, yyinput, yycursor, 55, 16); continue :loop; }
        <bin, oct, dec, hex> * {
            return if (num < ERROR) @intCast(num) else null;
        }
    %}}
}
test {
    try std.testing.expectEqual(parse_u32(""), null);
    try std.testing.expectEqual(parse_u32("1234567890"), 1234567890);
    try std.testing.expectEqual(parse_u32("0b1101"), 13);
    try std.testing.expectEqual(parse_u32("0x7Fe"), 2046);
    try std.testing.expectEqual(parse_u32("0644"), 420);
    try std.testing.expectEqual(parse_u32("9999999999"), null);
}

// re2zig $INPUT -o $OUTPUT -f
const std = @import("std");
const Status = enum {
    end,
    ready,
    waiting,
    bad_packet,
    big_packet
};
// Use a small buffer to cover the case when a lexeme doesn't fit.
// In real world use a larger buffer.
const bufsize = 10;
const State = struct {
    file: std.fs.File.Reader,
    yyinput: [bufsize + 1]u8,
    yycursor: usize,
    yymarker: usize,
    yylimit: usize,
    token: usize,
    yystate: i32,
    received: usize,
};
fn fill(st: *State) Status {
    // Error: lexeme too long. In real life can reallocate a larger buffer.
    if (st.token < 1) { return Status.big_packet; }
    // Shift buffer contents (discard everything up to the current token).
    std.mem.copyBackwards(
        u8, st.yyinput[0..st.yylimit - st.token], st.yyinput[st.token..st.yylimit]);
    st.yycursor -= st.token;
    st.yymarker = @subWithOverflow(st.yymarker, st.token)[0];
    st.yylimit -= st.token;
    st.token = 0;
    // Fill free space at the end of buffer with new data from file.
    st.yylimit += st.file.interface.readSliceShort(st.yyinput[st.yylimit..bufsize]) catch 0;
    st.yyinput[st.yylimit] = 0; // append sentinel symbol
    return Status.ready;
}
fn lex(yyrecord: *State) Status {
    var yych: u8 = 0;
    loop: while (true) {
        yyrecord.token = yyrecord.yycursor;
        %{
            re2c:api = record;
            re2c:eof = 0;
            re2c:YYFILL = "return Status.waiting;";
            packet = [a-z]+[;];
            *      { return Status.bad_packet; }
            $      { return Status.end; }
            packet { yyrecord.received += 1; continue :loop; }
        %}
    }
}
fn run(expect: Status, packets: []const []const u8) !void {
    // Create a "pipe" (open the same file for reading and writing).
    const fname = "input";
    var fw = try std.fs.cwd().createFile(fname, .{});
    var fr = try std.fs.cwd().openFile(fname, .{ .mode = .read_only});
    // Initialize lexer state: `state` value is -1, all offsets are at the end
    // of buffer. Use unbuffered reader - lexer does its own buffering.
    const zerobuf: [0]u8 = undefined;
    var st = State{
        .file = fr.reader(&zerobuf),
        .yyinput = undefined,
        .yycursor = bufsize,
        .yymarker = bufsize,
        .yylimit = bufsize,
        .token = bufsize,
        .yystate = -1,
        .received = 0,
    };
    // Sentinel at `yylimit` offset is set to zero, which triggers YYFILL.
    st.yyinput[st.yylimit] = 0;
    // Main loop. The buffer contains incomplete data which appears packet by
    // packet. When the lexer needs more input it saves its internal state and
    // returns to the caller which should provide more input and resume lexing.
    var status = Status.ready;
    var send: usize = 0;
    while (true) {
        status = lex(&st);
        if (status == Status.end) {
            break;
        } else if (status == Status.waiting) {
            if (send < packets.len) {
                std.log.debug("sending packet {}", .{send});
                try fw.writeAll(packets[send]);
                send += 1;
            }
            status = fill(&st);
            std.log.debug("filled buffer [{s}], status {}", .{st.yyinput, status});
            if (status != Status.ready) {
                break;
            }
        } else if (status == Status.bad_packet) {
            break;
        }
    }
    // Check results.
    try std.testing.expectEqual(status, expect);
    if (status == Status.end) { try std.testing.expectEqual(st.received, send); }
    // Cleanup: remove input file.
    fw.close();
    fr.close();
    try std.fs.cwd().deleteFile(fname);
}
test {
    try run(Status.end, &[_][]const u8{});
    try run(Status.end, &[_][]const u8{"zero;", "one;", "two;", "three;", "four;"});
    try run(Status.bad_packet, &[_][]const u8{"??;"});
    try run(Status.big_packet, &[_][]const u8{"looooooooooooong;"});
}

// re2zig $INPUT -o $OUTPUT
// This example shows how to combine reusable re2c blocks: two blocks
// ('colors' and 'fish') are merged into one. The 'salmon' rule occurs
// in both blocks; the 'fish' block takes priority because it is used
// earlier. Default rule * occurs in all three blocks; the local (not
// inherited) definition takes priority.
const std = @import("std");
const Ans = enum {color, fish, dunno};
%{rules:colors
    *                            { @panic("ah"); }
    "red" | "salmon" | "magenta" { return Ans.color; }
%}
%{rules:fish
    *                            { @panic("oh"); }
    "haddock" | "salmon" | "eel" { return Ans.fish; }
%}
fn lex(yyinput: [:0]const u8) Ans {
    var yycursor: usize = 0;
    var yymarker: usize = 0;
    %{
        re2c:yyfill:enable = 0;
        !use:fish;
        !use:colors;
        * { return Ans.dunno; } // overrides inherited '*' rules
    %}
}
test {
    try std.testing.expectEqual(lex("salmon"), Ans.fish);
    try std.testing.expectEqual(lex("what?"), Ans.dunno);
}

// re2zig $INPUT -o $OUTPUT --input-encoding utf8
// This example supports multiple input encodings: UTF-8 and UTF-32.
// Both lexers are generated from the same rules block, and the use
// blocks add only encoding-specific configurations.
const std = @import("std");
%{rules
    re2c:yyfill:enable = 0;
    "∀x ∃y" { return yycursor; }
    *       { return null; }
%}
fn lex_utf8(yyinput: []const u8) ?usize {
    var yycursor: usize = 0;
    var yymarker: usize = 0;
    %{use
        re2c:encoding:utf8 = 1;
        re2c:YYCTYPE = u8; // the default
    %}
}
fn lex_utf32(yyinput: []const u32) ?usize {
    var yycursor: usize = 0;
    var yymarker: usize = 0;
    %{use
        re2c:encoding:utf32 = 1;
        re2c:YYCTYPE = u32;
    %}
}
test {
    const s8 = [_]u8{0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79};
    try std.testing.expectEqual(lex_utf8(&s8), s8.len);
    const s32 = [_]u32{0x2200, 0x78, 0x20, 0x2203, 0x79};
    try std.testing.expectEqual(lex_utf32(&s32), s32.len);
}

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
const SemVer = struct {
    major: u32,
    minor: u32,
    patch: u32,
};
const none = std.math.maxInt(usize);
fn s2n(str: []const u8) u32 { // convert a pre-parsed string to a number
    var n: u32 = 0;
    for (str) |c| { n = n * 10 + (c - 48); }
    return n;
}
fn parse(yyinput: [:0]const u8) ?SemVer {
    var yycursor: usize = 0;
    var yymarker: usize = 0;
    // Final tag variables available in semantic action.
    %{svars format = "var @@: usize = none;"; %}
    // Intermediate tag variables used by the lexer (must be autogenerated).
    %{stags format = "var @@: usize = none;"; %}
    %{
        re2c:yyfill:enable = 0;
        re2c:tags = 1;
        num = [0-9]+;
        @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\x00] {
            return SemVer {
                .major = s2n(yyinput[t1..t2]),
                .minor = s2n(yyinput[t3..t4]),
                .patch = if (t5 == none) 0 else s2n(yyinput[t5..yycursor - 1]),
            };
        }
        * { return null; }
    %}
}
test {
    try std.testing.expectEqual(parse("23.34"), SemVer{.major = 23, .minor = 34, .patch = 0});
    try std.testing.expectEqual(parse("1.2.99999"), SemVer{.major = 1, .minor = 2, .patch = 99999});
    try std.testing.expectEqual(parse("1.a"), null);
}

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
const bufsize = 4095;
const none = std.math.maxInt(usize);
const err = error.SyntaxError;
const SemVer = struct {
    major: u32,
    minor: u32,
    patch: u32,
};
fn s2n(str: []const u8) u32 { // convert a pre-parsed string to a number
    var n: u32 = 0;
    for (str) |c| { n = n * 10 + (c - 48); }
    return n;
}
const State = struct {
    file: *std.Io.Reader,
    yyinput: [bufsize + 1]u8,
    yycursor: usize,
    yymarker: usize,
    yylimit: usize,
    token: usize,
    // Intermediate tag variables must be part of the lexer state passed to YYFILL.
    // They don't correspond to tags and should be autogenerated by re2c.
    %{stags format = "@@: usize,\n"; %}
    eof: bool
};
fn fill(st: *State) i32 {
    if (st.eof) { return -1; } // unexpected EOF
    // Error: lexeme too long. In real life can reallocate a larger buffer.
    if (st.token < 1) { return -2; }
    // Shift buffer contents (discard everything up to the current token).
    std.mem.copyBackwards(
        u8, st.yyinput[0..st.yylimit - st.token], st.yyinput[st.token..st.yylimit]);
    st.yycursor -= st.token;
    st.yymarker = @subWithOverflow(st.yymarker, st.token)[0];
    st.yylimit -= st.token;
    // Tag variables need to be shifted like other input positions. The check
    // for NONE is only needed if some tags are nested inside of alternative or
    // repetition, so that they can have NONE value.
    %{stags format = "if (st.@@ != none) st.@@ = @subWithOverflow(st.@@, st.token)[0];\n"; %}
    st.token = 0;
    // Fill free space at the end of buffer with new data from file.
    st.yylimit += st.file.readSliceShort(st.yyinput[st.yylimit..bufsize]) catch 0;
    st.yyinput[st.yylimit] = 0; // append sentinel symbol
    // If read less than expected, this is the end of input.
    st.eof = st.yylimit < bufsize;
    return 0;
}
fn parse(st: *State) !std.ArrayList(SemVer) {
    var vers = try std.ArrayList(SemVer).initCapacity(std.testing.allocator, 0);
    // Final tag variables available in semantic action.
    %{svars format = "var @@: usize = 0;\n"; %}
    loop: while (true) {
        st.token = st.yycursor;
        %{
            re2c:api = record;
            re2c:eof = 0;
            re2c:tags = 1;
            re2c:yyrecord = st;
            re2c:YYFILL = "fill(st) == 0";
            num = [0-9]+;
            num @t1 "." @t2 num @t3 ("." @t4 num)? [\n] {
                try vers.append(std.testing.allocator, SemVer {
                    .major = s2n(st.yyinput[st.token..t1]),
                    .minor = s2n(st.yyinput[t2..t3]),
                    .patch = if (t4 == none) 0 else s2n(st.yyinput[t4..st.yycursor - 1]),
                });
                continue :loop;
            }
            $ { return vers; }
            * { return error.SyntaxError; }
        %}
    }
}
test {
    const fname = "input";
    const content = "1.22.333\n" ** bufsize;
    // Prepare input file: a few times the size of the buffer, containing
    // strings with zeroes and escaped quotes.
    var fw = try std.fs.cwd().createFile(fname, .{});
    try fw.writeAll(content);
    fw.close();
    // Prepare lexer state: all offsets are at the end of buffer.
    // Use unbuffered reader - lexer does its own buffering.
    const zerobuf: [0]u8 = undefined;
    var fr = try std.fs.cwd().openFile(fname, .{.mode = .read_only});
    var reader = fr.reader(&zerobuf);
    var st = State{
        .file = &reader.interface,
        .yyinput = undefined,
        .yycursor = bufsize,
        .yymarker = bufsize,
        .yylimit = bufsize,
        .token = bufsize,
        %{stags format = ".@@ = none,\n"; %}
        .eof = false,
    };
    // Sentinel at `yylimit` offset is set to zero, which triggers YYFILL.
    st.yyinput[st.yylimit] = 0;
    // Manually construct expected result.
    var expect = [_]SemVer{SemVer{.major = 1, .minor = 22, .patch = 333}} ** bufsize;
    // Run the lexer.
    var result = try parse(&st);
    try std.testing.expectEqualDeep(&expect, result.items);
    // Cleanup: free memory and remove input file.
    result.deinit(std.testing.allocator);
    fr.close();
    try std.fs.cwd().deleteFile(fname);
}

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
const none = std.math.maxInt(usize);
const SemVer = struct {
    major: u32,
    minor: u32,
    patch: u32,
};
fn s2n(str: []const u8) u32 { // convert pre-parsed string to a number
    var n: u32 = 0;
    for (str) |c| { n = n * 10 + (c - 48); }
    return n;
}
fn parse(yyinput: [:0]const u8) ?SemVer {
    var yycursor: usize = 0;
    var yymarker: usize = 0;
    // Final tag variables available in semantic action.
    %{svars format = "var @@: usize = none;"; %}
    // Intermediate tag variables used by the lexer (must be autogenerated).
    %{stags format = "var @@: usize = none;"; %}
    %{
        re2c:yyfill:enable = 0;
        re2c:captvars = 1;
        num = [0-9]+;
        (num) "." (num) ("." num)? [\x00] {
            return SemVer {
                .major = s2n(yyinput[yytl1..yytr1]),
                .minor = s2n(yyinput[yytl2..yytr2]),
                .patch = if (yytl3 == none) 0 else s2n(yyinput[yytl3 + 1..yytr3])
            };
        }
        * { return null; }
    %}
}
test {
    try std.testing.expectEqual(parse("23.34"), SemVer{.major = 23, .minor = 34, .patch = 0});
    try std.testing.expectEqual(parse("1.2.99999"), SemVer{.major = 1, .minor = 2, .patch = 99999});
    try std.testing.expectEqual(parse("1.a"), null);
}

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
const none = std.math.maxInt(usize);
const mtag_root = none - 1;
const err = error.SyntaxError;
// An m-tag tree is a way to store histories with an O(1) copy operation.
// Histories naturally form a tree, as they have common start and fork at some
// point. The tree is stored as an array of pairs (tag value, link to parent).
// An m-tag is represented with a single link in the tree (array index).
const MtagElem = struct {
    elem: usize, // tag value
    pred: usize, // index of the predecessor node or root
};
// Append a single value to an m-tag history.
fn add_mtag(trie: *std.ArrayList(MtagElem), mtag: usize, value: usize) !usize {
    try trie.append(std.testing.allocator, MtagElem{.elem = value, .pred = mtag});
    return trie.items.len - 1;
}
// Recursively unwind tag histories and collect version components.
fn unwind(trie: *std.ArrayList(MtagElem),
          x: usize,
          y: usize,
          str: []const u8,
          ver: *std.ArrayList(u32)) !void {
    // Reached the root of the m-tag tree, stop recursion.
    if (x == mtag_root and y == mtag_root) return;
    // Unwind history further.
    try unwind(trie, trie.items[x].pred, trie.items[y].pred, str, ver);
    // Get tag values. Tag histories must have equal length.
    std.debug.assert(x != mtag_root and y != mtag_root);
    const ex = trie.items[x].elem;
    const ey = trie.items[y].elem;
    if (ex != none and ey != none) {
        // Both tags are valid string indices, extract component.
        try ver.append(std.testing.allocator, s2n(str[ex..ey]));
    } else {
        // Both tags are none (this corresponds to zero repetitions).
        std.debug.assert(ex == none and ey == none);
    }
}
fn s2n(str: []const u8) u32 { // convert a pre-parsed string to a number
    var n: u32 = 0;
    for (str) |c| { n = n * 10 + (c - 48); }
    return n;
}
fn parse(yyinput: [:0]const u8) !std.ArrayList(u32) {
    var yycursor: usize = 0;
    var yymarker: usize = 0;
    var mt = try std.ArrayList(MtagElem).initCapacity(std.testing.allocator, 0);
    defer mt.deinit(std.testing.allocator);
    // Final tag variables available in semantic action.
    %{svars format = "var @@: usize = none;"; %}
    %{mvars format = "var @@: usize = mtag_root;"; %}
    // Intermediate tag variables used by the lexer (must be autogenerated).
    %{stags format = "var @@: usize = none;"; %}
    %{mtags format = "var @@: usize = mtag_root;"; %}
    %{
        re2c:YYMTAGP = "@@ = add_mtag(&mt, @@, yycursor) catch none;";
        re2c:YYMTAGN = "@@ = add_mtag(&mt, @@, none) catch none;";
        re2c:yyfill:enable = 0;
        re2c:tags = 1;
        num = [0-9]+;
        @t1 num @t2 ("." #t3 num #t4)* [\x00] {
            var ver = try std.ArrayList(u32).initCapacity(std.testing.allocator, 0);
            try ver.append(std.testing.allocator, s2n(yyinput[t1..t2]));
            try unwind(&mt, t3, t4, yyinput, &ver);
            return ver;
        }
        * { return error.SyntaxError; }
    %}
}
test {
    var result = try parse("1");
    var expect = [_]u32{1};
    try std.testing.expectEqualDeep(&expect, result.items);
    result.deinit(std.testing.allocator);
}
test {
    var result = try parse("1.2.3.4.5.6.7");
    var expect = [_]u32{1, 2, 3, 4, 5, 6, 7};
    try std.testing.expectEqualDeep(&expect, result.items);
    result.deinit(std.testing.allocator);
}
test {
    const result = parse("1.2.") catch null;
    try std.testing.expectEqualDeep(result, null);
}

// re2zig $INPUT -o $OUTPUT --utf8
const std = @import("std");
%{include "unicode_categories.re" %}
fn lex(yyinput: [:0]const u8) bool {
    var yycursor: u32 = 0;
    var yymarker: u32 = 0;
    %{
        re2c:yyfill:enable = 0;
        // Simplified "Unicode Identifier and Pattern Syntax"
        // (see https://unicode.org/reports/tr31)
        id_start    = L | Nl | [$_];
        id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3];
        identifier  = id_start id_continue*;
        identifier { return true; }
        *          { return false; }
    %}
}
test {
    try std.testing.expect(lex("_Ыдентификатор"));
}

const Num = enum { integer, floating, nan };
%{
    number = [1-9][0-9]*;
%}

// floating-point numbers
frac  = [0-9]* "." [0-9]+ | [0-9]+ ".";
exp   = 'e' [+-]? [0-9]+;
float = frac exp? | [0-9]+ exp;
float { return Num.floating; }

// re2zig $INPUT -o $OUTPUT
const std = @import("std");
%{include "definitions.zig" %}
fn lex(yyinput: [:0]const u8) Num {
    var yycursor: u32 = 0;
    var yymarker: u32 = 0;
    %{
        re2c:yyfill:enable = 0;
        *      { return Num.nan; }
        number { return Num.integer; }
        !include "extra_rules.re.inc";
    %}
}
test {
    try std.testing.expectEqual(lex("123"), Num.integer);
    try std.testing.expectEqual(lex("123.4567"), Num.floating);
}

// re2zig $INPUT -o $OUTPUT --header lexer/state.zig
const std = @import("std");
const state = @import("lexer/state.zig"); // the module is generated by re2c
%{header:on %}
pub const State = struct {
    yyinput: [:0]const u8,
    yycursor: usize,
    %{stags format = "@@: usize,"; %}
};
%{header:off %}
fn lex(yyrecord: *state.State) usize {
    var t: usize = 0;
    %{
        re2c:header = "lexer/state.zig";
        re2c:api = record;
        re2c:yyfill:enable = 0;
        re2c:tags = 1;
        [a]* @t [b]* { return t; }
    %}
}
test {
    var st = state.State {
        .yyinput = "ab",
        .yycursor = 0,
        %{stags format = ".@@ = 0,"; %}
    };
    try std.testing.expectEqual(lex(&st), 1);
}

// Generated by re2c
pub const State = struct {
    yyinput: [:0]const u8,
    yycursor: usize,
    yyt1: usize,
};