RE2RUST(1) RE2RUST(1) (NAME) re2rust - Rust (SYNOPSIS) re2rust [ OPTIONS ] [ WARNINGS ] INPUT - stdin . (INTRODUCTION) re2rust . ( Rust ) / . . re2rust re2rust Rust . (decimal) : // re2rust $INPUT -o $OUTPUT --no-unsafe --api simple fn lex(yyinput: &[u8]) -> bool { let mut yycursor = 0; /*!re2c re2c:YYCTYPE = u8; re2c:yyfill:enable = 0; [1-9][0-9]* { return true; } * { return false; } */ } fn main() { assert!(lex(b"1234\0")); } re2rust : /* Generated by re2rust */ // re2rust $INPUT -o $OUTPUT --no-unsafe --api simple fn lex(yyinput: &[u8]) -> bool { let mut yycursor = 0; { #[allow(unused_assignments)] let mut yych : u8 = 0; let mut yystate : usize = 0; 'yyl: loop { match yystate { 0 => { yych = yyinput[yycursor]; yycursor += 1; match yych { 0x31 ..= 0x39 => { yystate = 2; continue 'yyl; } _ => { yystate = 1; continue 'yyl; } } } 1 => { return false; }, 2 => { yych = yyinput[yycursor]; match yych { 0x30 ..= 0x39 => { yycursor += 1; yystate = 2; continue 'yyl; } _ => { yystate = 3; continue 'yyl; } } } 3 => { return true; }, _ => panic!("internal lexer error"), } } } } fn main() { assert!(lex(b"1234\0")); } re2rust . : name = regular-expression ; name regular-expression . - . re2rust . name . . . . configuration = value ; re2rust . re2rust . value - . . . regular-expression code regular-expression ( := ). regular-expression code . . . : * $. ( ). . . !action code code ( ). . !directive ; . . . (Blocks) /*!re2c */ %{ %} ( ). 2.2 . : /*!re2c[:] ... */ %{[:] ... %} . re2rust . . . /*!local:re2c[:] ... */ %{local[:] ... %} ( ). /*!rules:re2c[:] ... */ %{rules[:] ... %} -- . ( ). 2.2 re2rust -r --reusable . /*!use:re2c[:] ... */ %{use[:] ... %} . re2rust . ( ). . 2.2 re2rust -r --reusable . /*!max:re2c[:[:...]] ... */ %{max[:[:...]] ... %} YYMAXFILL . YYMAXFILL ( ). C (#define YYMAXFILL ) Go (var YYMAXFILL int = ) . format @@{max} ( @@) YYMAXFILL . /*!maxnmatch:re2c[:[:...]] ... */ %{maxnmatch[:[:...]] ... %} YYMAXNMATCH ( -P --posix-captures ). YYMAXNMATCH ( ). C (#define YYMAXNMATCH ) Go (var YYMAXNMATCH int = ) . format @@{max} ( @@) YYMAXNMATCH . /*!stags:re2c[:[:...]] ... */, /*!mtags:re2c[:[:...]] ... */ %{stags[:[:...]] ... %}, %{mtags[:[:...]] ... %{ s-tag/m-tag re2rust . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!svars:re2c[:[:...]] ... */, /*!mvars:re2c[:[:...]] ... */ %{svars[:[:...]] ... %}, %{mvars[:[:...]] ... %{ s-tag/m-tag ( --tags) re2rust ( --captvars --posix-captvars) . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!getstate:re2c[:[:...]] ... */ %{getstate[:[:...]] ... %} ( --storable-state ). . . . --loop-switch goto . /*!conditions:re2c[:[:...]] ... */, /*!types:re2c... */ %{conditions[:[:...]] ... %}, %{types... %} ( --conditions ). ( ). YYCONDTYPE . format separator . format @@{cond} ( @@) @@{num} . separator format . /*!include:re2c */ %{include %} . #include C/C++. --depfile . /*!header:re2c:on*/ %{header:on %} . header:off re2rust -t --type-header . /*!header:re2c:off*/ %{header:off %} header:on*/ . /*!ignore:re2c ... */ %{ignore ... %} . re2rust : re2c:api, re2c:input --api. re2c:api:sigil ("sigil") API . @@ . sigil . sigil $ ${name} . . API (override) re2c:YYFILL@len YYFILL. re2c:api:style API . functions ( C) free-form ( Go Rust) . functions API . ( YYFILL) (lexer context) C/C++ (closures) Go Rust . free-form API : @@{var} @@ ( ). API (override) re2c:YYFILL:naked YYFILL . re2c:bit-vectors, re2c:flags:bit-vectors, re2c:flags:b --bit-vectors . re2c:captures, re2c:leftmost-captures --leftmost-captures . re2c:captvars, re2c:leftmost-captvars --leftmost-captvars . re2c:case-insensitive, re2c:flags:case-insensitive --case-insensitive . re2c:case-inverted, re2c:flags:case-inverted --case-inverted . re2c:case-ranges, re2c:flags:case-ranges --case-ranges . re2c:computed-gotos, re2c:flags:computed-gotos, re2c:flags:g --computed-gotos . re2c:computed-gotos:relative, re2c:cgoto:relative --computed-gotos-relative . re2c:computed-gotos:threshold, re2c:cgoto:threshold goto (jump tables) if (bitmaps) . 9 . re2c:cond:abort (default case) . re2c:cond:goto :=> . goto @@; . @@ ( re2c:api:sigil re2c:cond:goto@cond ). re2c:cond:goto@cond (sigil) re2c:cond:goto . @@ . re2c:api:sigil (override) . re2c:cond:divider . /* *********************************** */ . ( re2c:api;sigil re2c:cond:divider@cond ). re2c:cond:divider@cond (sigil) re2c:cond:divider . @@ . re2c:api:sigil (override) . re2c:cond:prefix, re2c:condprefix . yyc_ . re2c:cond:enumprefix, re2c:condenumprefix . yyc . re2c:debug-output, re2c:flags:debug-output, re2c:flags:d --debug-output . re2c:empty-class, re2c:flags:empty-class --empty-class . re2c:encoding:ebcdic, re2c:flags:ecb, re2c:flags:e --ebcdic . re2c:encoding:ucs2, re2c:flags:wide-chars, re2c:flags:w --ucs2 . re2c:encoding:utf8, re2c:flags:utf-8, re2c:flags:8 --utf8 . re2c:encoding:utf16, re2c:flags:utf-16, re2c:flags:x --utf16 . re2c:encoding:utf32, re2c:flags:unicode, re2c:flags:u --utf32 . re2c:encoding-policy, re2c:flags:encoding-policy --encoding-policy . re2c:eof (sentinel symbol) $ . -1 ( $ ). . . re2c:header, re2c:flags:type-header, re2c:flags:t . --header . re2c:indent:string . "\t" . . . re2c:indent:top . . . re2c:invert-captures --invert-captures . re2c:label:prefix, re2c:labelprefix DFA . yy . re2c:label:start, re2c:startlabel . . . . ( ). re2c:label:yyFillLabel YYFILL re2c:eof . re2c:label:yyloop --loop-switch . yyloop . re2c:label:yyNext (switch) YYGETSTATE ( re2c:state:nextlabel) . yyNext . re2c:lookahead, re2c:flags:lookahead ( --no-lookahead ). re2c:monadic (monadic notation) ( Haskell )..TP re2c:nested-ifs, re2c:flags:nested-ifs, re2c:flags:s --nested-ifs . re2c:posix-captures, re2c:flags:posix-captures, re2c:flags:P --posix-captures . re2c:posix-captvars --posix-captvars . re2c:tags, re2c:flags:tags, re2c:flags:T --tags . re2c:tags:expression . re2rust yyt . struct . @@{tag} @@ . re2c:tags:expression = "s.@@"; s.yyt . re2c:api:sigil . re2c:tags:negative ( -1 null ). re2c:tags:prefix . yyt . re2c:sentinel (sentinel) ( re2c:yyfill:enable = 0; re2c:eof ). : . -1` ( re2rust ). . re2c:state:abort (default case) (state dispatch) (abort) -1 . re2c:state:nextlabel switch YYGETSTATE yyNext ( ). re2c:label:start getstate switch YYGETSTATE . re2c:unsafe, re2c:flags:unsafe --no-unsafe . (wrappers) unsafe YYPEEK . ( ). Rust . re2c:YYBACKUP, re2c:define:YYBACKUP API YYBACKUP . re2c:YYBACKUPCTX, re2c:define:YYBACKUPCTX API YYBACKUPCTX . re2c:YYCONDTYPE, re2c:define:YYCONDTYPE API YYCONDTYPE . re2c:YYCTYPE, re2c:define:YYCTYPE API YYCTYPE . re2c:YYCTXMARKER, re2c:define:YYCTXMARKER API YYCTXMARKER . re2c:YYCURSOR, re2c:define:YYCURSOR API YYCURSOR . re2c:YYDEBUG, re2c:define:YYDEBUG API YYDEBUG . re2c:YYFILL, re2c:define:YYFILL API YYFILL . re2c:YYFILL@len, re2c:define:YYFILL@len (sigil) YYFILL . @@ . re2c:api:sigil . re2c:YYFILL:naked, re2c:define:YYFILL:naked re2c:api:style YYFILL . API (free-form) . re2c:YYFN API YYFN . re2c:YYINPUT API YYINPUT . re2c:YYGETCOND, re2c:define:YYGETCONDITION API YYGETCOND . re2c:YYGETCOND:naked, re2c:define:YYGETCONDITION:naked re2c:api:style YYGETCOND . API (free-form) . re2c:YYGETSTATE, re2c:define:YYGETSTATE API YYGETSTATE . re2c:YYGETSTATE:naked, re2c:define:YYGETSTATE:naked re2c:api:style YYGETSTATE . API (free-form) . re2c:YYGETACCEPT, re2c:define:YYGETACCEPT API YYGETACCEPT . re2c:YYLESSTHAN, re2c:define:YYLESSTHAN API YYLESSTHAN . re2c:YYLIMIT, re2c:define:YYLIMIT API YYLIMIT . re2c:YYMARKER, re2c:define:YYMARKER API YYMARKER . re2c:YYMTAGN, re2c:define:YYMTAGN API YYMTAGN . re2c:YYMTAGP, re2c:define:YYMTAGP API YYMTAGP . re2c:YYPEEK, re2c:define:YYPEEK API YYPEEK . re2c:YYRESTORE, re2c:define:YYRESTORE API YYRESTORE . re2c:YYRESTORECTX, re2c:define:YYRESTORECTX API YYRESTORECTX . re2c:YYRESTORETAG, re2c:define:YYRESTORETAG API YYRESTORETAG . re2c:YYSETCOND, re2c:define:YYSETCONDITION API YYSETCOND . re2c:YYSETCOND@cond, re2c:define:YYSETCONDITION@cond (sigil) YYSETCOND . @@ . re2c:api:sigil . re2c:YYSETCOND:naked, re2c:define:YYSETCONDITION:naked re2c:api:style YYSETCOND . API (free-form) . re2c:YYSETSTATE, re2c:define:YYSETSTATE API YYSETSTATE . re2c:YYSETSTATE@state, re2c:define:YYSETSTATE@state (sigil) YYSETSTATE . @@ . re2c:api:sigil . re2c:YYSETSTATE:naked, re2c:define:YYSETSTATE:naked re2c:api:style YYSETSTATE . API (free-form) . re2c:YYSETACCEPT, re2c:define:YYSETACCEPT API YYSETACCEPT . re2c:YYSKIP, re2c:define:YYSKIP API YYSKIP . re2c:YYSHIFT, re2c:define:YYSHIFT API YYSHIFT . re2c:YYCOPYMTAG, re2c:define:YYCOPYMTAG API YYCOPYMTAG . re2c:YYCOPYSTAG, re2c:define:YYCOPYSTAG API YYCOPYSTAG ..TP re2c:YYSHIFTMTAG, re2c:define:YYSHIFTMTAG API YYSHIFTMTAG . re2c:YYSHIFTSTAG, re2c:define:YYSHIFTSTAG API YYSHIFTSTAG . re2c:YYSTAGN, re2c:define:YYSTAGN API YYSTAGN . re2c:YYSTAGP, re2c:define:YYSTAGP API YYSTAGP . re2c:yyaccept, re2c:variable:yyaccept API yyaccept . re2c:yybm, re2c:variable:yybm API yybm . re2c:yybm:hex, re2c:variable:yybm:hex --bit-vectors . ( ). re2c:yych, re2c:variable:yych API yych . re2c:yych:emit, re2c:variable:yych:emit yych . . re2c:yych:conversion, re2c:variable:yych:conversion re2rust yych YYCTYPE . ( ). re2c:yych:literals, re2c:variable:yych:literals yych . : char ( ) hex ( ) char_or_hex ( ). re2c:yyctable, re2c:variable:yyctable API yyctable . re2c:yynmatch, re2c:variable:yynmatch API yynmatch . re2c:yypmatch, re2c:variable:yypmatch API yypmatch . re2c:yytarget, re2c:variable:yytarget API yytarget . re2c:yystable, re2c:variable:yystable . re2c:yystate, re2c:variable:yystate API yystate . re2c:yyfill, re2c:variable:yyfill API yyfill . re2c:yyfill:check YYFILL ( YYLESSTHAN API YYLIMIT API C) . ( ). re2c:yyfill:enable YYFILL ( ) . ( ) YYFILL ( ). ( -W) re2c:sentinel . (YYFILL ). re2c:yyfill:parameter YYFILL . . ( ). re2c:YYFILL:naked re2c:api:style . re2c:yyfn:sep YYFN ( ). re2c:yyfn:throw YYFN ( ). re2rust : "foo" . 'foo' . [a-xyz], [^a-xyz] ( ). . (newline). R \ S R S. R* R. R+ R. R? R. R{n} R n . R{n,} R n . R{n,m} R n m . (R) R ( ) . (R) --invert-captures (capturing) (non-capturing) . (!R) (!R) --invert-captures . R S : R S. R | S : R S. R / S (Lookahead): R S S . name name ( "name" Flex). {name} name Flex. @stag s-tag: @stag stag . #mtag m-tag: #mtag mtag . $ . : \a \b \f \n \r \t \v \\ \ooo \xhh \uhhhh \Uhhhhhhhh. (Actions) re2rust : !entry code code . (start conditions) . (lexeme). !pre_rule code code ( ) . ( ). !post_rule code code ( ) . (trap statements) . re2rust : !use:name ; use name . . : . * $ . use use . !include file ; include : file . (Program interface) (primitives) API . . . re2rust ( ) . API . re2rust API : Simple API ( 4.0 ) API --api simple re2c:api = simple . : YYINPUT ( ) YYCURSOR YYMARKER YYCTXMARKER YYLIMIT ( YYINPUT ). Record API ( 4.0 ) Record API (lexer) struct . --api record re2c:api = record . API yyrecord ( re2c:yyrecord ) struct yyinput yycursor yymarker yyctxmarker yylimit ( ). Generic API API backend Rust . API : YYPEEK YYSKIP YYBACKUP YYBACKUPCTX YYSTAGP YYSTAGN YYMTAGP YYMTAGN YYRESTORE YYRESTORECTX YYRESTORETAG YYSHIFT YYSHIFTSTAG YYSHIFTMTAG YYLESSTHAN YYEND. buffer: &[u8] cursor limit marker ctxmarker usize NONE Generic API : /*!re2c re2c:YYPEEK = "*buffer.get_unchecked(cursor)"; re2c:YYSKIP = "cursor += 1;"; re2c:YYBACKUP = "marker = cursor;"; re2c:YYRESTORE = "cursor = marker;"; re2c:YYBACKUPCTX = "ctxmarker = cursor;"; re2c:YYRESTORECTX = "cursor = ctxmarker;"; re2c:YYRESTORETAG = "cursor = @@{tag};"; re2c:YYLESSTHAN = "limit - cursor < @@{len}"; re2c:YYEND = "limit == cursor"; re2c:YYSTAGP = "@@{tag} = cursor;"; re2c:YYSTAGN = "@@{tag} = NONE;"; re2c:YYSHIFT = "cursor = (cursor as isize + @@{shift}) as usize;"; re2c:YYSHIFTSTAG = "@@{tag} = (@@{tag} as isize + @@{shift}) as usize;"; */ API . YYCTYPE ( ). ASCII EBCDIC UTF-8 . UTF-16 UCS-2 . UTF-32 . YYCURSOR l-value ( YYINPUT). YYCURSOR . YYCURSOR . YYLIMIT r-value ( YYINPUT). YYLIMIT . . YYCURSOR YYLIMIT . YYMARKER l-value ( YYINPUT). YYCURSOR (rollback) . . YYCTXMARKER l-value (trailing context) ( YYINPUT). . YYCTXMARKER / . YYFILL Generic API len. YYFILL len . re2c:eof len 1 YYFILL . re2c:eof YYFILL . len YYMAXFILL . YYFN --recursive-functions . re2c:fn:sep ( ) . ( void ). : ( ) ( - ). YYINPUT r-value ( ) . YYMAXFILL YYFILL. max . YYLESSTHAN Generic API len. r-value true len . YYEND Generic API . r-value true ( ). YYEND $ . YYLESSTHAN ..TP YYPEEK (primitive) API . r-value YYCTYPE . YYSKIP (primitive) API (code unit) . YYBACKUP (primitive) API ( YYRESTORE ). YYRESTORE (primitive) API YYBACKUP . YYBACKUPCTX (primitive) API (trailing context) ( YYRESTORECTX ). YYRESTORECTX (primitive) API YYBACKUPCTX . YYRESTORETAG (primitive) API tag tag . YYSTAGP (primitive) API tag tag YYINPUT ( ). YYSTAGP tag . YYSTAGN (primitive) API tag tag YYINPUT ( ). YYSTAGN tag . YYMTAGP (primitive) API tag. YYMTAGP tag ( ). YYMTAGN (primitive) API tag. YYMTAGN tag ( ). YYSHIFT (primitive) API shift shift ( ). YYCOPYSTAG (primitive) API lhs rhs s-tag (rhs) s-tag (lhs) . lhs rhs . YYCOPYMTAG (primitive) API lhs rhs m-tag (rhs) m-tag (lhs) . lhs rhs . YYSHIFTSTAG (primitive) API tag shift tag shift (code unit) ( ). YYSHIFTMTAG (primitive) API tag shift tag shift ( ). YYMAXNMATCH POSIX . maxnmatch . YYCONDTYPE (condition enum). conditions --header . YYGETACCEPT (primitive) var . var . YYSETACCEPT (primitive) : var ( l-value ) val ( ). var val . YYGETCOND r-value YYCONDTYPE . YYSETCOND (primitive) cond cond . YYGETSTATE r-value . -1 . YYSETSTATE (primitive) state state . YYDEBUG -d --debug-output . (logging) ( YYDEBUG ). YYDEBUG : state ( DFA -1) symbol ( ). yyaccept l-value . --storable-state . yybm ( --bitmaps ). . . k- n- (true) n- k- . if switch . yych l-value YYCTYPE . -f --storable-state . yyctable (dispatch) ( --conditions --computed-gotos ). yyfill l-value YYFILL ( YYFILL ). yynmatch l-value POSIX . -P --posix-captures . yypmatch l-value . yynmatch * 2 ( YYMAXNMATCH * 2 ). -P --posix-captures . yystable . yystate l-value --loop-switch DFA . yytarget ( ) . . yytarget --computed-gotos . re2c . re2c . re2c . -? --help -h . --api API . simple (lexer) . record (struct class ) . generic API . --bit-vectors -b . --nested-ifs . --captures, --leftmost-captures (submatch) . yybmatch 2 * YYMAXNMATCH yynmatch . --captvars, --leftmost-captvars . k- yytl yytr . --case-insensitive . --case-inverted : . --case-ranges case switch low ... high. C/C++ GCC Clang Tcc . case Tcc . --computed-gotos -g "computed goto" ( ). re2rust . cgoto:threshold . cgoto:relative . --bit-vectors . --computed-gotos-relative --computed-gotos ( ). --computed-gotos . --conditions --start-conditions -c "" Flex: . re2rust goto . --depfile FILE FILE Makefile : [include-file ...]. / include include . --output . --ebcdic --ecb -e EBCDIC . re2rust 0 -- 0xFF . --empty-class re2rust . match-empty () ( ). match-none . error . --encoding-policy re2rust (surrogates). fail re2rust . substitute re2rust 0xFFFD . ignore () re2rust . . --flex-syntax -F Flex: . . --goto-label "goto/label": DFA goto . goto . --header --type-header -t HEADER HEADER. header:on header:off . (enum) ( conditions ). -I PATH PATH include . include . re2rust FILE include -I . --input --api. default simple ( ). custom generic . --input-encoding re2rust. ascii () re2rust ASCII : . utf8 re2rust UTF8 . --invert-captures . (...) (! ...) . (! ...) (...) . --lang . C D Go Haskell Java JS OCaml Python Rust Swift V Zig ( --syntax ). none . --location-format . gnu 'filename:line:column: ...' . msvc 'filename(line,column) ...' . gnu . --loop-switch "loop/switch": DFA switch case switch . yystate . yystate case . --nested-ifs -s if switch . . --no-debug-info -i . ( ). --no-generation-date . --no-version . --no-unsafe unsafe YYPEEK ( Rust ). YYPEEK . YYPEEK (release) --no-unsafe unsafe . --output -o OUTPUT OUTPUT. --posix-captures, -P POSIX. yybmatch 2 * YYMAXNMATCH yynmatch ..TP --posix-captvars POSIX. yytl yytr k- . --recursive-functions DFA . --reusable -r 2.2 ( ). --skeleton -S "" . "" . . C . --storable-state -f . push . YYGETSTATE YYSETSTATE yych yyaccept state . --syntax FILE FILE (syntax) . FILE ( ) ( --lang none ). --tags -T (tags). --ucs2 --wide-chars -w UCS2 . re2rust 0 -- 0xFFFF . --nested-ifs . --utf8 --utf-8 -8 UTF-8 . re2rust 0 -- 0x10FFFF . --utf16 --utf-16 -x UTF16 . re2rust 0 -- 0x10FFFF . --nested-ifs . --utf32 --unicode -u UTF32 . re2rust 0 -- 0x10FFFF . --nested-ifs . --verbose . --vernum -V MMmmpp ( ). --version -v . --single-pass -1 . ( ). --debug-output -d YYDEBUG . . --dump-adfa : DFA ( .dot). --dump-cfg : ( .dot). --dump-closure-stats : (closure). --dump-dfa-det : DFA ( .dot). --dump-dfa-min : DFA ( .dot). --dump-dfa-tagopt : DFA ( .dot). --dump-dfa-tree : DFA ( .dot). --dump-dfa-raw : DFA ( .dot). --dump-interf : . --dump-nfa : NFA ( .dot). --emit-dot -D .dot . Graphviz ( dot -Tpng -odfa.png dfa.dot). --dfa-minimization : DFA re2rust . moore () . table " " . DFA . --eager-skip : -- . -- -- . --no-lookahead . TDFA(0) . TDFA(1) TDFA(0) . . TDFA(0) TDFA(1) . --no-optimize-tags : ( ). --posix-closure : POSIX: gor1 () - (Goldberg-Radzik) gtop " " . --posix-prectable : POSIX. complex TNFA . naive TNFA complex . --stadfa . staDFA TDFA . staDFA TDFA . --fixed-tags : (all) (none) (toplevel). all . "" ( "") . . ( ). . . -W . -Werror . . -W warning. -Wno- warning. -Werror- warning ( -W ). -Wno-error- warning . . -Wcondition-order . --header conditions . -Wempty-character-class . : . re2rust . --empty-class . -Wmatch-empty-string ( ). . -Wswapped-range . . -Wundefined-control-flow ( ). . * . -Wunreachable-rules . -Wdeprecated-eof_rule $ . $ . -Wuseless-escape . re2rust . -Wnondeterministic-tags n- n . -Wsentinel-in-midrule --- . . re2c:sentinel . -Wundefined-syntax-config --syntax . : . . re2c (syntax files) . -- . re2c re2c: : (lexer) . ( include/syntax ) re2rust . ( --lang none ). : . (Feature lists) re2rust : supported_apis API simple record generic. supported_api_styles API functions free-form. supported_code_models goto-label loop-switch recursive-functions. supported_targets (codegen) code dot skeleton. supported_features nested-ifs bitmaps computed-gotos case-ranges monadic unsafe tags captures captvars. (Language configurations) re2rust : semicolons (semicolon) . backtick_quoted_strings (backtick) . single_quoted_strings (single-quote) . indentation_sensitive () . wrap_blocks_in_braces . (Inplace configurations) re2c: . . (Code templates) . - : code-template :: name '=' code-exprs ';' | CODE_TEMPLATE ';' | '' ';' code-exprs :: | code-exprs code-expr code-expr :: STRING | VARIABLE | optional | list optional :: '(' CONDITIONAL '?' code-exprs ')' | '(' CONDITIONAL '?' code-exprs ':' code-exprs ')' list :: '[' VARIABLE ':' code-exprs ']' | '[' VARIABLE '{' NUMBER '}' ':' code-exprs ']' | '[' VARIABLE '{' NUMBER ',' NUMBER '}' ':' code-exprs ']' (string literals) . (placeholders) . : re2rust . ( -1 ). . ( ) ( ) . re2rust . if-then-else C : code:if_then_else = [branch{0}: topindent "if " cond " {" nl indent [stmt: stmt] dedent] [branch{1:-1}: topindent "} else" (.cond ? " if " cond) " {" nl indent [stmt: stmt] dedent] topindent "}" nl; branch : branch{0} ( else ) branch{1:-1} ( ) . stmt : [stmt: stmt] . topindent indent dedent nl .cond ( ). : if x { // do something } else if y { // do something else } else { // don't do anything } re2rust . . . code:var_local . : type ( ) name ( ) init ( ). : .init ( ). code:var_global code:var_local (top-level) . code:const_local . : type ( ) name ( ) init ( ). code:const_global code:const_local (top-level) . code:array_local () . : type ( ) name ( ) size ( ) row ( ) elem ( -- row ). : .const ( ). code:array_global code:array_local (top-level) . code:array_elem (). : array ( ) index ( ). code:enum ( ). type ( ) elem ( ) init ( ). : .init ( ). code:enum_elem ( code:enum). name ( ) type ( ). code:assign . lhs ( ) rhs ( ). code:type_int . code:type_uint . code:type_yybm yybm. code:type_yytarget yytarget. code:type_yyctable yyctable. code:cmp_eq " ". code:cmp_ne " ". code:cmp_lt " ". code:cmp_gt " ". code:cmp_le " ". code:cmp_ge " ". code:if_then_else if-then-else . : branch ( ) cond ( ) stmt ( ). : .cond ( ) .many ( ). code:if_then_else_oneline code:if_then_else . code:if_then_else . code:switch switch (case). : expr ( switch) case ( case ). code:switch_cases (cases) switch . case ( ) stmt ( ). code:switch_cases_oneline code:switch_cases . code:switch_cases . code:switch_case_range switch ( ). : val ( ). : .many ( ) .char_literals ( switch -- ). code:switch_case_default (default) switch. code:loop ( ). : label ( ) stmt ( ). code:continue continue. : label ( ). code:goto goto. : label ( ). code:cgoto goto (computed). : array ( goto ) index ( ) base ( .cgoto.relative ). code:cgoto:data goto . : label ( ) type ( ) base ( - .cgoto.relative ). code:fndecl . : name ( ) type ( ) throw ( re2c:yyfn:throw ) arg ( ) argname ( ) argtype ( ). : .type ( non-void ). code:fndef code:fndecl stmt . code:fncall . : name ( ) retval ( l-value ) arg ( ). : .args ( ) .retval ( ). code:tailcall (tail call). : name ( ) arg ( ). : .args ( ) .retval ( non-void ). code:recursive_functions --recursive-functions. : fn ( ) fndecl ( ) fndef ( )..TP code:fingerprint . : ver ( re2rust ) date ( ). code:line_info ( ). : line ( ) file ( ). code:abort (abort) . code:yydebug YYDEBUG API . : YYDEBUG yyrecord yych ( re2c:) state ( DFA). code:yypeek YYPEEK API . : YYPEEK YYCTYPE YYINPUT YYCURSOR yyrecord yych ( re2c:). : .cast ( re2c:yych:conversion ). code:yyskip YYSKIP API . : YYSKIP YYCURSOR yyrecord ( re2c:). code:yybackup YYBACKUP API . : YYBACKUP YYCURSOR YYMARKER yyrecord ( re2c:). code:yybackupctx YYBACKUPCTX API . : YYBACKUPCTX YYCURSOR YYCTXMARKER yyrecord ( re2c:). code:yyskip_yypeek code:yyskip code:yypeek ( code:yyskip code:yypeek ). code:yypeek_yyskip code:yypeek code:yyskip ( code:yypeek code:yyskip ). code:yyskip_yybackup code:yyskip code:yybackup ( code:yyskip code:yybackup ). code:yybackup_yyskip code:yybackup code:yyskip ( code:yybackup code:yyskip ). code:yybackup_yypeek code:yybackup code:yypeek ( code:yybackup code:yypeek ). code:yyskip_yybackup_yypeek code:yyskip code:yybackup code:yypeek ( ``code:yyskip`` code:yybackup code:yypeek ). code:yybackup_yypeek_yyskip code:yybackup code:yypeek code:yyskip ( ``code:yybackup`` code:yypeek code:yyskip ). code:yyrestore YYRESTORE API . : YYRESTORE YYCURSOR YYMARKER yyrecord ( re2c:). code:yyrestorectx YYRESTORECTX API . : YYRESTORECTX YYCURSOR YYCTXMARKER yyrecord ( re2c:). code:yyrestoretag YYRESTORETAG API . : YYRESTORETAG YYCURSOR yyrecord ( re2c:) tag ( ). code:yyshift YYSHIFT API . : YYSHIFT YYCURSOR yyrecord ( re2c:) offset ( ). code:yyshiftstag YYSHIFTSTAG API . : YYSHIFTSTAG yyrecord negative ( re2c:) tag ( ) offset ( ). : .nested ( -- re2c:tags:negative ). code:yyshiftmtag YYSHIFTMTAG API . : YYSHIFTMTAG ( re2c:) tag ( ) offset ( ). code:yystagp YYSTAGP API . : YYSTAGP YYCURSOR yyrecord ( re2c:) tag ( ). code:yymtagp YYMTAGP API . : YYMTAGP ( re2c:) tag ( ). code:yystagn YYSTAGN API . : YYSTAGN negative yyrecord ( re2c:) tag ( ). code:yymtagn YYMTAGN API . : YYMTAGN ( re2c:) tag ( ). code:yycopystag YYCOPYSTAG API . : YYCOPYSTAG yyrecord ( re2c:) lhs rhs ( ). code:yycopymtag YYCOPYMTAG API . : YYCOPYMTAG yyrecord ( re2c:) lhs rhs ( ). code:yygetaccept YYGETACCEPT API . : YYGETACCEPT yyrecord ( re2c:) var ( re2c:yyaccept). code:yysetaccept YYSETACCEPT API . : YYSETACCEPT yyrecord ( re2c:) var ( re2c:yyaccept) val ( ). code:yygetcond YYGETCOND API . : YYGETCOND yyrecord ( re2c:) var ( re2c:yycond). code:yysetcond YYSETCOND API . : YYSETCOND yyrecord ( re2c:) var ( re2c:yycond) val ( ). code:yygetstate YYGETSTATE API . : YYGETSTATE yyrecord ( re2c:) var ( re2c:yystate). code:yysetstate YYSETSTATE API . : YYSETSTATE yyrecord ( re2c:) var ( re2c:yystate) val ( ). code:yylessthan YYLESSTHAN API . : YYLESSTHAN YYCURSOR YYLIMIT yyrecord ( re2c:) need ( ). : .many ( need )..TP code:yyend YYEND API . : YYEND YYCURSOR YYLIMIT. code:yybm_filter yych yybm ( --bitmaps ). : yych ( re2c:yych ). code:yybm_match yybm ( --bitmaps). : yybm yych ( re2c: ) offset ( yybm yych ) mask ( ) code:yytarget_filter yych yytarget ( --computed-gotos ). : yych ( re2c:yych ). : nl . indent . dedent . topindent . . : .api.simple API (--api simple re2c:api = simple). .api.generic API (--api generic re2c:api = generic). .api.record API (--api record re2c:api = record). .api_style.functions API (re2c:api-style = functions). .api_style.freeform API (re2c:api-style = free-form). .case_ranges case (--case-ranges re2c:case-ranges = 1). .cgoto.relative goto (--computed-gotos-relative re2c:cgoto:relative = 1). .code_model.goto_label goto/label (--goto-label). .code_model.loop_switch loop/switch (--loop-switch). .code_model.recursive_functions (--recursive-function). .date . .loop_label re2rust (re2c:label:yyloop ). .monadic (re2c:monadic = 1). . .start_conditions (--start-conditions). .storable_state (--storable-state). .unsafe re2rust "unsafe" (--unsafe re2c:unsafe = 1). "unsafe" . .version re2rust . .yyfill.enable YYFILL (re2c:yyfill:enable = 1). .yyfn.throw re2c:yyfn:throw . (lexer) . : o ( *) o o "" : . : . . re2rust . . : o (Sentinel). . "" . . : . . YYFILL re2c:yyfill:enable = 0; re2c:eof -1 . o (Sentinel with bounds checks). : . . "" "" . ( ). . . YYFILL ( ). . . re2c:eof ( ). YYFILL . o (Bounds checks with padding). " " . DFA (SCC) SCC SCC . (lexeme) . (padding) ( ) . YYMAXFILL max . YYFILL . YYFILL re2c:eof -1 ( ). o (Custom checks). (Generic API) . . --api custom re2c:api = custom; re2c:yyfill:enable = 0; re2c:yyfill:check = 0; . ..SS (Sentinel) (sentinel) . (null-terminated) . null : (lexeme) ( ). null * (crash) ( -Wsentinel-in-midrule re2c:sentinel ). re2c:yyfill:enable = 0; YYFILL . // re2rust $INPUT -o $OUTPUT --api simple fn lex(yyinput: &[u8]) -> isize { // The input must be null-terminated, otherwise the function has UB. assert_eq!(yyinput.last(), Some(&0)); let mut yycursor = 0; let mut count = 0; 'lex: loop { /*!re2c re2c:YYCTYPE = u8; re2c:yyfill:enable = 0; * { return -1; } [\x00] { return count; } [a-z]+ { count += 1; continue 'lex; } [ ]+ { continue 'lex; } */} } fn main() { assert_eq!(lex(b"\x00"), 0); assert_eq!(lex(b"one two three\x00"), 3); assert_eq!(lex(b"f0ur\x00"), -1); } (Sentinel with bounds checks) ( 1.2 ). . null re2c:eof = 0; . sentinel null ( 'aaa\0aa'\0 'aaa\0 ). null . YYLIMIT <= YYCURSOR API YYLESSTHAN(1) . (YYFILL re2c:yyfill:enable = 0; YYFILL YYFILL with sentinel ). : $ ( *) -Wundefined-control-flow . // re2rust $INPUT -o $OUTPUT --api simple fn lex(yyinput: &[u8]) -> isize { // The input must be null-terminated, otherwise the function has UB. assert_eq!(yyinput.last(), Some(&0)); let (mut yycursor, mut yymarker) = (0, 0); let yylimit = yyinput.len() - 1; // null-terminator not included let mut count = 0; 'lex: loop { /*!re2c re2c:YYCTYPE = u8; re2c:yyfill:enable = 0; re2c:eof = 0; str = ['] ([^'\\] | [\\][^])* [']; * { return -1; } $ { return count; } str { count += 1; continue 'lex; } [ ]+ { continue 'lex; } */} } fn main() { assert_eq!(lex(b"\0"), 0); assert_eq!(lex(b"'qu\0tes' 'are' 'fine: \\'' \0"), 3); assert_eq!(lex(b"'unterminated\\'\0"), -1); } ( ). . YYMAXFILL null YYMAXFILL max . null --- (lexeme) ( ). "stop" (null) (lexer) ( null ). . (YYLIMIT - YYCURSOR) < n YYLESSTHAN(n) API n ( n ). YYFILL(n) n . YYFILL ( ). YYFILL YYFILL with padding . // re2rust $INPUT -o $OUTPUT --api simple /*!max:re2c*/ fn lex(s: &[u8]) -> isize { let mut count = 0; let mut yycursor = 0; let yylimit = s.len() + YYMAXFILL; // Copy string to a buffer and add YYMAXFILL zero padding. let mut yyinput = Vec::with_capacity(yylimit); yyinput.extend_from_slice(s); yyinput.extend([0 as u8; YYMAXFILL]); 'lex: loop { /*!re2c re2c:YYCTYPE = u8; re2c:YYFILL = "return -1;"; str = ['] ([^'\\] | [\\][^])* [']; [\x00] { // Check that it is the sentinel, not some unexpected null. return if yycursor == s.len() + 1 { count } else { -1 } } str { count += 1; continue 'lex; } [ ]+ { continue 'lex; } * { return -1; } */} } fn main() { assert_eq!(lex(b""), 0); assert_eq!(lex(b"'qu\0tes' 'are' 'fine: \\'' "), 3); assert_eq!(lex(b"'unterminated\\'"), -1); assert_eq!(lex(b"'unexpected \0 null"), -1); } (Custom checks) generic API . . sentinel null . YYPEEK . . YYPEEK . // re2rust $INPUT -o $OUTPUT // Expect a string without terminating null. fn lex(s: &[u8]) -> isize { let mut count = 0; let mut cur = 0; let lim = s.len(); 'lex: loop {/*!re2c re2c:YYCTYPE = u8; re2c:YYPEEK = "if cur < lim {*s.get_unchecked(cur)} else {0}"; re2c:YYSKIP = "cur += 1;"; re2c:yyfill:enable = 0; * { return -1; } [\x00] { return count; } [a-z]+ { count += 1; continue 'lex; } [ ]+ { continue 'lex; } */} } fn main() { assert_eq!(lex(b""), 0); assert_eq!(lex(b"one two three "), 3); assert_eq!(lex(b"f0ur"), -1); } : ( ) . (chunks) . . (lexer) : o (cursor): (YYCURSOR C pointer API YYSKIP/YYPEEK generic API) o (limit): (YYLIMIT C pointer API YYLESSTHAN generic API ) o (marker): (YYMARKER API YYBACKUP/YYRESTORE generic API) o (token): (lexeme) ( re2rust API ) o (context marker): (YYCTXMARKER C pointer API YYBACKUPCTX/YYRESTORECTX generic API) o (tag variables): ( stags mtags generic API YYSTAGP/YYSTAGN/YYMTAGP/YYMTAGN) YYFILL . token cursor token token limit . YYFILL ( ). YYFILL EOF : EOF (padding) . -f --storable-state YYFILL ( ). YYFILL (YYFILL with sentinel) EOF YYFILL . YYFILL YYLIMIT <= YYCURSOR C pointer API YYLESSTHAN() generic API . YYFILL . YYFILL . (limit) limit (sentinel) re2c:eof . YYFILL ( # ). <-- shift --> >-A------------B---------C-------------D#-----------E-> buffer token marker limit, cursor >-A------------B---------C-------------D------------E#-> buffer, marker cursor limit token <-- shift --> >-A------------B---------C-------------D#--E (EOF) buffer token marker limit, cursor >-A------------B---------C-------------D---E#........ buffer, marker cursor limit token input.txt EOF . // re2rust $INPUT -o $OUTPUT use std::fs::File; use std::io::{Read, Write}; const BUFSIZE: usize = 4096; struct State { file: File, yyinput: [u8; BUFSIZE], yylimit: usize, yycursor: usize, yymarker: usize, token: usize, eof: bool, } #[derive(PartialEq)] enum Fill { Ok, Eof, LongLexeme } fn fill(st: &mut State) -> Fill { if st.eof { return Fill::Eof; } // Error: lexeme too long. In real life could reallocate a larger buffer. if st.token < 1 { return Fill::LongLexeme; } // Shift buffer contents (discard everything up to the current token). st.yyinput.copy_within(st.token..st.yylimit, 0); st.yylimit -= st.token; st.yycursor -= st.token; st.yymarker = st.yymarker.overflowing_sub(st.token).0; // may underflow if marker is unused st.token = 0; // Fill free space at the end of buffer with new data from file. match st.file.read(&mut st.yyinput[st.yylimit..BUFSIZE - 1]) { // -1 for sentinel Ok(n) => { st.yylimit += n; st.eof = n == 0; // end of file st.yyinput[st.yylimit] = 0; // append sentinel } Err(why) => panic!("cannot read from file: {}", why) } return Fill::Ok; } fn lex(yyrecord: &mut State) -> isize { let mut count: isize = 0; 'lex: loop { yyrecord.token = yyrecord.yycursor; /*!re2c re2c:api = record; re2c:YYCTYPE = u8; re2c:YYFILL = "fill(yyrecord) == Fill::Ok"; re2c:eof = 0; str = ['] ([^'\\] | [\\][^])* [']; * { return -1; } $ { return count; } str { count += 1; continue 'lex; } [ ]+ { continue 'lex; } */} } fn main() { let fname = "input"; let content = b"'qu\0tes' 'are' 'fine: \\'' "; // Prepare input file: a few times the size of the buffer, containing // strings with zeroes and escaped quotes. match File::create(fname) { Err(why) => panic!("cannot open {}: {}", fname, why), Ok(mut file) => match file.write_all(&content.repeat(BUFSIZE)) { Err(why) => panic!("cannot write to {}: {}", fname, why), Ok(_) => {} } }; let count = 3 * BUFSIZE; // number of quoted strings written to file // Reopen input file for reading. let file = match File::open(fname) { Err(why) => panic!("cannot read file {}: {}", fname, why), Ok(file) => file, }; // Initialize lexer state: all offsets are at the end of buffer. let yylimit = BUFSIZE - 1; let mut st = State { file: file, // Sentinel (at `yylimit` offset) is set to null, which triggers YYFILL. yyinput: [0; BUFSIZE], yylimit: yylimit, yycursor: yylimit, yymarker: yylimit, token: yylimit, eof: false, }; // Run the lexer. assert_eq!(lex(&mut st), count as isize); // Cleanup: remove input file. match std::fs::remove_file(fname) { Err(why) => panic!("cannot remove {}: {}", fname, why), Ok(_) => {} } } ( EOF ) YYFILL (primitive) . YYFILL (YYLIMIT - YYCURSOR) < n API C YYLESSTHAN(n) API . YYFILL . YYFILL ( ). YYFILL (limit) YYMAXFILL ( YYFILL n ). YYFILL ( YYMAXFILL # ). <-- shift --> <-- need --> >-A------------B---------C-----D-------E---F--------G-> buffer token marker cursor limit >-A------------B---------C-----D-------E---F--------G-> buffer, marker cursor limit token <-- shift --> <-- need --> >-A------------B---------C-----D-------E-F (EOF) buffer token marker cursor limit >-A------------B---------C-----D-------E-F############### buffer, marker cursor limit token <- YYMAXFILL -> input.txt . // re2rust $INPUT -o $OUTPUT use std::fs::File; use std::io::{Read, Write}; /*!max:re2c*/ const BUFSIZE: usize = 4096; struct State { file: File, yyinput: [u8; BUFSIZE], yylimit: usize, yycursor: usize, yymarker: usize, token: usize, eof: bool, } #[derive(PartialEq)] enum Fill { Ok, Eof, LongLexeme } fn fill(st: &mut State, need: usize) -> Fill { if st.eof { return Fill::Eof; } // Error: lexeme too long. In real life can reallocate a larger buffer. if st.token < need { return Fill::LongLexeme; } // Shift buffer contents (discard everything up to the current token). st.yyinput.copy_within(st.token..st.yylimit, 0); st.yylimit -= st.token; st.yycursor -= st.token; st.yymarker = st.yymarker.overflowing_sub(st.token).0; // underflows if marker is unused st.token = 0; // Fill free space at the end of buffer with new data from file. let n = match st.file.read(&mut st.yyinput[st.yylimit..BUFSIZE - YYMAXFILL]) { Ok(n) => n, Err(why) => panic!("cannot read from file: {}", why) }; st.yylimit += n; // If read zero characters, this is end of input => add zero padding // so that the lexer can access characters at the end of buffer. if n == 0 { st.eof = true; for i in 0..YYMAXFILL { st.yyinput[st.yylimit + i] = 0; } st.yylimit += YYMAXFILL; } return Fill::Ok; } fn lex(yyrecord: &mut State) -> isize { let mut count: isize = 0; 'lex: loop { yyrecord.token = yyrecord.yycursor; /*!re2c re2c:api = record; re2c:YYCTYPE = u8; re2c:YYFILL = "if fill(yyrecord, @@) != Fill::Ok { return -1; }"; str = ['] ([^'\\] | [\\][^])* [']; [\x00] { // Check that it is the sentinel, not some unexpected null. return if yyrecord.token == yyrecord.yylimit - YYMAXFILL { count } else { -1 } } str { count += 1; continue 'lex; } [ ]+ { continue 'lex; } * { return -1; } */} } fn main() { let fname = "input"; let content = b"'qu\0tes' 'are' 'fine: \\'' "; // Prepare input file: a few times the size of the buffer, containing // strings with zeroes and escaped quotes. match File::create(fname) { Err(why) => panic!("cannot open {}: {}", fname, why), Ok(mut file) => match file.write_all(&content.repeat(BUFSIZE)) { Err(why) => panic!("cannot write to {}: {}", fname, why), Ok(_) => {} } }; let count = 3 * BUFSIZE; // number of quoted strings written to file // Reopen input file for reading. let file = match File::open(fname) { Err(why) => panic!("cannot read file {}: {}", fname, why), Ok(file) => file, }; // Initialize lexer state: all offsets are at the end of buffer. // This immediately triggers YYFILL, as the YYLESSTHAN condition is true. let yylimit = BUFSIZE - YYMAXFILL; let mut st = State { file: file, yyinput: [0; BUFSIZE], yylimit: yylimit, yycursor: yylimit, yymarker: yylimit, token: yylimit, eof: false, }; // Run the lexer. assert_eq!(lex(&mut st), count as isize); // Cleanup: remove input file. match std::fs::remove_file(fname) { Err(why) => panic!("cannot remove {}: {}", fname, why), Ok(_) => {} } } (Multiple blocks) (lexer) ( ). re2rust . start conditions . . goto ( C/C++ Go) (label) . goto . goto ( Rust) switch /switch yystate re2rust . (binary) (octal) (decimal) (hexadecimal) . . . . // re2rust $INPUT -o $OUTPUT // Store u32 number in u64 during parsing to simplify overflow handling. struct State<'a> { yyinput: &'a [u8], yycursor: usize, yymarker: usize, num: u64, } /*!re2c // Common re2c definitions shared between all functions. re2c:api = record; re2c:yyrecord = st; re2c:yyfill:enable = 0; re2c:YYCTYPE = u8; */ const ERROR: u64 = std::u32::MAX as u64 + 1; // overflow macro_rules! maybe { // Convert the number from u64 to optional u32. ($n:expr) => { if $n < ERROR { Some($n as u32) } else { None } } } // Add digit with the given base, checking for overflow. fn add(st: &mut State, offs: u8, base: u64) { let digit = unsafe { st.yyinput.get_unchecked(st.yycursor - 1) } - offs; st.num = std::cmp::min(st.num * base + digit as u64, ERROR); } fn parse_u32(s: & [u8]) -> Option { assert_eq!(s.last(), Some(&0)); // expect null-terminated input let mut st = State {yyinput: s, yycursor: 0, yymarker: 0, num: 0}; /*!re2c '0b' / [01] { return parse_bin(&mut st); } "0" { return parse_oct(&mut st); } "" / [1-9] { return parse_dec(&mut st); } '0x' / [0-9a-fA-F] { return parse_hex(&mut st); } * { return None; } */ } fn parse_bin(st: &mut State) -> Option { 'bin: loop {/*!re2c [01] { add(st, 48, 2); continue 'bin; } * { return maybe!(st.num); } */} } fn parse_oct(st: &mut State) -> Option { 'oct: loop {/*!re2c [0-7] { add(st, 48, 8); continue 'oct; } * { return maybe!(st.num); } */} } fn parse_dec(st: &mut State) -> Option { 'dec: loop {/*!re2c [0-9] { add(st, 48, 10); continue 'dec; } * { return maybe!(st.num); } */} } fn parse_hex(st: &mut State) -> Option { 'hex: loop {/*!re2c [0-9] { add(st, 48, 16); continue 'hex; } [a-f] { add(st, 87, 16); continue 'hex; } [A-F] { add(st, 55, 16); continue 'hex; } * { return maybe!(st.num); } */} } fn main() { assert_eq!(parse_u32(b"\0"), None); assert_eq!(parse_u32(b"1234567890\0"), Some(1234567890)); assert_eq!(parse_u32(b"0b1101\0"), Some(13)); assert_eq!(parse_u32(b"0x7Fe\0"), Some(2046)); assert_eq!(parse_u32(b"0644\0"), Some(420)); assert_eq!(parse_u32(b"9999999999\0"), None); } --start-conditions . () re2rust . re2rust . : YYGETCOND YYSETCOND . conditions --header re2c:header . re2c:cond:enumprefix . <*> . : < condition-list > regular-expression code condition-list . regular-expression code . < condition-list > regular-expression => condition code condition-list . regular-expression condition code . < condition-list > regular-expression :=> condition condition-list . regular-expression condition ( ). < condition-list > !action code code action condition-list ( actions ). code code condition-list . !pre_rule ( ). < > code "0" code . !entry ( (condition dispatch) ). < > => condition code condition . < > :=> condition condition . re2rust re2rust goto/label loop/switch . goto ( C/C++ Go) yyc_ cond ( re2c:cond:prefix ). goto . re2rust switch YYGETSTATE . :=> switch ( re2c:cond:goto ). . goto ( Rust) re2rust yystate . DFA . switch . ( YYGETCOND/YYSETCOND ). () () () () . . cond conditions . // re2rust $INPUT -o $OUTPUT -c --api simple /*!conditions:re2c*/ const ERROR: u64 = std::u32::MAX as u64 + 1; // overflow // Add digit with the given base, checking for overflow. fn add(num: &mut u64, str: &[u8], cur: usize, offs: u8, base: u64) { let digit = unsafe { str.get_unchecked(cur - 1) } - offs; *num = std::cmp::min(*num * base + digit as u64, ERROR); } fn parse_u32(yyinput: &[u8]) -> Option { assert_eq!(yyinput.last(), Some(&0)); // expect null-terminated input let (mut yycursor, mut yymarker) = (0, 0); let mut yycond = YYC_INIT; let mut num = 0u64; // Store number in u64 to simplify overflow checks. 'lex: loop { /*!re2c re2c:YYCTYPE = u8; re2c:yyfill:enable = 0; '0b' / [01] :=> BIN "0" :=> OCT "" / [1-9] :=> DEC '0x' / [0-9a-fA-F] :=> HEX * { return None; } [01] { add(&mut num, yyinput, yycursor, 48, 2); continue 'lex; } [0-7] { add(&mut num, yyinput, yycursor, 48, 8); continue 'lex; } [0-9] { add(&mut num, yyinput, yycursor, 48, 10); continue 'lex; } [0-9] { add(&mut num, yyinput, yycursor, 48, 16); continue 'lex; } [a-f] { add(&mut num, yyinput, yycursor, 87, 16); continue 'lex; } [A-F] { add(&mut num, yyinput, yycursor, 55, 16); continue 'lex; } * { return if num < ERROR { Some(num as u32) } else { None }; } */} } fn main() { assert_eq!(parse_u32(b"\0"), None); assert_eq!(parse_u32(b"1234567890\0"), Some(1234567890)); assert_eq!(parse_u32(b"0b1101\0"), Some(13)); assert_eq!(parse_u32(b"0x7Fe\0"), Some(2046)); assert_eq!(parse_u32(b"0644\0"), Some(420)); assert_eq!(parse_u32(b"9999999999\0"), None); } --storable-state re2rust . re2rust <<>> ("pull") . ( ). : <<>> ("push") . . . <<>> ("pull") : o YYSETSTATE() YYGETSTATE(state). o yych yyaccept ( ) state . state -1 . o YYFILL . . o . o getstate YYGETSTATE . goto ( --loop-switch). o re2c:eof sentinel with bounds checks . (padding) : (lexeme) . ( ). <<>> ("push") . . <<>> . . // re2rust $INPUT -o $OUTPUT -f use std::fs::File; use std::io::{Read, Write}; const DEBUG: bool = false; macro_rules! log { ($($fmt:expr)? $(, $args:expr)*) => { if DEBUG { println!($($fmt)? $(, $args)*) } } } // Use a small buffer to cover the case when a lexeme doesn't fit. // In real world use a larger buffer. const BUFSIZE: usize = 10; struct State { file: File, yyinput: [u8; BUFSIZE], yylimit: usize, yycursor: usize, yymarker: usize, token: usize, yystate: isize, } #[derive(Debug, PartialEq)] enum Status {End, Ready, Waiting, BadPacket, BigPacket} fn fill(st: &mut State) -> Status { // Error: lexeme too long. In real life can reallocate a larger buffer. if st.token < 1 { return Status::BigPacket; } // Shift buffer contents (discard everything up to the current lexeme). st.yyinput.copy_within(st.token..st.yylimit, 0); st.yylimit -= st.token; st.yycursor -= st.token; st.yymarker = st.yymarker.overflowing_sub(st.token).0; // underflows if marker is unused st.token = 0; // Fill free space at the end of buffer with new data. match st.file.read(&mut st.yyinput[st.yylimit..BUFSIZE - 1]) { // -1 for sentinel Ok(n) => { st.yylimit += n; st.yyinput[st.yylimit] = 0; // append sentinel symbol }, Err(why) => panic!("cannot read from file: {}", why) } return Status::Ready; } fn lex(yyrecord: &mut State, recv: &mut usize) -> Status { let mut yych; 'lex: loop { yyrecord.token = yyrecord.yycursor; /*!re2c re2c:api = record; re2c:eof = 0; re2c:YYCTYPE = "u8"; re2c:YYFILL = "return Status::Waiting;"; packet = [a-z]+[;]; * { return Status::BadPacket; } $ { return Status::End; } packet { *recv += 1; continue 'lex; } */} } fn test(packets: Vec<&[u8]>, expect: Status) { // Create a pipe (open the same file for reading and writing). let fname = "pipe"; let mut fw: File = match File::create(fname) { Err(why) => panic!("cannot open {}: {}", fname, why), Ok(file) => file, }; let fr: File = match File::open(fname) { Err(why) => panic!("cannot read file {}: {}", fname, why), Ok(file) => file, }; // Initialize lexer state: `state` value is -1, all offsets are at the end // of buffer, the character at `yylimit` offset is the sentinel (null). let yylimit = BUFSIZE - 1; let mut state = State { file: fr, // Sentinel (at `yylimit` offset) is set to null, which triggers YYFILL. yyinput: [0; BUFSIZE], yylimit: yylimit, yycursor: yylimit, yymarker: yylimit, token: yylimit, yystate: -1, }; // Main loop. The buffer contains incomplete data which appears packet by // packet. When the lexer needs more input it saves its internal state and // returns to the caller which should provide more input and resume lexing. let mut status; let mut send = 0; let mut recv = 0; loop { status = lex(&mut state, &mut recv); if status == Status::End { log!("done: got {} packets", recv); break; } else if status == Status::Waiting { log!("waiting..."); if send < packets.len() { log!("sent packet {}", send); match fw.write_all(packets[send]) { Err(why) => panic!("cannot write to {}: {}", fname, why), Ok(_) => send += 1, } } status = fill(&mut state); log!("queue: '{}'", String::from_utf8_lossy(&state.yyinput)); if status == Status::BigPacket { log!("error: packet too big"); break; } assert_eq!(status, Status::Ready); } else { assert_eq!(status, Status::BadPacket); log!("error: ill-formed packet"); break; } } // Check results. assert_eq!(status, expect); if status == Status::End { assert_eq!(recv, send); } // Cleanup: remove input file. match std::fs::remove_file(fname) { Err(why) => panic!("cannot remove {}: {}", fname, why), Ok(_) => {} } } fn main() { test(vec![], Status::End); test(vec![b"zero;", b"one;", b"two;", b"three;", b"four;"], Status::End); test(vec![b"zer0;"], Status::BadPacket); test(vec![b"goooooooooogle;"], Status::BigPacket); } /*!rules:re2c[:] ... */ %{rules[:] ... %} re2rust . . rules use . rules . use /*!use:re2c[:] ... */ %{use[:] ... %} . : rules ( ). use . use (lexer) : rules ( ). !use:; re2rust . . : . * $ ( ) . use re2rust ( ). use 2.2 re2rust . ( ). 2.2 -r --reusable . 1.2 . !use // re2rust $INPUT -o $OUTPUT --api simple // This example shows how to combine reusable re2c blocks: two blocks // ('colors' and 'fish') are merged into one. The 'salmon' rule occurs // in both blocks; the 'fish' block takes priority because it is used // earlier. Default rule * occurs in all three blocks; the local (not // inherited) definition takes priority. #[derive(Debug, PartialEq)] enum Ans { Color, Fish, Dunno } /*!rules:re2c:colors * { panic!("ah"); } "red" | "salmon" | "magenta" { return Ans::Color; } */ /*!rules:re2c:fish * { panic!("oh"); } "haddock" | "salmon" | "eel" { return Ans::Fish; } */ fn lex(yyinput: &[u8]) -> Ans { assert!(yyinput.len() > 0); // expect nonempty input let (mut yycursor, mut yymarker) = (0, 0); /*!re2c re2c:yyfill:enable = 0; re2c:YYCTYPE = u8; !use:fish; !use:colors; * { return Ans::Dunno; } // overrides inherited '*' rules */ } fn main() { assert_eq!(lex(b"salmon"), Ans::Fish); assert_eq!(lex(b"what?"), Ans::Dunno); } // re2rust $INPUT -o $OUTPUT --input-encoding utf8 --api simple // This example supports multiple input encodings: UTF-8 and UTF-32. // Both lexers are generated from the same rules block, and the use // blocks add only encoding-specific configurations. /*!rules:re2c re2c:yyfill:enable = 0; "x y" { return Some(yycursor); } * { return None; } */ fn lex_utf8(yyinput: &[u8]) -> Option { assert!(yyinput.len() > 0); // expect nonempty input let (mut yycursor, mut yymarker) = (0, 0); /*!use:re2c re2c:encoding:utf8 = 1; re2c:YYCTYPE = u8; */ } fn lex_utf32(yyinput: &[u32]) -> Option { assert!(yyinput.len() > 0); // expect nonempty input let (mut yycursor, mut yymarker) = (0, 0); /*!use:re2c re2c:encoding:utf32 = 1; re2c:YYCTYPE = u32; */ } fn main() { let s8 = vec![0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79]; assert_eq!(lex_utf8(&s8), Some(s8.len())); let s32 = vec![0x2200, 0x78, 0x20, 0x2203, 0x79]; assert_eq!(lex_utf32(&s32), Some(s32.len())); } (Submatch extraction) re2rust . Tags (tags) @stag #mtag stag mtag . -T --tags re2c:tags = 1 . : ( ) . s ( S-tags) m ( m-tags) ( ). svars mvars . (leftmost greedy) . Captures (capturing groups) . --captures re2c:captures = 1 . --leftmost-captures () --posix-captures POSIX . : (! ... ). --invert-captures re2c:invert-captures = 1 . yynmatch ( ) yypmatch . yynmatch yypmatch yypmatch [yynmatch * 2] . YYMAXNMATCH maxnmatch yynmatch . Captvars --captvars re2c:captvars = 1 . --captures : yynmatch yypmatch re2rust yytl yytr k- ( svars ). : --leftmost-captvars re2c:leftmost-captvars = 1 ( ) --posix-captvars re2c:posix-captvars POSIX. Tagged Deterministic Finite Automata with Lookahead ( TDFA) . TDFA (overhead) . TDFA DFA . . TDFA (tag variables) : . . stags mtags . . YYFILL . S-tag : o s-tag: t = YYCURSOR API C YYSTAGP(t) API (generic) o s-tag: t = NULL API C YYSTAGN(t) API (generic) o s-tag : t1 = t2 M-tag : o m-tag: YYMTAGP(t) API (generic) o m-tag: YYMTAGN(t) API (generic) o m-tag : t1 = t2 S-tag ( ) . M-tag . m-tag ( ) m-tag (prefix-tree) (v, p) v p . s-tag : major minor patch ( ). YYFILL . // re2rust $INPUT -o $OUTPUT --api simple #[derive(Debug, PartialEq)] struct SemVer(u32, u32, u32); // version: (major, minor, patch) const NONE: usize = std::usize::MAX; fn s2n(str: &[u8]) -> u32 { // convert a pre-parsed string to a number let mut n = 0; for i in str { n = n * 10 + *i as u32 - 48; } return n; } fn parse(yyinput: &[u8]) -> Option { assert_eq!(yyinput.last(), Some(&0)); // expect null-terminated input let (mut yycursor, mut yymarker) = (0, 0); // Final tag variables available in semantic action. /*!svars:re2c format = '#[allow(unused_mut)]\nlet mut @@;\n'; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = 'let mut @@ = NONE;'; */ /*!re2c re2c:YYCTYPE = u8; re2c:yyfill:enable = 0; re2c:tags = 1; num = [0-9]+; @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\x00] { let major = s2n(&yyinput[t1..t2]); let minor = s2n(&yyinput[t3..t4]); let patch = if t5 != NONE {s2n(&yyinput[t5..yycursor - 1])} else {0}; return Some(SemVer(major, minor, patch)); } * { return None; } */ } fn main() { assert_eq!(parse(b"23.34\0"), Some(SemVer(23, 34, 0))); assert_eq!(parse(b"1.2.99999\0"), Some(SemVer(1, 2, 99999))); assert_eq!(parse(b"1.a\0"), None); } s-tag YYFILL . YYFILL . s-tag . ( m-tag ) . // re2rust $INPUT -o $OUTPUT use std::fs::File; use std::io::{Read, Write}; const BUFSIZE: usize = 4096; const NONE: usize = usize::MAX; struct State { file: File, yyinput: [u8; BUFSIZE], yylimit: usize, yycursor: usize, yymarker: usize, token: usize, // Intermediate tag variables must be part of the lexer state passed to YYFILL. // They don't correspond to tags and should be autogenerated by re2c. /*!stags:re2c format = "@@: usize,\n"; */ eof: bool, } #[derive(PartialEq)] enum Fill { Ok, Eof, LongLexeme } #[derive(Debug, PartialEq)] struct SemVer(u32, u32, u32); // version: (major, minor, patch) fn s2n(str: &[u8]) -> u32 { // convert a pre-parsed string to a number let mut n = 0; for i in str { n = n * 10 + *i as u32 - 48; } return n; } macro_rules! shift { // ignore overflow, marker and tags may not be set yet ($x:expr, $y:expr) => { $x = $x.overflowing_sub($y).0 } } fn fill(st: &mut State) -> Fill { if st.eof { return Fill::Eof; } // Error: lexeme too long. In real life could reallocate a larger buffer. if st.token < 1 { return Fill::LongLexeme; } // Shift buffer contents (discard everything up to the current token). st.yyinput.copy_within(st.token..st.yylimit, 0); st.yylimit -= st.token; st.yycursor -= st.token; shift!(st.yymarker, st.token); // Tag variables need to be shifted like other input positions. The check // for NONE is only needed if some tags are nested inside of alternative or // repetition, so that they can have NONE value. /*!stags:re2c format = "if st.@@ != NONE { shift!(st.@@, st.token); }\n"; */ st.token = 0; // Fill free space at the end of buffer with new data from file. match st.file.read(&mut st.yyinput[st.yylimit..BUFSIZE - 1]) { Ok(n) => { st.yylimit += n; st.eof = n == 0; st.yyinput[st.yylimit] = 0; } Err(why) => panic!("cannot read from file: {}", why) } return Fill::Ok; } fn parse(st: &mut State) -> Option> { let mut vers = Vec::new(); // Final tag variables available in semantic action. /*!svars:re2c format = 'let mut @@;\n'; */ 'parse: loop { st.token = st.yycursor; /*!re2c re2c:api = record; re2c:eof = 0; re2c:tags = 1; re2c:yyrecord = st; re2c:YYCTYPE = u8; re2c:YYFILL = "fill(st) == Fill::Ok"; num = [0-9]+; num @t1 "." @t2 num @t3 ("." @t4 num)? [\n] { let major = s2n(&st.yyinput[st.token..t1]); let minor = s2n(&st.yyinput[t2..t3]); let patch = if t4 != NONE {s2n(&st.yyinput[t4..st.yycursor - 1])} else {0}; vers.push(SemVer(major, minor, patch)); continue 'parse; } $ { return Some(vers); } * { return None; } */ } } fn main() { let fname = "input"; let verstr = b"1.22.333\n"; let expect = (0..BUFSIZE).map(|_| SemVer(1, 22, 333)).collect(); // Prepare input file (make sure it exceeds buffer size). match File::create(fname) { Err(why) => panic!("cannot open {}: {}", fname, why), Ok(mut file) => match file.write_all(&verstr.repeat(BUFSIZE)) { Err(why) => panic!("cannot write to {}: {}", fname, why), Ok(_) => {} } }; // Reopen input file for reading. let file = match File::open(fname) { Err(why) => panic!("cannot read file {}: {}", fname, why), Ok(file) => file, }; // Initialize lexer state. let yylimit = BUFSIZE - 1; let mut st = State { file: file, yyinput: [0; BUFSIZE], // sentinel is set to zero, which triggers YYFILL yylimit: yylimit, yycursor: yylimit, yymarker: yylimit, token: yylimit, /*!stags:re2c format = "@@: NONE,\n"; */ eof: false, }; // Run the lexer and check results. assert_eq!(parse(&mut st), Some(expect)); // Cleanup: remove input file. match std::fs::remove_file(fname) { Err(why) => panic!("cannot remove {}: {}", fname, why), Ok(_) => {} } } (capturing groups) . // re2rust $INPUT -o $OUTPUT --api simple #[derive(Debug, PartialEq)] struct SemVer(u32, u32, u32); // version: (major, minor, patch) const NONE: usize = std::usize::MAX; fn s2n(str: &[u8]) -> u32 { // convert a pre-parsed string to a number let mut n = 0; for i in str { n = n * 10 + *i as u32 - 48; } return n; } fn parse(yyinput: &[u8]) -> Option { assert_eq!(yyinput.last(), Some(&0)); // expect null-terminated input let (mut yycursor, mut yymarker) = (0, 0); // Final tag variables available in semantic action. /*!stags:re2c format = 'let mut @@ = NONE;'; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!svars:re2c format = '#[allow(unused_mut)]\nlet mut @@;\n'; */ /*!re2c re2c:YYCTYPE = u8; re2c:yyfill:enable = 0; re2c:captvars = 1; num = [0-9]+; (num) "." (num) ("." num)? [\x00] { assert!(yytl0 == 0 && yytr0 == yyinput.len()); let major = s2n(&yyinput[yytl1..yytr1]); let minor = s2n(&yyinput[yytl2..yytr2]); let patch = if yytl3 == NONE {0} else {s2n(&yyinput[yytl3 + 1..yytr3])}; return Some(SemVer(major, minor, patch)); } * { return None; } */ } fn main() { assert_eq!(parse(b"23.34\0"), Some(SemVer(23, 34, 0))); assert_eq!(parse(b"1.2.99999\0"), Some(SemVer(1, 2, 99999))); assert_eq!(parse(b"1.a\0"), None); } m-tag . trie . // re2rust $INPUT -o $OUTPUT --api simple const NONE: usize = std::usize::MAX; const MTAG_ROOT: usize = NONE - 1; // An m-tag tree is a way to store histories with an O(1) copy operation. // Histories naturally form a tree, as they have common start and fork at some // point. The tree is stored as an array of pairs (tag value, link to parent). // An m-tag is represented with a single link in the tree (array index). type MtagTrie = Vec::; struct MtagElem { elem: usize, // tag value pred: usize, // index of the predecessor node or root } // Append a single value to an m-tag history. fn add_mtag(trie: &mut MtagTrie, mtag: usize, value: usize) -> usize { trie.push(MtagElem{elem: value, pred: mtag}); return trie.len() - 1; } // Recursively unwind tag histories and collect version components. fn unwind(trie: &MtagTrie, x: usize, y: usize, str: &[u8], ver: &mut Ver) { // Reached the root of the m-tag tree, stop recursion. if x == MTAG_ROOT && y == MTAG_ROOT { return; } // Unwind history further. unwind(trie, trie[x].pred, trie[y].pred, str, ver); // Get tag values. Tag histories must have equal length. assert!(x != MTAG_ROOT && y != MTAG_ROOT); let (ex, ey) = (trie[x].elem, trie[y].elem); if ex != NONE && ey != NONE { // Both tags are valid string indices, extract component. ver.push(s2n(&str[ex..ey])); } else { // Both tags are NONE (this corresponds to zero repetitions). assert!(ex == NONE && ey == NONE); } } type Ver = Vec::; // unbounded number of version components fn s2n(str: &[u8]) -> u32 { // convert a pre-parsed string to a number let mut n = 0; for i in str { n = n * 10 + *i as u32 - 48; } return n; } fn parse(yyinput: &[u8]) -> Option { assert_eq!(yyinput.last(), Some(&0)); // expect null-terminated input let (mut yycursor, mut yymarker) = (0, 0); let mut mt: MtagTrie = Vec::new(); // Final tag variables available in semantic action. /*!svars:re2c format = 'let @@;\n'; */ /*!mvars:re2c format = 'let @@;\n'; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = 'let mut @@ = NONE;'; */ /*!mtags:re2c format = 'let mut @@ = MTAG_ROOT;'; */ /*!re2c re2c:YYCTYPE = u8; re2c:YYMTAGP = "@@ = add_mtag(&mut mt, @@, yycursor);"; re2c:YYMTAGN = "@@ = add_mtag(&mut mt, @@, NONE);"; re2c:yyfill:enable = 0; re2c:tags = 1; num = [0-9]+; @t1 num @t2 ("." #t3 num #t4)* [\x00] { let mut ver: Ver = Vec::new(); ver.push(s2n(&yyinput[t1..t2])); unwind(&mt, t3, t4, yyinput, &mut ver); return Some(ver); } * { return None; } */ } fn main() { assert_eq!(parse(b"1\0"), Some(vec![1])); assert_eq!(parse(b"1.2.3.4.5.6.7\0"), Some(vec![1, 2, 3, 4, 5, 6, 7])); assert_eq!(parse(b"1.2.\0"), None); } code point ( ) code unit ( ) . code point . code unit . code point code unit . (fixed-length) code point code unit . (variable-length) code point code unit . "any" [^] code point code unit ( code unit * ). code unit : yych code unit YYCTYPE code unit . code point . re2rust (automata) code point code unit . : code point code unit . : o ASCII ( ). [0-255] code point code unit . o EBCDIC ( --ebcdic re2c:encoding:ebcdic). [0-255] code point code unit . o UCS2 ( --ucs2 re2c:encoding:ucs2). [0-0xFFFF] code point code unit . o UTF8 ( --utf8 re2c:encoding:utf8). . code unit . code point code unit . o UTF16 ( --utf16 re2c:encoding:utf16). . code unit . code point code unit . o UTF32 ( --utf32 re2c:encoding:utf32). [0-0x10FFFF] code point code unit . include/unicode_blocks.re include/unicode_categories.re include/unicode_properties.re re2rust Unicode . --input-encoding . --input-encoding utf8 re2rust UTF8 ( --utf8 ). --encoding-policy (Unicode surrogates code point [0xD800-0xDFFF]) re2rust . UTF8 . // re2rust $INPUT -o $OUTPUT --utf8 --api simple /*!include:re2c "unicode_categories.re" */ fn lex(yyinput: &[u8]) -> bool { assert_eq!(yyinput.last(), Some(&0)); // expect null-terminated input let (mut yycursor, mut yymarker) = (0, 0); /*!re2c re2c:YYCTYPE = u8; re2c:yyfill:enable = 0; // Simplified "Unicode Identifier and Pattern Syntax" // (see https://unicode.org/reports/tr31) id_start = L | Nl | [$_]; id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3]; identifier = id_start id_continue*; identifier { return true; } * { return false; } */ } fn main() { assert!(lex("_\0".as_bytes())); } re2rust /*!include:re2c FILE */ %{include FILE %} !include FILE ; FILE . re2rust include include -I . / include re2rust #include C/C++ : FILE / . include include . include --depfile . re2rust include include/ . ( ) re2rust . include . (definitions.rs) #[derive(Debug, PartialEq)] enum Num { Int, Float, NaN } /*!re2c number = [1-9][0-9]*; */ (extra_rules.re.inc) // floating-point numbers frac = [0-9]* "." [0-9]+ | [0-9]+ "."; exp = 'e' [+-]? [0-9]+; float = frac exp? | [0-9]+ exp; float { return Num::Float; } (Input file) // re2rust $INPUT -o $OUTPUT --api simple /*!include:re2c "definitions.rs" */ fn lex(yyinput: &[u8]) -> Num { assert_eq!(yyinput.last(), Some(&0)); // expect null-terminated input let mut yycursor = 0; let mut yymarker = 0; /*!re2c re2c:yyfill:enable = 0; re2c:YYCTYPE = u8; * { return Num::NaN; } number { return Num::Int; } !include "extra_rules.re.inc"; */ } fn main() { assert_eq!(lex(b"123\0"), Num::Int); assert_eq!(lex(b"123.4567\0"), Num::Float); } re2rust --header re2c:header /*!header:re2c:on*/ /*!header:re2c:off*/ %{header:on%} %{header:off%} .re . . re2rust --header re2c:header ( stdout) . re2rust (translation units) . (lexer state) (tag variables) ( ). (Input file) // re2rust $INPUT -o $OUTPUT --header lexer/state.rs mod lexer; use lexer::state::State; // the module is generated by re2c /*!header:re2c:on*/ pub struct State<'a> { pub yyinput: &'a [u8], pub yycursor: usize, /*!stags:re2c format = "pub @@: usize,"; */ } /*!header:re2c:off*/ fn lex(yyrecord: &mut State) -> usize { assert_eq!(yyrecord.yyinput.last(), Some(&0)); // expect null-terminated input let t: usize; /*!re2c re2c:header = "lexer/state.rs"; re2c:yyfill:enable = 0; re2c:api = record; re2c:YYCTYPE = "u8"; re2c:tags = 1; [a]* @t [b]* { return t; } */ } fn main() { let mut st = State { yyinput: b"ab\0", yycursor: 0, /*!stags:re2c format = "@@: 0,"; */ }; assert_eq!(lex(&mut st), 1); } (Header file) /* Generated by re2c */ pub struct State<'a> { pub yyinput: &'a [u8], pub yycursor: usize, pub yyt1: usize, } -S, --skeleton re2rust re2rust C . . DFA ( ) re2rust : .input DFA .keys . .input . : o re2rust ( DFA ). o . o (minimized test cases) ( ). ( ). re2rust DFA . : DFA . ( ASCII UTF-8 EBCDIC) DFA . DFA re2rust DFA . . re2rust ( ) . (re2rust ). (Visualization and debug) -D, --emit-dot re2rust DFA DOT (dump) . Graphviz DFA . DFA . --dump-nfa --dump-dfa-raw ( ). (SEE ALSO) re2c : http://re2c.org. flex(1) lex(1) quex(http://quex.sourceforge.net). (AUTHORS) re2rust Peter Bumbulis (peter@csg.uwaterloo.ca) . Marcus Boerger Dan Nuffer . : Brian Young (bayoung@acm.org) Marcus Boerger Dan Nuffer (nuffer@users.sourceforge.net) Ulya Trofimovich (skvadrik@gmail.com) Serghei Iakovlev Sergei Trofimovich Petr Skocik ligfx raekye PolarGoose. RE2RUST(1)