RE2C(1) RE2C(1) (NAME) re2c - C/C++ (SYNOPSIS) : C++ ( C ). re2c [ OPTIONS ] [ WARNINGS ] INPUT - stdin . (INTRODUCTION) re2c . ( C/C++ ) / . . re2c C/C++ . : // re2c $INPUT -o $OUTPUT -i --case-ranges #include int lex(const char* YYCURSOR) { /*!re2c re2c:yyfill:enable = 0; re2c:define:YYCTYPE = "unsigned char"; [1-9][0-9]* { return 0; } * { return 1; } */ } int main() { assert(lex("1234") == 0); return 0; } re2c : /* Generated by re2c */ // re2c $INPUT -o $OUTPUT -i --case-ranges #include int lex(const char* YYCURSOR) { { unsigned char yych; yych = *YYCURSOR; switch (yych) { case '1' ... '9': goto yy2; default: goto yy1; } yy1: ++YYCURSOR; { return 1; } yy2: yych = *++YYCURSOR; switch (yych) { case '0' ... '9': goto yy2; default: goto yy3; } yy3: { return 0; } } } int main() { assert(lex("1234") == 0); return 0; } re2c . : name = regular-expression ; name regular-expression . . re2c . name . . . . configuration = value ; re2c . re2c . value . . . regular-expression code regular-expression ( := ). regular-expression code . . . : * $. ( ). . . !action code code ( ). . !directive ; . . . (Blocks) /*!re2c */ %{ %} ( ). 2.2 . : /*!re2c[:] ... */ %{[:] ... %} . re2c . . . /*!local:re2c[:] ... */ %{local[:] ... %} ( ). /*!rules:re2c[:] ... */ %{rules[:] ... %} -- . ( ). 2.2 re2c -r --reusable . /*!use:re2c[:] ... */ %{use[:] ... %} . re2c . ( ). . 2.2 re2c -r --reusable . /*!max:re2c[:[:...]] ... */ %{max[:[:...]] ... %} YYMAXFILL . YYMAXFILL ( ). C (#define YYMAXFILL ) Go (var YYMAXFILL int = ) . format @@{max} ( @@) YYMAXFILL . /*!maxnmatch:re2c[:[:...]] ... */ %{maxnmatch[:[:...]] ... %} YYMAXNMATCH ( -P --posix-captures ). YYMAXNMATCH ( ). C (#define YYMAXNMATCH ) Go (var YYMAXNMATCH int = ) . format @@{max} ( @@) YYMAXNMATCH . /*!stags:re2c[:[:...]] ... */, /*!mtags:re2c[:[:...]] ... */ %{stags[:[:...]] ... %}, %{mtags[:[:...]] ... %{ s-tag/m-tag re2c . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!svars:re2c[:[:...]] ... */, /*!mvars:re2c[:[:...]] ... */ %{svars[:[:...]] ... %}, %{mvars[:[:...]] ... %{ s-tag/m-tag ( --tags) re2c ( --captvars --posix-captvars) . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!getstate:re2c[:[:...]] ... */ %{getstate[:[:...]] ... %} ( --storable-state ). . . . --loop-switch goto . /*!conditions:re2c[:[:...]] ... */, /*!types:re2c... */ %{conditions[:[:...]] ... %}, %{types... %} ( --conditions ). ( ). YYCONDTYPE . format separator . format @@{cond} ( @@) @@{num} . separator format . /*!include:re2c */ %{include %} . #include C/C++ . --depfile . /*!header:re2c:on*/ %{header:on %} . header:off re2c -t --type-header . /*!header:re2c:off*/ %{header:off %} header:on*/ . /*!ignore:re2c ... */ %{ignore ... %} . re2c : re2c:api, re2c:input --api. re2c:api:sigil ("" "sigil") API . @@ . sigil . sigil $ ${name} . API . API re2c:YYFILL@len YYFILL. re2c:api:style API . functions ( C) free-form ( Go Rust). functions API . ( YYFILL) C/C++ (closures) Go Rust . free-form API : @@{var} @@ ( ). API re2c:YYFILL:naked YYFILL . re2c:bit-vectors, re2c:flags:bit-vectors, re2c:flags:b --bit-vectors . re2c:captures, re2c:leftmost-captures --leftmost-captures . re2c:captvars, re2c:leftmost-captvars --leftmost-captvars . re2c:case-insensitive, re2c:flags:case-insensitive --case-insensitive . re2c:case-inverted, re2c:flags:case-inverted --case-inverted . re2c:case-ranges, re2c:flags:case-ranges --case-ranges . re2c:computed-gotos, re2c:flags:computed-gotos, re2c:flags:g --computed-gotos . re2c:computed-gotos:relative, re2c:cgoto:relative --computed-gotos-relative . re2c:computed-gotos:threshold, re2c:cgoto:threshold goto if (bitmaps) . 9 . re2c:cond:abort (default case) . re2c:cond:goto :=> . goto @@; . @@ ( re2c:api:sigil re2c:cond:goto@cond ). re2c:cond:goto@cond (sigil) re2c:cond:goto . @@ . re2c:api:sigil . re2c:cond:divider . /* *********************************** */ . ( re2c:api;sigil re2c:cond:divider@cond). re2c:cond:divider@cond (sigil) re2c:cond:divider . @@ . re2c:api:sigil . re2c:cond:prefix, re2c:condprefix . yyc_ . re2c:cond:enumprefix, re2c:condenumprefix . yyc . re2c:debug-output, re2c:flags:debug-output, re2c:flags:d --debug-output . re2c:empty-class, re2c:flags:empty-class --empty-class . re2c:encoding:ebcdic, re2c:flags:ecb, re2c:flags:e --ebcdic . re2c:encoding:ucs2, re2c:flags:wide-chars, re2c:flags:w --ucs2 . re2c:encoding:utf8, re2c:flags:utf-8, re2c:flags:8 --utf8 . re2c:encoding:utf16, re2c:flags:utf-16, re2c:flags:x --utf16 . re2c:encoding:utf32, re2c:flags:unicode, re2c:flags:u --utf32 . re2c:encoding-policy, re2c:flags:encoding-policy --encoding-policy . re2c:eof (sentinel symbol) $ . -1 ( $ ). (code units) . . re2c:header, re2c:flags:type-header, re2c:flags:t . --header . re2c:indent:string . "\t" . (whitespace) . . re2c:indent:top . . . re2c:invert-captures --invert-captures . re2c:label:prefix, re2c:labelprefix DFA . yy . re2c:label:start, re2c:startlabel . . . . ( ). re2c:label:yyFillLabel YYFILL re2c:eof (storable state mode) . re2c:label:yyloop --loop-switch . yyloop . re2c:label:yyNext YYGETSTATE ( re2c:state:nextlabel ) . yyNext . re2c:lookahead, re2c:flags:lookahead ( --no-lookahead ). re2c:monadic (monadic notation) ( Haskell )..TP re2c:nested-ifs, re2c:flags:nested-ifs, re2c:flags:s --nested-ifs . re2c:posix-captures, re2c:flags:posix-captures, re2c:flags:P --posix-captures . re2c:posix-captvars --posix-captvars . re2c:tags, re2c:flags:tags, re2c:flags:T --tags . re2c:tags:expression . re2c yyt . struct . @@{tag} @@ . re2c:tags:expression = "s.@@"; s.yyt . re2c:api:sigil . re2c:tags:negative ( -1 ). re2c:tags:prefix . yyt . re2c:sentinel (sentinel) ( re2c:yyfill:enable = 0; re2c:eof ). : . -1` ( re2c ). . re2c:state:abort (default case) -1 . re2c:state:nextlabel YYGETSTATE yyNext ( ). re2c:label:start getstate YYGETSTATE . re2c:unsafe, re2c:flags:unsafe --no-unsafe . unsafe YYPEEK . ( ). Rust . re2c:YYBACKUP, re2c:define:YYBACKUP API YYBACKUP . re2c:YYBACKUPCTX, re2c:define:YYBACKUPCTX API YYBACKUPCTX . re2c:YYCONDTYPE, re2c:define:YYCONDTYPE API YYCONDTYPE . re2c:YYCTYPE, re2c:define:YYCTYPE API YYCTYPE . re2c:YYCTXMARKER, re2c:define:YYCTXMARKER API YYCTXMARKER . re2c:YYCURSOR, re2c:define:YYCURSOR API YYCURSOR . re2c:YYDEBUG, re2c:define:YYDEBUG API YYDEBUG . re2c:YYFILL, re2c:define:YYFILL API YYFILL . re2c:YYFILL@len, re2c:define:YYFILL@len (sigil) YYFILL . @@ . re2c:api:sigil . re2c:YYFILL:naked, re2c:define:YYFILL:naked re2c:api:style YYFILL . API free-form . re2c:YYFN API YYFN . re2c:YYINPUT API YYINPUT . re2c:YYGETCOND, re2c:define:YYGETCONDITION API YYGETCOND . re2c:YYGETCOND:naked, re2c:define:YYGETCONDITION:naked re2c:api:style YYGETCOND . API free-form . re2c:YYGETSTATE, re2c:define:YYGETSTATE API YYGETSTATE . re2c:YYGETSTATE:naked, re2c:define:YYGETSTATE:naked re2c:api:style YYGETSTATE . API free-form . re2c:YYGETACCEPT, re2c:define:YYGETACCEPT API YYGETACCEPT . re2c:YYLESSTHAN, re2c:define:YYLESSTHAN API YYLESSTHAN . re2c:YYLIMIT, re2c:define:YYLIMIT API YYLIMIT . re2c:YYMARKER, re2c:define:YYMARKER API YYMARKER . re2c:YYMTAGN, re2c:define:YYMTAGN API YYMTAGN . re2c:YYMTAGP, re2c:define:YYMTAGP API YYMTAGP . re2c:YYPEEK, re2c:define:YYPEEK API YYPEEK . re2c:YYRESTORE, re2c:define:YYRESTORE API YYRESTORE . re2c:YYRESTORECTX, re2c:define:YYRESTORECTX API YYRESTORECTX . re2c:YYRESTORETAG, re2c:define:YYRESTORETAG API YYRESTORETAG . re2c:YYSETCOND, re2c:define:YYSETCONDITION API YYSETCOND . re2c:YYSETCOND@cond, re2c:define:YYSETCONDITION@cond (sigil) YYSETCOND . @@ . re2c:api:sigil . re2c:YYSETCOND:naked, re2c:define:YYSETCONDITION:naked re2c:api:style YYSETCOND . API free-form . re2c:YYSETSTATE, re2c:define:YYSETSTATE API YYSETSTATE . re2c:YYSETSTATE@state, re2c:define:YYSETSTATE@state (sigil) YYSETSTATE . @@ . re2c:api:sigil . re2c:YYSETSTATE:naked, re2c:define:YYSETSTATE:naked re2c:api:style YYSETSTATE . API free-form . re2c:YYSETACCEPT, re2c:define:YYSETACCEPT API YYSETACCEPT . re2c:YYSKIP, re2c:define:YYSKIP API YYSKIP . re2c:YYSHIFT, re2c:define:YYSHIFT API YYSHIFT . re2c:YYCOPYMTAG, re2c:define:YYCOPYMTAG API YYCOPYMTAG . re2c:YYCOPYSTAG, re2c:define:YYCOPYSTAG API YYCOPYSTAG ..TP re2c:YYSHIFTMTAG, re2c:define:YYSHIFTMTAG generic API YYSHIFTMTAG . re2c:YYSHIFTSTAG, re2c:define:YYSHIFTSTAG generic API YYSHIFTSTAG . re2c:YYSTAGN, re2c:define:YYSTAGN generic API YYSTAGN . re2c:YYSTAGP, re2c:define:YYSTAGP generic API YYSTAGP . re2c:yyaccept, re2c:variable:yyaccept API yyaccept . re2c:yybm, re2c:variable:yybm API yybm . re2c:yybm:hex, re2c:variable:yybm:hex (bitmaps) --bit-vectors (hexadecimal) . ( ). re2c:yych, re2c:variable:yych API yych . re2c:yych:emit, re2c:variable:yych:emit yych . . re2c:yych:conversion, re2c:variable:yych:conversion re2c yych YYCTYPE . ( ). re2c:yych:literals, re2c:variable:yych:literals yych . : char ( ) hex ( ) char_or_hex ( ). re2c:yyctable, re2c:variable:yyctable API yyctable . re2c:yynmatch, re2c:variable:yynmatch API yynmatch . re2c:yypmatch, re2c:variable:yypmatch API yypmatch . re2c:yytarget, re2c:variable:yytarget API yytarget . re2c:yystable, re2c:variable:yystable . re2c:yystate, re2c:variable:yystate API yystate . re2c:yyfill, re2c:variable:yyfill API yyfill . re2c:yyfill:check YYFILL ( YYLESSTHAN generic API YYLIMIT C pointer API) . ( ). re2c:yyfill:enable YYFILL ( ) . ( ) YYFILL ( /sentinel ). ( -W) re2c:sentinel . (YYFILL ). re2c:yyfill:parameter YYFILL . . ( ). re2c:YYFILL:naked re2c:api:style . re2c:yyfn:sep YYFN ( ). re2c:yyfn:throw YYFN ( ). re2c : "foo" . 'foo' . [a-xyz], [^a-xyz] ( ). . . R \ S R S. R* R. R+ R. R? R. R{n} R n . R{n,} R n . R{n,m} R n m . (R) R . (R) --invert-captures . (!R) (!R) --invert-captures . R S : R S. R | S : R S. R / S (Lookahead): R S S . name name ( "name" Flex). {name} name Flex. @stag s-tag: @stag stag . #mtag m-tag: #mtag mtag . $ . : \a \b \f \n \r \t \v \\ \ooo \xhh \uhhhh \Uhhhhhhhh. (Actions) re2c : !entry code (Entry action) code . start conditions ( ) . (lexeme) . !pre_rule code (Pre-rule action) code ( start conditions) . ( ) . !post_rule code (Post-rule action) code ( start conditions) . (trap statements) . re2c : !use:name ; use name . . : . * $ . use use . !include file ; include : file . (Program interface) (primitives) API . . . re2c ( ) . API . re2c API : Simple API API C/C++ . YYCURSOR YYMARKER YYCTXMARKER YYLIMIT YYCTYPE* . Record API ( 4.0 ) Record API (lexer) struct . --api record re2c:api = record . API yyrecord ( re2c:yyrecord ) struct yycursor yymarker yyctxmarker yylimit ( ). Generic API ( 0.14 ) API . --api generic re2c:api = generic . API : YYPEEK YYSKIP YYBACKUP YYBACKUPCTX YYSTAGP YYSTAGN YYMTAGP YYMTAGN YYRESTORE YYRESTORECTX YYRESTORETAG YYSHIFT YYSHIFTSTAG YYSHIFTMTAG YYLESSTHAN. YYEND. Generic API : Free-form (Free-form) re2c:api:style = free-form . @@{var} @@ . . Generic API cursor limit marker ctxmarker : /*!re2c re2c:YYPEEK = "*cursor"; re2c:YYSKIP = "++cursor;"; re2c:YYBACKUP = "marker = cursor;"; re2c:YYRESTORE = "cursor = marker;"; re2c:YYBACKUPCTX = "ctxmarker = cursor;"; re2c:YYRESTORECTX = "cursor = ctxmarker;"; re2c:YYRESTORETAG = "cursor = ${tag};"; re2c:YYLESSTHAN = "limit - cursor < @@{len}"; re2c:YYEND = "limit == cursor"; re2c:YYSTAGP = "@@{tag} = cursor;"; re2c:YYSTAGN = "@@{tag} = NULL;"; re2c:YYSHIFT = "cursor += @@{shift};"; re2c:YYSHIFTSTAG = "@@{tag} += @@{shift};"; */ Function-like (function-like) . re2c:api:style = functions . . Generic API cursor limit marker ctxmarker : #define YYPEEK() *cursor #define YYSKIP() ++cursor #define YYBACKUP() marker = cursor #define YYRESTORE() cursor = marker #define YYBACKUPCTX() ctxmarker = cursor #define YYRESTORECTX() cursor = ctxmarker #define YYRESTORETAG(tag) cursor = tag #define YYLESSTHAN(len) limit - cursor < len #define YYEND() limit == cursor #define YYSTAGP(tag) tag = cursor #define YYSTAGN(tag) tag = NULL #define YYSHIFT(shift) cursor += shift #define YYSHIFTSTAG(tag, shift) tag += shift API . YYCTYPE ( ). ASCII EBCDIC UTF-8 . UTF-16 UCS-2 . UTF-32 . YYCURSOR l-value ( YYINPUT). YYCURSOR . YYCURSOR . YYLIMIT r-value ( YYINPUT). YYLIMIT . . YYCURSOR YYLIMIT ..TP YYMARKER l-value ( YYINPUT). YYCURSOR (rollback) . . YYCTXMARKER l-value (trailing context) ( YYINPUT). . YYCTXMARKER / . YYFILL API len. YYFILL len . re2c:eof len 1 YYFILL . re2c:eof YYFILL . len YYMAXFILL . YYFN --recursive-functions . re2c:fn:sep ( ) . ( void ). : ( ) ( - ). YYINPUT r-value ( ) . YYMAXFILL YYFILL. max . YYLESSTHAN API len. r-value len true . YYEND API . r-value ( ) true . YYEND $ . YYLESSTHAN . YYPEEK API . r-value YYCTYPE . YYSKIP API . YYBACKUP API ( YYRESTORE ). YYRESTORE API YYBACKUP . YYBACKUPCTX API ( YYRESTORECTX ). YYRESTORECTX API YYBACKUPCTX . YYRESTORETAG API tag tag . YYSTAGP API tag tag YYINPUT ( ). YYSTAGP tag . YYSTAGN API tag tag YYINPUT ( ). YYSTAGN tag . YYMTAGP API tag. YYMTAGP tag ( ). YYMTAGN API tag. YYMTAGN tag ( ). YYSHIFT API shift shift ( ). YYCOPYSTAG API lhs rhs s-tag rhs s-tag lhs . lhs rhs . YYCOPYMTAG API lhs rhs m-tag rhs m-tag lhs . lhs rhs . YYSHIFTSTAG API tag shift tag shift ( ). YYSHIFTMTAG API tag shift tag shift ( ). YYMAXNMATCH POSIX . maxnmatch . YYCONDTYPE . conditions --header . YYGETACCEPT var . var . YYSETACCEPT : var ( l-value ) val ( ). var val . YYGETCOND r-value YYCONDTYPE . YYSETCOND cond cond . YYGETSTATE r-value . -1 . YYSETSTATE state state . YYDEBUG -d --debug-output . ( YYDEBUG ). YYDEBUG : state ( DFA -1) symbol ( ). yyaccept l-value . --storable-state . yybm ( --bitmaps). - . - . k- n- n- k- . if switch ..TP yych l-value YYCTYPE . -f --storable-state . yyctable ( --conditions --computed-gotos ). yyfill l-value YYFILL ( YYFILL ). yynmatch l-value POSIX . -P --posix-captures . yypmatch l-value . yynmatch * 2 ( YYMAXNMATCH * 2 ). -P --posix-captures . yystable . yystate l-value --loop-switch DFA . yytarget ( ) . . yytarget --computed-gotos . (OPTIONS) re2c . re2c . re2c . -? --help -h . --api API . simple . record (struct class ) . generic API . --bit-vectors -b . --nested-ifs . --captures, --leftmost-captures . yybmatch 2 * YYMAXNMATCH yynmatch . --captvars, --leftmost-captvars . yytl yytr k- . --case-insensitive . --case-inverted : . --case-ranges switch low ... high. C/C++ GCC Clang Tcc . Tcc . --computed-gotos -g "computed goto" ( ). re2c . cgoto:threshold . cgoto:relative . --bit-vectors . --computed-gotos-relative --computed-gotos ( ). --computed-gotos . --conditions --start-conditions -c "conditions" Flex: . re2c goto . --depfile FILE FILE Makefile : [include-file ...]. / include include . --output . --ebcdic --ecb -e EBCDIC . re2c 0 -- 0xFF . --empty-class re2c . match-empty () ( ). match-none . error . --encoding-policy re2c . fail re2c . substitute re2c 0xFFFD . ignore () re2c . . --flex-syntax -F Flex: - . . --goto-label "goto/label": DFA goto . goto . --header --type-header -t HEADER HEADER. header:on header:off . enum ( conditions ). -I PATH PATH include . include . re2c FILE include -I . --input --api. default simple ( ). custom generic . --input-encoding re2c. ascii () re2c ASCII : . utf8 re2c UTF8 . --invert-captures . (...) (! ...) . (! ...) (...) . --lang . C D Go Haskell Java JS OCaml Python Rust Swift V Zig ( --syntax ). none . --location-format . gnu 'filename:line:column: ...' . msvc 'filename(line,column) ...' . gnu . --loop-switch "loop/switch": DFA switch switch . yystate . yystate case ..TP --nested-ifs -s if switch . . --no-debug-info -i (line directives) . ( diff ). --no-generation-date . --no-version . --no-unsafe unsafe YYPEEK ( Rust ). YYPEEK (bounds-checking) (lexer) . YYPEEK release --no-unsafe unsafe . --output -o OUTPUT OUTPUT . --posix-captures, -P (submatch) POSIX . yybmatch 2 * YYMAXNMATCH yynmatch . --posix-captvars POSIX . yytl yytr k- . --recursive-functions (co-recursive) DFA . --reusable -r 2.2 ( ). --skeleton -S "skeleton" () . "skeleton" . . C . --storable-state -f . push . YYGETSTATE YYSETSTATE yych yyaccept state . --syntax FILE FILE (syntax) . FILE ( ) ( --lang none ). --tags -T (tags) . --ucs2 --wide-chars -w UCS2 . re2c 0 -- 0xFFFF . --nested-ifs . --utf8 --utf-8 -8 UTF-8 . re2c 0 -- 0x10FFFF . --utf16 --utf-16 -x UTF16 . re2c 0 -- 0x10FFFF . --nested-ifs . --utf32 --unicode -u UTF32 . re2c 0 -- 0x10FFFF . --nested-ifs . --verbose . --vernum -V MMmmpp ( ). --version -v . --single-pass -1 . ( ). --debug-output -d YYDEBUG . . --dump-adfa : DFA (tunneling) ( .dot). --dump-cfg : ( .dot). --dump-closure-stats : (closure). --dump-dfa-det : DFA (determinization) ( .dot). --dump-dfa-min : DFA (minimization) ( .dot). --dump-dfa-tagopt : DFA ( .dot). --dump-dfa-tree : DFA ( .dot). --dump-dfa-raw : DFA ( .dot). --dump-interf : (liveness analysis) . --dump-nfa : NFA ( .dot). --emit-dot -D .dot . Graphviz ( dot -Tpng -odfa.png dfa.dot). --dfa-minimization : DFA re2c. moore (Moore) ( ). table " " (table filling) . DFA . --eager-skip : (eagerly) -- . (lazily) -- -- . --no-lookahead . TDFA(0) . TDFA(1) TDFA(0) (one-symbol lookahead) . . TDFA(0) TDFA(1) . --no-optimize-tags : ( ). --posix-closure : (epsilon-closure) POSIX: gor1 () - (Goldberg-Radzik) gtop " " (global topological order) . --posix-prectable : POSIX. complex TNFA . naive TNFA complex ..TP --stadfa . staDFA TDFA . staDFA TDFA . --fixed-tags : (fixed-tag) (all) (none) (toplevel). all . "" ( "") . . (no-match) ( no-match ). . (Warnings) . -W . -Werror . . -W warning. -Wno- warning. -Werror- warning ( -W ). -Wno-error- warning . . -Wcondition-order . --header conditions . -Wempty-character-class . : . re2c . --empty-class . -Wmatch-empty-string ( ). . -Wswapped-range . . -Wundefined-control-flow ( ). . * . -Wunreachable-rules (shadowed) . -Wdeprecated-eof_rule $ . $ . -Wuseless-escape ( escape ). re2c (escape sequence) . -Wnondeterministic-tags n n . -Wsentinel-in-midrule --- . . re2c:sentinel . -Wundefined-syntax-config --syntax . : . . re2c syntax files ( ) . -- . re2c re2c: : (lexer) . ( include/syntax ) re2c . ( --lang none ). : . (Feature lists) re2c : supported_apis API simple record generic. supported_api_styles API functions free-form. supported_code_models goto-label loop-switch recursive-functions. supported_targets code dot skeleton. supported_features nested-ifs bitmaps computed-gotos case-ranges monadic unsafe tags captures captvars. (Language configurations) (parser) re2c : semicolons () . backtick_quoted_strings . single_quoted_strings . indentation_sensitive (indentation) . wrap_blocks_in_braces . (Inplace configurations) re2c: . . (Code templates) . : code-template :: name '=' code-exprs ';' | CODE_TEMPLATE ';' | '' ';' code-exprs :: | code-exprs code-expr code-expr :: STRING | VARIABLE | optional | list optional :: '(' CONDITIONAL '?' code-exprs ')' | '(' CONDITIONAL '?' code-exprs ':' code-exprs ')' list :: '[' VARIABLE ':' code-exprs ']' | '[' VARIABLE '{' NUMBER '}' ':' code-exprs ']' | '[' VARIABLE '{' NUMBER ',' NUMBER '}' ':' code-exprs ']' . . : re2c . ( -1 ). . ( ) ( ) . re2c . if-then-else C : code:if_then_else = [branch{0}: topindent "if " cond " {" nl indent [stmt: stmt] dedent] [branch{1:-1}: topindent "} else" (.cond ? " if " cond) " {" nl indent [stmt: stmt] dedent] topindent "}" nl; branch : branch{0} ( else ) branch{1:-1} ( ) . stmt : [stmt: stmt] . topindent indent dedent nl .cond ( ). : if x { // do something } else if y { // do something else } else { // don't do anything } re2c . . . code:var_local . : type ( ) name ( ) init ( ). : .init ( ). code:var_global code:var_local (top-level) . code:const_local . : type ( ) name ( ) init ( ). code:const_global code:const_local (top-level) . code:array_local () . : type ( ) name ( ) size ( ) row ( ) elem ( -- row ). : .const ( ). code:array_global code:array_local (top-level) . code:array_elem (). : array ( ) index ( ). code:enum (enumeration) ( ). type ( ) elem ( ) init ( ). : .init ( ). code:enum_elem ( code:enum). name ( ) type ( ). code:assign (assignment). lhs ( ) rhs ( ). code:type_int (signed integer). code:type_uint (unsigned integer). code:type_yybm yybm. code:type_yytarget yytarget. code:type_yyctable yyctable. code:cmp_eq <<>> ("equals"). code:cmp_ne <<>> ("not equals"). code:cmp_lt << >> ("less than"). code:cmp_gt << >> ("greater than"). code:cmp_le << >> ("less or equal"). code:cmp_ge << >> ("greater or equal"). code:if_then_else if-then-else . : branch ( ) cond ( ) stmt ( ). : .cond ( ) .many ( ). code:if_then_else_oneline code:if_then_else . code:if_then_else . code:switch switch (case). : expr ( switch ) case ( case ). code:switch_cases switch . case ( ) stmt ( ). code:switch_cases_oneline code:switch_cases . code:switch_cases . code:switch_case_range (case) switch ( ). : val ( ). : .many ( ) .char_literals ( switch -- ). code:switch_case_default (default) switch. code:loop ( ). : label ( ) stmt ( ). code:continue continue. : label ( ). code:goto goto. : label ( ). code:cgoto goto (computed). : array ( goto ) index ( ) base ( .cgoto.relative ). code:cgoto:data goto . : label ( ) type ( ) base ( - .cgoto.relative ). code:fndecl (function declaration). : name ( ) type ( ) throw ( re2c:yyfn:throw ) arg ( ) argname ( ) argtype ( ). : .type ( void ). code:fndef code:fndecl stmt . code:fncall . : name ( ) retval ( l-value ) arg ( ). : .args ( ) .retval ( ). code:tailcall (tail call). : name ( ) arg ( ). : .args ( ) .retval ( void ). code:recursive_functions --recursive-functions. : fn ( ) fndecl ( ) fndef ( )..TP code:fingerprint (fingerprint) . : ver ( re2c ) date ( ). code:line_info ( ). : line ( ) file ( ). code:abort (abort) . code:yydebug YYDEBUG API . : YYDEBUG yyrecord yych ( re2c: ) state ( DFA). code:yypeek YYPEEK API . : YYPEEK YYCTYPE YYINPUT YYCURSOR yyrecord yych ( re2c: ). : .cast ( re2c:yych:conversion ). code:yyskip YYSKIP API . : YYSKIP YYCURSOR yyrecord ( re2c: ). code:yybackup YYBACKUP API . : YYBACKUP YYCURSOR YYMARKER yyrecord ( re2c: ). code:yybackupctx YYBACKUPCTX API . : YYBACKUPCTX YYCURSOR YYCTXMARKER yyrecord ( re2c: ). code:yyskip_yypeek code:yyskip code:yypeek ( code:yyskip code:yypeek). code:yypeek_yyskip code:yypeek code:yyskip ( code:yypeek code:yyskip). code:yyskip_yybackup code:yyskip code:yybackup ( code:yyskip code:yybackup). code:yybackup_yyskip code:yybackup code:yyskip ( code:yybackup code:yyskip). code:yybackup_yypeek code:yybackup code:yypeek ( code:yybackup code:yypeek). code:yyskip_yybackup_yypeek code:yyskip code:yybackup code:yypeek ( ``code:yyskip`` code:yybackup code:yypeek). code:yybackup_yypeek_yyskip code:yybackup code:yypeek code:yyskip ( ``code:yybackup`` code:yypeek code:yyskip). code:yyrestore YYRESTORE API . : YYRESTORE YYCURSOR YYMARKER yyrecord ( re2c: ). code:yyrestorectx YYRESTORECTX API . : YYRESTORECTX YYCURSOR YYCTXMARKER yyrecord ( re2c: ). code:yyrestoretag YYRESTORETAG API . : YYRESTORETAG YYCURSOR yyrecord ( re2c: ) tag ( ). code:yyshift YYSHIFT API . : YYSHIFT YYCURSOR yyrecord ( re2c: ) offset ( ). code:yyshiftstag YYSHIFTSTAG API . : YYSHIFTSTAG yyrecord negative ( re2c: ) tag ( ) offset ( ). : .nested ( -- re2c:tags:negative ). code:yyshiftmtag YYSHIFTMTAG API . : YYSHIFTMTAG ( re2c: ) tag ( ) offset ( ). code:yystagp YYSTAGP API . : YYSTAGP YYCURSOR yyrecord ( re2c: ) tag ( ). code:yymtagp YYMTAGP API . : YYMTAGP ( re2c: ) tag ( ). code:yystagn YYSTAGN API . : YYSTAGN negative yyrecord ( re2c: ) tag ( ). code:yymtagn YYMTAGN API . : YYMTAGN ( re2c: ) tag ( ). code:yycopystag YYCOPYSTAG API . : YYCOPYSTAG yyrecord ( re2c: ) lhs rhs ( ). code:yycopymtag YYCOPYMTAG API . : YYCOPYMTAG yyrecord ( re2c: ) lhs rhs ( ). code:yygetaccept YYGETACCEPT API . : YYGETACCEPT yyrecord ( re2c: ) var ( re2c:yyaccept ). code:yysetaccept YYSETACCEPT API . : YYSETACCEPT yyrecord ( re2c: ) var ( re2c:yyaccept ) val ( ). code:yygetcond YYGETCOND API . : YYGETCOND yyrecord ( re2c: ) var ( re2c:yycond ). code:yysetcond YYSETCOND API . : YYSETCOND yyrecord ( re2c: ) var ( re2c:yycond ) val ( ). code:yygetstate YYGETSTATE API . : YYGETSTATE yyrecord ( re2c: ) var ( re2c:yystate ). code:yysetstate YYSETSTATE API . : YYSETSTATE yyrecord ( re2c: ) var ( re2c:yystate ) val ( ). code:yylessthan YYLESSTHAN API . : YYLESSTHAN YYCURSOR YYLIMIT yyrecord ( re2c: ) need ( ). : .many ( need )..TP code:yyend YYEND API . : YYEND YYCURSOR YYLIMIT. code:yybm_filter yych yybm ( --bitmaps ). : yych ( re2c:yych ). code:yybm_match yybm ( --bitmaps). : yybm yych ( re2c: ) offset ( yybm yych ) mask ( ). code:yytarget_filter yych yytarget ( --computed-gotos ). : yych ( re2c:yych ). : nl (newline). indent . dedent . topindent . . : .api.simple True API (--api simple re2c:api = simple). .api.generic True API (--api generic re2c:api = generic). .api.record True API (--api record re2c:api = record). .api_style.functions True API (re2c:api-style = functions). .api_style.freeform True API (re2c:api-style = free-form). .case_ranges True case (--case-ranges re2c:case-ranges = 1). .cgoto.relative True goto (--computed-gotos-relative re2c:cgoto:relative = 1). .code_model.goto_label True goto/label (--goto-label). .code_model.loop_switch True loop/switch (--loop-switch). .code_model.recursive_functions True (--recursive-function). .date True (fingerprint) . .loop_label True re2c (re2c:label:yyloop ). .monadic True (re2c:monadic = 1). . .start_conditions True (--start-conditions). .storable_state True (--storable-state). .unsafe True re2c "unsafe" (--unsafe re2c:unsafe = 1). "unsafe" . .version True re2c . .yyfill.enable True YYFILL (re2c:yyfill:enable = 1). .yyfn.throw True re2c:yyfn:throw . (lexer) . : o ( *) o o "" : . : . . re2c . . : o (Sentinel). . "" . . : . . YYFILL re2c:yyfill:enable = 0; re2c:eof -1 . o (Sentinel with bounds checks). (generic) : . . "" "" . ( ). . . YYFILL ( ). . . re2c:eof ( ). YYFILL . o (Bounds checks with padding). " " . DFA (SCCs) SCC SCC . (lexemes) . () ( ) . YYMAXFILL max . YYFILL . YYFILL re2c:eof -1 ( ). o (Custom checks). (Generic API) . . Generic API --api custom re2c:api = custom; re2c:yyfill:enable = 0; re2c:yyfill:check = 0; . ..SS (Sentinel) (sentinel) . null . null : (lexeme) ( ). null (syntax error) * (crash) ( -Wsentinel-in-midrule re2c:sentinel ). re2c:yyfill:enable = 0; YYFILL . // re2c $INPUT -o $OUTPUT #include // Expect a null-terminated string. static int lex(const char *YYCURSOR) { int count = 0; for (;;) { /*!re2c re2c:define:YYCTYPE = "unsigned char"; re2c:yyfill:enable = 0; * { return -1; } [\x00] { return count; } [a-z]+ { ++count; continue; } [ ]+ { continue; } */ } } int main() { assert(lex("") == 0); assert(lex("one two three") == 3); assert(lex("f0ur") == -1); return 0; } (Sentinel with bounds checks) (bounds checks) ( 1.2 ). (single-quoted) . null re2c:eof = 0; . sentinel null ( 'aaa\0aa'\0 'aaa\0 ). null . YYLIMIT <= YYCURSOR YYLESSTHAN(1) API (generic API) . (YYFILL re2c:yyfill:enable = 0; YYFILL YYFILL with sentinel ). : $ ( *) -Wundefined-control-flow . // re2c $INPUT -o $OUTPUT #include // Expect a null-terminated string. static int lex(const char *str, unsigned int len) { const char *YYCURSOR = str, *YYLIMIT = str + len, *YYMARKER; int count = 0; for (;;) { /*!re2c re2c:define:YYCTYPE = "unsigned char"; re2c:yyfill:enable = 0; re2c:eof = 0; str = ['] ([^'\\] | [\\][^])* [']; * { return -1; } $ { return count; } str { ++count; continue; } [ ]+ { continue; } */ } } #define TEST(s, r) assert(lex(s, sizeof(s) - 1) == r) int main() { TEST("", 0); TEST("'qu\0tes' 'are' 'fine: \\'' ", 3); TEST("'unterminated\\'", -1); return 0; } ( ). . YYMAXFILL null YYMAXFILL max . null --- (lexeme) ( ). "stop" (null) ( null ). . (YYLIMIT - YYCURSOR) < n API YYLESSTHAN(n) n ( n ). YYFILL(n) n . YYFILL ( ). YYFILL YYFILL with padding . // re2c $INPUT -o $OUTPUT #include #include #include /*!max:re2c*/ static int lex(const char *str, unsigned int len) { // Make a copy of the string with YYMAXFILL zeroes at the end. char *buf = (char*) malloc(len + YYMAXFILL); memcpy(buf, str, len); memset(buf + len, 0, YYMAXFILL); const char *YYCURSOR = buf, *YYLIMIT = buf + len + YYMAXFILL; int count = 0; loop: /*!re2c re2c:api:style = free-form; re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYFILL = "goto fail;"; str = ['] ([^'\\] | [\\][^])* [']; [\x00] { // Check that it is the sentinel, not some unexpected null. if (YYCURSOR - 1 == buf + len) goto exit; else goto fail; } str { ++count; goto loop; } [ ]+ { goto loop; } * { goto fail; } */ fail: count = -1; exit: free(buf); return count; } #define TEST(s, r) assert(lex(s, sizeof(s) - 1) == r) int main() { TEST("", 0); TEST("'qu\0tes' 'are' 'fine: \\'' ", 3); TEST("'unterminated\\'", -1); TEST("'unexpected \0 null\\'", -1); return 0; } generic API . . sentinel null . YYPEEK . . YYPEEK . // re2c $INPUT -o $OUTPUT #include #include #include static int lex(const char *str, unsigned int len) { // For the sake of example create a string without terminating null. char *buf = (char*) malloc(len); memcpy(buf, str, len); const char *cur = buf, *lim = buf + len; int count = 0; for (;;) { /*!re2c re2c:yyfill:enable = 0; re2c:api = custom; re2c:api:style = free-form; re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYPEEK = "cur < lim ? *cur : 0"; // fake null re2c:define:YYSKIP = "++cur;"; * { count = -1; break; } [\x00] { break;; } [a-z]+ { ++count; continue;; } [ ]+ { continue; } */ } free(buf); return count; } #define TEST(s, r) assert(lex(s, sizeof(s) - 1) == r) int main() { TEST("", 0); TEST("one two three ", 3); TEST("f0ur", -1); return 0; } (BUFFER REFILLING) : ( ) . . . : o (cursor): (YYCURSOR C pointer API YYSKIP/YYPEEK generic API) o (limit): (YYLIMIT C pointer API YYLESSTHAN generic API ) o (marker): (YYMARKER default API YYBACKUP/YYRESTORE generic API) o (token): (lexeme) ( re2c API ) o (context marker): (YYCTXMARKER C pointer API YYBACKUPCTX/YYRESTORECTX generic API) o (tag variables): ( stags mtags generic API YYSTAGP/YYSTAGN/YYMTAGP/YYMTAGN) YYFILL . token cursor token token limit . YYFILL ( ). YYFILL EOF : EOF (padding) . -f --storable-state YYFILL ( )..SS YYFILL (sentinel) EOF YYFILL . YYFILL YYLIMIT <= YYCURSOR API C YYLESSTHAN() API . YYFILL . YYFILL . Limit limit re2c:eof . YYFILL ( # ). <-- shift --> >-A------------B---------C-------------D#-----------E-> buffer token marker limit, cursor >-A------------B---------C-------------D------------E#-> buffer, marker cursor limit token <-- shift --> >-A------------B---------C-------------D#--E (EOF) buffer token marker limit, cursor >-A------------B---------C-------------D---E#........ buffer, marker cursor limit token input.txt EOF . // re2c $INPUT -o $OUTPUT #include #include #include #define BUFSIZE 4095 typedef struct { FILE *file; char buf[BUFSIZE + 1], *lim, *cur, *mar, *tok; // +1 for sentinel int eof; } Input; static int fill(Input *in) { if (in->eof) return 1; const size_t shift = in->tok - in->buf; const size_t used = in->lim - in->tok; // Error: lexeme too long. In real life could reallocate a larger buffer. if (shift < 1) return 2; // Shift buffer contents (discard everything up to the current token). memmove(in->buf, in->tok, used); in->lim -= shift; in->cur -= shift; in->mar -= shift; in->tok -= shift; // Fill free space at the end of buffer with new data from file. in->lim += fread(in->lim, 1, BUFSIZE - used, in->file); in->lim[0] = 0; in->eof = in->lim < in->buf + BUFSIZE; return 0; } static int lex(Input *in) { int count = 0; loop: in->tok = in->cur; /*!re2c re2c:api:style = free-form; re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYCURSOR = in->cur; re2c:define:YYMARKER = in->mar; re2c:define:YYLIMIT = in->lim; re2c:define:YYFILL = "fill(in) == 0"; re2c:eof = 0; str = ['] ([^'\\] | [\\][^])* [']; * { return -1; } $ { return count; } str { ++count; goto loop; } [ ]+ { goto loop; } */ } int main() { const char *fname = "input"; const char content[] = "'qu\0tes' 'are' 'fine: \\'' "; // Prepare input file: a few times the size of the buffer, containing // strings with zeroes and escaped quotes. FILE *f = fopen(fname, "w"); for (int i = 0; i < BUFSIZE; ++i) { fwrite(content, 1, sizeof(content) - 1, f); } fclose(f); int count = 3 * BUFSIZE; // number of quoted strings written to file // Initialize lexer state: all pointers are at the end of buffer. Input in; in.file = fopen(fname, "r"); in.cur = in.mar = in.tok = in.lim = in.buf + BUFSIZE; in.eof = 0; // Sentinel (at YYLIMIT pointer) is set to zero, which triggers YYFILL. in.lim[0] = 0; // Run the lexer. assert(lex(&in) == count); // Cleanup: remove input file. fclose(in.file); remove(fname); return 0; } ( EOF ) YYFILL (primitive) . YYFILL (YYLIMIT - YYCURSOR) < n API C YYLESSTHAN(n) API . YYFILL . YYFILL (lexer) ( ). YYFILL (limit) YYMAXFILL ( YYFILL n ). YYFILL ( YYMAXFILL # ). <-- shift --> <-- need --> >-A------------B---------C-----D-------E---F--------G-> buffer token marker cursor limit >-A------------B---------C-----D-------E---F--------G-> buffer, marker cursor limit token <-- shift --> <-- need --> >-A------------B---------C-----D-------E-F (EOF) buffer token marker cursor limit >-A------------B---------C-----D-------E-F############### buffer, marker cursor limit token <- YYMAXFILL -> input.txt (bounds-checking) . // re2c $INPUT -o $OUTPUT #include #include #include /*!max:re2c*/ #define BUFSIZE (4096 - YYMAXFILL) typedef struct { FILE *file; char buf[BUFSIZE + YYMAXFILL], *lim, *cur, *tok; int eof; } Input; static int fill(Input *in, size_t need) { if (in->eof) return 1; const size_t shift = in->tok - in->buf; const size_t used = in->lim - in->tok; // Error: lexeme too long. In real life could reallocate a larger buffer. if (shift < need) return 2; // Shift buffer contents (discard everything up to the current token). memmove(in->buf, in->tok, used); in->lim -= shift; in->cur -= shift; in->tok -= shift; // Fill free space at the end of buffer with new data from file. in->lim += fread(in->lim, 1, BUFSIZE - used, in->file); // If read less than expected, this is end of input => add zero padding // so that the lexer can access characters at the end of buffer. if (in->lim < in->buf + BUFSIZE) { in->eof = 1; memset(in->lim, 0, YYMAXFILL); in->lim += YYMAXFILL; } return 0; } static int lex(Input *in) { int count = 0; loop: in->tok = in->cur; /*!re2c re2c:api:style = free-form; re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYCURSOR = in->cur; re2c:define:YYLIMIT = in->lim; re2c:define:YYFILL = "if (fill(in, @@) != 0) return -1;"; str = ['] ([^'\\] | [\\][^])* [']; [\x00] { // Check that it is the sentinel, not some unexpected null. return in->tok == in->lim - YYMAXFILL ? count : -1; } str { ++count; goto loop; } [ ]+ { goto loop; } * { return -1; } */ } int main() { const char *fname = "input"; const char content[] = "'qu\0tes' 'are' 'fine: \\'' "; // Prepare input file: a few times the size of the buffer, containing // strings with zeroes and escaped quotes. FILE *f = fopen(fname, "w"); for (int i = 0; i < BUFSIZE; ++i) { fwrite(content, 1, sizeof(content) - 1, f); } fclose(f); int count = 3 * BUFSIZE; // number of quoted strings written to file // Initialize lexer state: all pointers are at the end of buffer. // This immediately triggers YYFILL, as the check `in->cur < in->lim` fails. Input in; in.file = fopen(fname, "r"); in.cur = in.tok = in.lim = in.buf + BUFSIZE; in.eof = 0; // Run the lexer. assert(lex(&in) == count); // Cleanup: remove input file. fclose(in.file); remove(fname); return 0; } (Multiple blocks) (lexer) ( ). re2c . start conditions . . goto ( C/C++ Go) (label) . goto . goto ( Rust) switch /switch yystate re2c . (binary) (octal) (decimal) (hexadecimal) . . . . // re2c $INPUT -o $OUTPUT -i #include #include #include static const uint64_t ERROR = UINT64_MAX; #define CHECK(n) if (n > UINT32_MAX) return ERROR; static uint64_t parse_u32(const char *s) { const char *YYCURSOR = s, *YYMARKER; uint64_t u = 0; /*!re2c re2c:yyfill:enable = 0; re2c:define:YYCTYPE = "unsigned char"; end = "\x00"; '0b' / [01] { goto bin; } "0" { goto oct; } "" / [1-9] { goto dec; } '0x' / [0-9a-fA-F] { goto hex; } * { return ERROR; } */ bin: /*!re2c end { return u; } [01] { u = u * 2 + (YYCURSOR[-1] - '0'); CHECK(u); goto bin; } * { return ERROR; } */ oct: /*!re2c end { return u; } [0-7] { u = u * 8 + (YYCURSOR[-1] - '0'); CHECK(u); goto oct; } * { return ERROR; } */ dec: /*!re2c end { return u; } [0-9] { u = u * 10 + (YYCURSOR[-1] - '0'); CHECK(u); goto dec; } * { return ERROR; } */ hex: /*!re2c end { return u; } [0-9] { u = u * 16 + (YYCURSOR[-1] - '0'); CHECK(u); goto hex; } [a-f] { u = u * 16 + (YYCURSOR[-1] - 'a' + 10); CHECK(u); goto hex; } [A-F] { u = u * 16 + (YYCURSOR[-1] - 'A' + 10); CHECK(u); goto hex; } * { return ERROR; } */ } int main() { assert(parse_u32("") == ERROR); assert(parse_u32("1234567890") == 1234567890); assert(parse_u32("0b1101") == 13); assert(parse_u32("0x7Fe") == 2046); assert(parse_u32("0644") == 420); assert(parse_u32("9999999999") == ERROR); return 0; } --start-conditions . (automata) re2c . re2c . : YYGETCOND YYSETCOND . conditions --header re2c:header . re2c:cond:enumprefix . (condition mode) (angle brackets) <*> . : < condition-list > regular-expression code condition-list . regular-expression code . < condition-list > regular-expression => condition code condition-list . regular-expression condition code . < condition-list > regular-expression :=> condition condition-list . regular-expression condition ( ). < condition-list > !action code code action condition-list ( actions ). code code condition-list . !pre_rule ( ). < > code "0" code . !entry ( [condition dispatch] ). < > => condition code condition . < > :=> condition condition . re2c re2c goto/label loop/switch. goto ( C/C++ Go) yyc_ cond ( re2c:cond:prefix ). goto . re2c switch YYGETSTATE . :=> switch ( re2c:cond:goto ). . goto ( Rust) re2c yystate . switch switch DFA . switch . ( YYGETCOND/YYSETCOND ). () () () () . . cond conditions . // re2c $INPUT -o $OUTPUT -ci #include #include #include static const uint64_t ERROR = UINT64_MAX; /*!conditions:re2c*/ static uint64_t parse_u32(const char *s) { const char *YYCURSOR = s, *YYMARKER; int c = yycinit; uint64_t u = 0; /*!re2c re2c:api:style = free-form; re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYGETCONDITION = "c"; re2c:define:YYSETCONDITION = "c = @@;"; re2c:yyfill:enable = 0; <*> * { return ERROR; } '0b' / [01] :=> bin "0" :=> oct "" / [1-9] :=> dec '0x' / [0-9a-fA-F] :=> hex "\x00" { return u; } [01] { u = u * 2 + (YYCURSOR[-1] - '0'); goto yyc_bin; } [0-7] { u = u * 8 + (YYCURSOR[-1] - '0'); goto yyc_oct; } [0-9] { u = u * 10 + (YYCURSOR[-1] - '0'); goto yyc_dec; } [0-9] { u = u * 16 + (YYCURSOR[-1] - '0'); goto yyc_hex; } [a-f] { u = u * 16 + (YYCURSOR[-1] - 'a' + 10); goto yyc_hex; } [A-F] { u = u * 16 + (YYCURSOR[-1] - 'A' + 10); goto yyc_hex; } { if (u > UINT32_MAX) return ERROR; } */ } int main() { assert(parse_u32("") == ERROR); assert(parse_u32("1234567890") == 1234567890); assert(parse_u32("0b1101") == 13); assert(parse_u32("0x7Fe") == 2046); assert(parse_u32("0644") == 420); assert(parse_u32("9999999999") == ERROR); return 0; } --storable-state re2c (lexer) . re2c <<>> ("pull") <<>> ( ). ( (parser) ). : <<>> ("push") . . . <<>> : o YYSETSTATE() YYGETSTATE(state). o yych yyaccept ( ) state . state -1 . o YYFILL . . o . o getstate YYGETSTATE . goto ( --loop-switch). o re2c:eof sentinel with bounds checks . (padding) : (lexeme) . ( ). <<>> ("push") . . <<>> . . // re2c $INPUT -o $OUTPUT -f #include #include #include #define DEBUG 0 #define LOG(...) if (DEBUG) fprintf(stderr, __VA_ARGS__); // Use a small buffer to cover the case when a lexeme doesn't fit. // In real world use a larger buffer. #define BUFSIZE 10 typedef struct { FILE *file; char buf[BUFSIZE + 1], *lim, *cur, *mar, *tok; int state; } State; typedef enum {END, READY, WAITING, BAD_PACKET, BIG_PACKET} Status; static Status fill(State *st) { const size_t shift = st->tok - st->buf; const size_t used = st->lim - st->tok; const size_t free = BUFSIZE - used; // Error: no space. In real life can reallocate a larger buffer. if (free < 1) return BIG_PACKET; // Shift buffer contents (discard already processed data). memmove(st->buf, st->tok, used); st->lim -= shift; st->cur -= shift; st->mar -= shift; st->tok -= shift; // Fill free space at the end of buffer with new data. const size_t read = fread(st->lim, 1, free, st->file); st->lim += read; st->lim[0] = 0; // append sentinel symbol return READY; } static Status lex(State *st, unsigned int *recv) { char yych; /*!getstate:re2c*/ for (;;) { st->tok = st->cur; /*!re2c re2c:api:style = free-form; re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYCURSOR = "st->cur"; re2c:define:YYMARKER = "st->mar"; re2c:define:YYLIMIT = "st->lim"; re2c:define:YYGETSTATE = "st->state"; re2c:define:YYSETSTATE = "st->state = @@;"; re2c:define:YYFILL = "return WAITING;"; re2c:eof = 0; packet = [a-z]+[;]; * { return BAD_PACKET; } $ { return END; } packet { *recv = *recv + 1; continue; } */ } } void test(const char **packets, Status expect) { // Create a "socket" (open the same file for reading and writing). const char *fname = "pipe"; FILE *fw = fopen(fname, "w"); FILE *fr = fopen(fname, "r"); setvbuf(fw, NULL, _IONBF, 0); setvbuf(fr, NULL, _IONBF, 0); // Initialize lexer state: `state` value is -1, all pointers are at the end // of buffer. State st; st.file = fr; st.cur = st.mar = st.tok = st.lim = st.buf + BUFSIZE; // Sentinel (at YYLIMIT pointer) is set to zero, which triggers YYFILL. st.lim[0] = 0; st.state = -1; // Main loop. The buffer contains incomplete data which appears packet by // packet. When the lexer needs more input it saves its internal state and // returns to the caller which should provide more input and resume lexing. Status status; unsigned int send = 0, recv = 0; for (;;) { status = lex(&st, &recv); if (status == END) { LOG("done: got %u packets\n", recv); break; } else if (status == WAITING) { LOG("waiting...\n"); if (*packets) { LOG("sent packet %u\n", send); fprintf(fw, "%s", *packets++); ++send; } status = fill(&st); LOG("queue: '%s'\n", st.buf); if (status == BIG_PACKET) { LOG("error: packet too big\n"); break; } assert(status == READY); } else { assert(status == BAD_PACKET); LOG("error: ill-formed packet\n"); break; } } // Check results. assert(status == expect); if (status == END) assert(recv == send); // Cleanup: remove input file. fclose(fw); fclose(fr); remove(fname); } int main() { const char *packets1[] = {0}; const char *packets2[] = {"zero;", "one;", "two;", "three;", "four;", 0}; const char *packets3[] = {"zer0;", 0}; const char *packets4[] = {"looooooooooong;", 0}; test(packets1, END); test(packets2, END); test(packets3, BAD_PACKET); test(packets4, BIG_PACKET); return 0; } /*!rules:re2c[:] ... */ %{rules[:] ... %} re2c . . rules use . rules . use /*!use:re2c[:] ... */ %{use[:] ... %} . : rules ( ). use . use : rules (encoding-specific) ( ). use !use:; re2c . . : . * $ ( ) . use re2c ( ). use 2.2 re2c . ( ). 2.2 -r --reusable . 1.2 . !use (Example of a !use directive) // re2c $INPUT -o $OUTPUT #include // This example shows how to combine reusable re2c blocks: two blocks // ('colors' and 'fish') are merged into one. The 'salmon' rule occurs // in both blocks; the 'fish' block takes priority because it is used // earlier. Default rule * occurs in all three blocks; the local (not // inherited) definition takes priority. typedef enum { COLOR, FISH, DUNNO } What; /*!rules:re2c:colors * { assert(false); } "red" | "salmon" | "magenta" { return COLOR; } */ /*!rules:re2c:fish * { assert(false); } "haddock" | "salmon" | "eel" { return FISH; } */ static What lex(const char *s) { const char *YYCURSOR = s, *YYMARKER; /*!re2c re2c:yyfill:enable = 0; re2c:define:YYCTYPE = "unsigned char"; !use:fish; !use:colors; * { return DUNNO; } // overrides inherited '*' rules */ } int main() { assert(lex("salmon") == FISH); assert(lex("what?") == DUNNO); return 0; } /*!use:re2c ... */ (Example of a /*!use:re2c ... */ block) // re2c $INPUT -o $OUTPUT --input-encoding utf8 #include #include // This example supports multiple input encodings: UTF-8 and UTF-32. // Both lexers are generated from the same rules block, and the use // blocks add only encoding-specific configurations. /*!rules:re2c re2c:yyfill:enable = 0; "x y" { return 0; } * { return 1; } */ static int lex_utf8(const uint8_t *s) { const uint8_t *YYCURSOR = s, *YYMARKER; /*!use:re2c re2c:define:YYCTYPE = uint8_t; re2c:encoding:utf8 = 1; */ } static int lex_utf32(const uint32_t *s) { const uint32_t *YYCURSOR = s, *YYMARKER; /*!use:re2c re2c:define:YYCTYPE = uint32_t; re2c:encoding:utf32 = 1; */ } int main() { static const uint8_t s8[] = // UTF-8 { 0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79 }; static const uint32_t s32[] = // UTF32 { 0x00002200, 0x00000078, 0x00000020, 0x00002203, 0x00000079 }; assert(lex_utf8(s8) == 0); assert(lex_utf32(s32) == 0); return 0; } re2c . Tags (tags) @stag #mtag stag mtag . -T --tags re2c:tags = 1 . : ( ) . S-tag m-tag ( ). svars mvars . (leftmost greedy) . Captures (capturing groups) . --captures re2c:captures = 1 . --leftmost-captures () (leftmost greedy) --posix-captures (POSIX longest-match) . : (! ... ). --invert-captures re2c:invert-captures = 1 . yynmatch ( ) yypmatch . yynmatch yypmatch yypmatch [yynmatch * 2] . maxnmatch YYMAXNMATCH yynmatch . Captvars --captvars re2c:captvars = 1 . --captures : yynmatch yypmatch re2c yytl yytr k- ( svars ). : --leftmost-captvars re2c:leftmost-captvars = 1 ( ) --posix-captvars re2c:posix-captvars . (Tagged Deterministic Finite Automata with Lookahead) . TDFA . TDFA DFA . . TDFA (tag variables) : . . stags mtags . (lexer) . YYFILL . S : o s-tag: t = YYCURSOR API C YYSTAGP(t) API (generic) o s-tag: t = NULL API C YYSTAGN(t) API o s-tag : t1 = t2 M : o m-tag: YYMTAGP(t) API o m-tag: YYMTAGN(t) API o m-tag : t1 = t2 S ( ) . M . m-tag ( ) m-tag (prefix-tree) (v, p) v p . s-tag (semantic versions) : major minor patch ( ). YYFILL . // re2c $INPUT -o $OUTPUT #include #include typedef struct { int major, minor, patch; } SemVer; static int s2n(const char *s, const char *e) { // pre-parsed string to number int n = 0; for (; s < e; ++s) n = n * 10 + (*s - '0'); return n; } static int lex(const char *str, SemVer *ver) { const char *YYCURSOR = str, *YYMARKER; // User-defined tag variables that are available in semantic action. const char *t1, *t2, *t3, *t4, *t5; // Autogenerated tag variables used by the lexer to track tag values. /*!stags:re2c format = 'const char *@@;\n'; */ /*!re2c re2c:yyfill:enable = 0; re2c:define:YYCTYPE = "unsigned char"; re2c:tags = 1; num = [0-9]+; @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\x00] { ver->major = s2n(t1, t2); ver->minor = s2n(t3, t4); ver->patch = t5 != NULL ? s2n(t5, YYCURSOR - 1) : 0; return 0; } * { return 1; } */ } int main() { SemVer v; assert(lex("23.34", &v) == 0 && v.major == 23 && v.minor == 34 && v.patch == 0); assert(lex("1.2.999", &v) == 0 && v.major == 1 && v.minor == 2 && v.patch == 999); assert(lex("1.a", &v) == 1); return 0; } s-tag YYFILL . YYFILL . s-tag . ( ) m-tag . // re2c $INPUT -o $OUTPUT --tags #include #include #include #include #include #define BUFSIZE 4095 struct Input { FILE *file; char buf[BUFSIZE + 1], *lim, *cur, *mar, *tok; // Tag variables must be part of the lexer state passed to YYFILL. // They don't correspond to tags and should be autogenerated by re2c. /*!stags:re2c format = 'const char *@@;'; */ bool eof; }; struct SemVer { int major, minor, patch; }; static bool operator==(const SemVer &x, const SemVer &y) { return x.major == y.major && x.minor == y.minor && x.patch == y.patch; } static int s2n(const char *s, const char *e) { // pre-parsed string to number int n = 0; for (; s < e; ++s) n = n * 10 + (*s - '0'); return n; } static int fill(Input &in) { if (in.eof) return 1; const size_t shift = in.tok - in.buf; const size_t used = in.lim - in.tok; // Error: lexeme too long. In real life could reallocate a larger buffer. if (shift < 1) return 2; // Shift buffer contents (discard everything up to the current token). memmove(in.buf, in.tok, used); in.lim -= shift; in.cur -= shift; in.mar -= shift; in.tok -= shift; // Tag variables need to be shifted like other input positions. The check // for non-NULL is only needed if some tags are nested inside of alternative // or repetition, so that they can have NULL value. /*!stags:re2c format = "if (in.@@) in.@@ -= shift;\n"; */ // Fill free space at the end of buffer with new data from file. in.lim += fread(in.lim, 1, BUFSIZE - used, in.file); in.lim[0] = 0; in.eof = in.lim < in.buf + BUFSIZE; return 0; } static bool lex(Input &in, std::vector &vers) { // User-defined local variables that store final tag values. // They are different from tag variables autogenerated with `stags:re2c`, // as they are set at the end of match and used only in semantic actions. const char *t1, *t2, *t3, *t4; for (;;) { in.tok = in.cur; /*!re2c re2c:eof = 0; re2c:api:style = free-form; re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYCURSOR = in.cur; re2c:define:YYMARKER = in.mar; re2c:define:YYLIMIT = in.lim; re2c:define:YYFILL = "fill(in) == 0"; re2c:tags:expression = "in.@@"; num = [0-9]+; num @t1 "." @t2 num @t3 ("." @t4 num)? [\n] { int major = s2n(in.tok, t1); int minor = s2n(t2, t3); int patch = t4 != NULL ? s2n(t4, in.cur - 1) : 0; SemVer ver = {major, minor, patch}; vers.push_back(ver); continue; } $ { return true; } * { return false; } */} }int main() { const char *fname = "input"; const SemVer semver = {1, 22, 333}; std::vector expect(BUFSIZE, semver), actual; // Prepare input file (make sure it exceeds buffer size). FILE *f = fopen(fname, "w"); for (int i = 0; i < BUFSIZE; ++i) fprintf(f, "1.22.333\n"); fclose(f); // Reopen input file for reading. f = fopen(fname, "r"); // Initialize lexer state: all pointers are at the end of buffer. Input in; in.file = f; in.cur = in.mar = in.tok = in.lim = in.buf + BUFSIZE; /*!stags:re2c format = "in.@@ = in.lim;\n"; */ in.eof = false; // Sentinel (at YYLIMIT pointer) is set to zero, which triggers YYFILL. *in.lim = 0; // Run the lexer and check results. assert(lex(in, actual) && expect == actual); // Cleanup: remove input file. fclose(f); remove(fname); return 0; } (capturing groups) . // re2c $INPUT -o $OUTPUT #include #include typedef struct { int major, minor, patch; } SemVer; static int s2n(const char *s, const char *e) { // pre-parsed string to number int n = 0; for (; s < e; ++s) n = n * 10 + (*s - '0'); return n; } static int lex(const char *str, SemVer *ver) { const char *YYCURSOR = str, *YYMARKER; // Final tag variables available in semantic action. /*!svars:re2c format = 'const char *@@;\n'; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = 'const char *@@;\n'; */ /*!re2c re2c:yyfill:enable = 0; re2c:define:YYCTYPE = "unsigned char"; re2c:captvars = 1; num = [0-9]+; (num) "." (num) ("." num)? [\x00] { (void) yytl0; (void) yytr0; // some variables are unused ver->major = s2n(yytl1, yytr1); ver->minor = s2n(yytl2, yytr2); ver->patch = yytl3 ? s2n(yytl3 + 1, yytr3) : 0; return 0; } * { return 1; } */ } int main() { SemVer v; assert(lex("23.34", &v) == 0 && v.major == 23 && v.minor == 34 && v.patch == 0); assert(lex("1.2.999", &v) == 0 && v.major == 1 && v.minor == 2 && v.patch == 999); assert(lex("1.a", &v) == 1); return 0; } m-tag . trie . // re2c $INPUT -o $OUTPUT #include #include #include static const int MTAG_ROOT = -1; // An m-tag tree is a way to store histories with an O(1) copy operation. // Histories naturally form a tree, as they have common start and fork at some // point. The tree is stored as an array of pairs (tag value, link to parent). // An m-tag is represented with a single link in the tree (array index). struct Mtag { const char *elem; // tag value int pred; // index of the predecessor node or root }; typedef std::vector MtagTrie; typedef std::vector Ver; // unbounded number of version components static int s2n(const char *s, const char *e) { // pre-parsed string to number int n = 0; for (; s < e; ++s) n = n * 10 + (*s - '0'); return n; } // Append a single value to an m-tag history. static void add_mtag(MtagTrie &trie, int &mtag, const char *value) { Mtag m = {value, mtag}; mtag = (int)trie.size(); trie.push_back(m); } // Recursively unwind tag histories and collect version components. static void unfold(const MtagTrie &trie, int x, int y, Ver &ver) { // Reached the root of the m-tag tree, stop recursion. if (x == MTAG_ROOT && y == MTAG_ROOT) return; // Unwind history further. unfold(trie, trie[x].pred, trie[y].pred, ver); // Get tag values. Tag histories must have equal length. assert(x != MTAG_ROOT && y != MTAG_ROOT); const char *ex = trie[x].elem, *ey = trie[y].elem; if (ex != NULL && ey != NULL) { // Both tags are valid pointers, extract component. ver.push_back(s2n(ex, ey)); } else { // Both tags are NULL (this corresponds to zero repetitions). assert(ex == NULL && ey == NULL); } } static bool parse(const char *str, Ver &ver) { const char *YYCURSOR = str, *YYMARKER; MtagTrie mt; // User-defined tag variables that are available in semantic action. const char *t1, *t2; int t3, t4; // Autogenerated tag variables used by the lexer to track tag values. /*!stags:re2c format = 'const char *@@ = NULL;'; */ /*!mtags:re2c format = 'int @@ = MTAG_ROOT;'; */ /*!re2c re2c:api:style = free-form; re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYSTAGP = "@@ = YYCURSOR;"; re2c:define:YYSTAGN = "@@ = NULL;"; re2c:define:YYMTAGP = "add_mtag(mt, @@, YYCURSOR);"; re2c:define:YYMTAGN = "add_mtag(mt, @@, NULL);"; re2c:yyfill:enable = 0; re2c:tags = 1; num = [0-9]+; @t1 num @t2 ("." #t3 num #t4)* [\x00] { ver.clear(); ver.push_back(s2n(t1, t2)); unfold(mt, t3, t4, ver); return true; } * { return false; } */ } int main() { Ver v; assert(parse("1", v) && v == Ver({1})); assert(parse("1.2.3.4.5.6.7", v) && v == Ver({1, 2, 3, 4, 5, 6, 7})); assert(!parse("1.2.", v)); return 0; } (code points) (code units) . . . . (fixed-length) . (variable-length) . "any" ( ) [^] ( * ). : yych YYCTYPE . . re2c . : . : o ASCII ( ). [0-255] . o EBCDIC ( --ebcdic re2c:encoding:ebcdic ). [0-255] . o UCS2 ( --ucs2 re2c:encoding:ucs2 ). [0-0xFFFF] . o UTF8 ( --utf8 re2c:encoding:utf8 ). . . . o UTF16 ( --utf16 re2c:encoding:utf16 ). . . . o UTF32 ( --utf32 re2c:encoding:utf32 ). [0-0x10FFFF] . include/unicode_blocks.re include/unicode_categories.re include/unicode_properties.re re2c Unicode . --input-encoding (literals) . --input-encoding utf8 re2c UTF8 ( --utf8 ). --encoding-policy re2c (Unicode surrogates [0xD800-0xDFFF]) . UTF8 . // re2c $INPUT -o $OUTPUT -8 --case-ranges -i #include #include /*!include:re2c "unicode_categories.re" */ static int lex(const char *s) { const char *YYCURSOR = s, *YYMARKER; /*!re2c re2c:define:YYCTYPE = "unsigned char"; re2c:yyfill:enable = 0; // Simplified "Unicode Identifier and Pattern Syntax" // (see https://unicode.org/reports/tr31) id_start = L | Nl | [$_]; id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3]; identifier = id_start id_continue*; identifier { return 0; } * { return 1; } */ } int main() { assert(lex("_") == 0); return 0; } re2c /*!include:re2c FILE */ %{include FILE %} !include FILE ; FILE . re2c include ( -I ) . / include re2c #include C/C++ : FILE / . . --depfile . re2c include include/ . ( Unicode) re2c . include . (definitions.h) typedef enum { OK, FAIL } Result; /*!re2c number = [1-9][0-9]*; */ (extra_rules.re.inc) // floating-point numbers frac = [0-9]* "." [0-9]+ | [0-9]+ "."; exp = 'e' [+-]? [0-9]+; float = frac exp? | [0-9]+ exp; float { return OK; } (Input file) // re2c $INPUT -o $OUTPUT -i #include /*!include:re2c "definitions.h" */ Result lex(const char *s) { const char *YYCURSOR = s, *YYMARKER; /*!re2c re2c:define:YYCTYPE = "unsigned char"; re2c:yyfill:enable = 0; * { return FAIL; } number { return OK; } !include "extra_rules.re.inc"; */ } int main() { assert(lex("123") == OK); assert(lex("123.4567") == OK); return 0; } re2c --header re2c:header /*!header:re2c:on*/ /*!header:re2c:off*/ %{header:on%} %{header:off%} .re . . re2c --header re2c:header ( stdout) . re2c (translation units) . ( ). (Input file) // re2c $INPUT -o $OUTPUT -i --header lexer/state.h #include #include #include "lexer/state.h" // the header is generated by re2c /*!header:re2c:on*/ typedef struct { const char *str, *cur; /*!stags:re2c format = "const char *@@;"; */ } LexerState; /*!header:re2c:off*/ long lex(LexerState* st) { const char *t; /*!re2c re2c:header = "lexer/state.h"; re2c:yyfill:enable = 0; re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYCURSOR = "st->cur"; re2c:tags = 1; re2c:tags:expression = "st->@@"; [a]* @t [b]* { return t - st->str; } */ } int main() { const char *s = "ab"; LexerState st = { s, s /*!stags:re2c format = ", NULL"; */ }; assert(lex(&st) == 1); return 0; } (Header file) /* Generated by re2c */ typedef struct { const char *str, *cur; const char *yyt1; } LexerState; (Skeleton programs) -S, --skeleton re2c re2c C . . DFA ( ) re2c : .input DFA .keys . .input . : o re2c ( DFA ). o (benchmarking) . o (minimized test cases) ( ). ( ). re2c DFA . : DFA . ( ASCII UTF-8 EBCDIC) DFA . DFA : re2c DFA . . re2c ( ) . (re2c )..SS " (Visualization and debug)" -D, --emit-dot re2c DFA DOT ( ). Graphviz DFA . DFA . --dump-nfa --dump-dfa-raw ( ). (SEE ALSO) re2c : http://re2c.org. flex(1) lex(1) quex(http://quex.sourceforge.net). (AUTHORS) re2c Peter Bumbulis (peter@csg.uwaterloo.ca) . Marcus Boerger Dan Nuffer . : Brian Young (bayoung@acm.org) Marcus Boerger Dan Nuffer (nuffer@users.sourceforge.net) Ulya Trofimovich (skvadrik@gmail.com) Serghei Iakovlev Sergei Trofimovich Petr Skocik ligfx raekye PolarGoose. RE2C(1)