RE2JAVA(1) RE2JAVA(1) (NAME) re2java - Java (SYNOPSIS) re2java [ OPTIONS ] [ WARNINGS ] INPUT - stdin . (INTRODUCTION) re2java . ( Java ) / . . re2java . Java . : // re2java $INPUT -o $OUTPUT class Main { static boolean lex(String yyinput) { int yycursor = 0; /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:yyfill:enable = 0; [1-9][0-9]* { return true; } * { return false; } */ } public static void main(String []args) { assert lex("1234\0"); } }; re2java : // Generated by re2java // re2java $INPUT -o $OUTPUT class Main { static boolean lex(String yyinput) { int yycursor = 0; { char yych = 0; int yystate = 0; yyl: while (true) { switch (yystate) { case 0: yych = yyinput.charAt(yycursor); yycursor += 1; switch (yych) { case 0x31: case 0x32: case 0x33: case 0x34: case 0x35: case 0x36: case 0x37: case 0x38: case 0x39: yystate = 2; continue yyl; default: yystate = 1; continue yyl; } case 1: { return false; } case 2: yych = yyinput.charAt(yycursor); switch (yych) { case 0x30: case 0x31: case 0x32: case 0x33: case 0x34: case 0x35: case 0x36: case 0x37: case 0x38: case 0x39: yycursor += 1; yystate = 2; continue yyl; default: yystate = 3; continue yyl; } case 3: { return true; } default: throw new IllegalStateException("internal lexer error"); } } } } public static void main(String []args) { assert lex("1234\0"); } }; re2java (blocks) . : name = regular-expression ; name regular-expression . - . re2java . name . ( ) . (global scope) . . configuration = value ; re2java . re2java . value - . . . regular-expression code regular-expression ( := ) . regular-expression code . . . : * $. (code unit) ( ). . . !action code (code) ( ). . !directive ; . . . (Blocks) /*!re2c */ %{ %} ( ). 2.2 . : /*!re2c[:] ... */ or %{[:] ... %} . re2java (DFA) . . . /*!local:re2c[:] ... */ or %{local[:] ... %} ( ). /*!rules:re2c[:] ... */ or %{rules[:] ... %} -- . ( ). 2.2 re2java -r --reusable . /*!use:re2c[:] ... */ or %{use[:] ... %} use . re2java . ( ). use . 2.2 re2java use -r --reusable . /*!max:re2c[:[:...]] ... */ or %{max[:[:...]] ... %} YYMAXFILL . YYMAXFILL ( ). C (#define YYMAXFILL ) Go (var YYMAXFILL int = ) . format @@{max} ( @@) YYMAXFILL . /*!maxnmatch:re2c[:[:...]] ... */ or %{maxnmatch[:[:...]] ... %} YYMAXNMATCH ( -P --posix-captures ). YYMAXNMATCH ( ). C (#define YYMAXNMATCH ) Go (var YYMAXNMATCH int = ) . format @@{max} ( @@) YYMAXNMATCH . /*!stags:re2c[:[:...]] ... */, /*!mtags:re2c[:[:...]] ... */ or %{stags[:[:...]] ... %}, %{mtags[:[:...]] ... %{ s-tag/m-tag re2java . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!svars:re2c[:[:...]] ... */, /*!mvars:re2c[:[:...]] ... */ or %{svars[:[:...]] ... %}, %{mvars[:[:...]] ... %{ s-tag/m-tag ( --tags) re2java ( --captvars --posix-captvars) . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!getstate:re2c[:[:...]] ... */ or %{getstate[:[:...]] ... %} (conditional dispatch) ( --storable-state ). . . . --loop-switch goto . /*!conditions:re2c[:[:...]] ... */, /*!types:re2c... */ or %{conditions[:[:...]] ... %}, %{types... %} / (condition enumeration) ( --conditions ). ( ). YYCONDTYPE . format separator . format @@{cond} ( @@) @@{num} . separator format . /*!include:re2c */ or %{include %} () . #include C/C++. --depfile . /*!header:re2c:on*/ or %{header:on %} . header:off re2java -t --type-header . /*!header:re2c:off*/ or %{header:off %} header:on*/ . /*!ignore:re2c ... */ or %{ignore ... %} . re2java : re2c:api, re2c:input --api. re2c:api:sigil ("sigil") API . @@ . . $ ${name} . API . API re2c:YYFILL@len YYFILL. re2c:api:style API . functions ( C) free-form ( Go Rust). functions API . ( YYFILL) C/C++ (closures) Go Rust . free-form API : @@{var} @@ ( ). API re2c:YYFILL:naked YYFILL . re2c:bit-vectors, re2c:flags:bit-vectors, re2c:flags:b --bit-vectors . re2c:captures, re2c:leftmost-captures --leftmost-captures . re2c:captvars, re2c:leftmost-captvars --leftmost-captvars . re2c:case-insensitive, re2c:flags:case-insensitive --case-insensitive . re2c:case-inverted, re2c:flags:case-inverted --case-inverted . re2c:case-ranges, re2c:flags:case-ranges --case-ranges . re2c:computed-gotos, re2c:flags:computed-gotos, re2c:flags:g --computed-gotos . re2c:computed-gotos:relative, re2c:cgoto:relative --computed-gotos-relative . re2c:computed-gotos:threshold, re2c:cgoto:threshold goto if . 9 . re2c:cond:abort (default case) (abort) . re2c:cond:goto :=> . goto @@; . @@ ( re2c:api:sigil re2c:cond:goto@cond ). re2c:cond:goto@cond re2c:cond:goto . @@ . re2c:api:sigil . re2c:cond:divider . /* *********************************** */ . (re2c:api;sigil re2c:cond:divider@cond ). re2c:cond:divider@cond re2c:cond:divider . @@ . re2c:api:sigil . re2c:cond:prefix, re2c:condprefix . yyc_ . re2c:cond:enumprefix, re2c:condenumprefix . yyc . re2c:debug-output, re2c:flags:debug-output, re2c:flags:d --debug-output . re2c:empty-class, re2c:flags:empty-class --empty-class . re2c:encoding:ebcdic, re2c:flags:ecb, re2c:flags:e --ebcdic . re2c:encoding:ucs2, re2c:flags:wide-chars, re2c:flags:w --ucs2 . re2c:encoding:utf8, re2c:flags:utf-8, re2c:flags:8 --utf8 . re2c:encoding:utf16, re2c:flags:utf-16, re2c:flags:x --utf16 . re2c:encoding:utf32, re2c:flags:unicode, re2c:flags:u --utf32 . re2c:encoding-policy, re2c:flags:encoding-policy --encoding-policy . re2c:eof (sentinel) $ . -1 ( $ ). . () . re2c:header, re2c:flags:type-header, re2c:flags:t . --header . re2c:indent:string . "\t" . . . re2c:indent:top . . . re2c:invert-captures --invert-captures . re2c:label:prefix, re2c:labelprefix DFA . yy . re2c:label:start, re2c:startlabel . . . . ( ). re2c:label:yyFillLabel YYFILL re2c:eof . re2c:label:yyloop --loop-switch . yyloop . re2c:label:yyNext switch YYGETSTATE ( re2c:state:nextlabel) . yyNext . re2c:lookahead, re2c:flags:lookahead ( --no-lookahead ). re2c:monadic ( Haskell )..TP re2c:nested-ifs, re2c:flags:nested-ifs, re2c:flags:s --nested-ifs . re2c:posix-captures, re2c:flags:posix-captures, re2c:flags:P --posix-captures . re2c:posix-captvars --posix-captvars . re2c:tags, re2c:flags:tags, re2c:flags:T --tags . re2c:tags:expression . re2java yyt . struct . @@{tag} @@ . re2c:tags:expression = "s.@@"; s.yyt . re2c:api:sigil . re2c:tags:negative ( -1 (null pointer) ). re2c:tags:prefix . yyt . re2c:sentinel (sentinel) ( re2c:yyfill:enable = 0; re2c:eof ). : (rule) . -1` ( re2java ). . re2c:state:abort (case) (state dispatch) (abort) -1 . re2c:state:nextlabel (switch) YYGETSTATE yyNext ( ). re2c:label:start getstate switch YYGETSTATE . re2c:unsafe, re2c:flags:unsafe --no-unsafe . unsafe YYPEEK . ( ). Rust . re2c:YYBACKUP, re2c:define:YYBACKUP API YYBACKUP . re2c:YYBACKUPCTX, re2c:define:YYBACKUPCTX API YYBACKUPCTX . re2c:YYCONDTYPE, re2c:define:YYCONDTYPE API YYCONDTYPE . re2c:YYCTYPE, re2c:define:YYCTYPE API YYCTYPE . re2c:YYCTXMARKER, re2c:define:YYCTXMARKER API YYCTXMARKER . re2c:YYCURSOR, re2c:define:YYCURSOR API YYCURSOR . re2c:YYDEBUG, re2c:define:YYDEBUG API YYDEBUG . re2c:YYFILL, re2c:define:YYFILL API YYFILL . re2c:YYFILL@len, re2c:define:YYFILL@len (sigil) YYFILL . @@ . re2c:api:sigil . re2c:YYFILL:naked, re2c:define:YYFILL:naked re2c:api:style YYFILL . (free-form) API . re2c:YYFN API YYFN . re2c:YYINPUT API YYINPUT . re2c:YYGETCOND, re2c:define:YYGETCONDITION API YYGETCOND . re2c:YYGETCOND:naked, re2c:define:YYGETCONDITION:naked re2c:api:style YYGETCOND . (free-form) API . re2c:YYGETSTATE, re2c:define:YYGETSTATE API YYGETSTATE . re2c:YYGETSTATE:naked, re2c:define:YYGETSTATE:naked re2c:api:style YYGETSTATE . (free-form) API . re2c:YYGETACCEPT, re2c:define:YYGETACCEPT API YYGETACCEPT . re2c:YYLESSTHAN, re2c:define:YYLESSTHAN API YYLESSTHAN . re2c:YYLIMIT, re2c:define:YYLIMIT API YYLIMIT . re2c:YYMARKER, re2c:define:YYMARKER API YYMARKER . re2c:YYMTAGN, re2c:define:YYMTAGN API YYMTAGN . re2c:YYMTAGP, re2c:define:YYMTAGP API YYMTAGP . re2c:YYPEEK, re2c:define:YYPEEK API YYPEEK . re2c:YYRESTORE, re2c:define:YYRESTORE API YYRESTORE . re2c:YYRESTORECTX, re2c:define:YYRESTORECTX API YYRESTORECTX . re2c:YYRESTORETAG, re2c:define:YYRESTORETAG API YYRESTORETAG . re2c:YYSETCOND, re2c:define:YYSETCONDITION API YYSETCOND . re2c:YYSETCOND@cond, re2c:define:YYSETCONDITION@cond (sigil) YYSETCOND . @@ . re2c:api:sigil . re2c:YYSETCOND:naked, re2c:define:YYSETCONDITION:naked re2c:api:style YYSETCOND . (free-form) API . re2c:YYSETSTATE, re2c:define:YYSETSTATE API YYSETSTATE . re2c:YYSETSTATE@state, re2c:define:YYSETSTATE@state (sigil) YYSETSTATE . @@ . re2c:api:sigil . re2c:YYSETSTATE:naked, re2c:define:YYSETSTATE:naked re2c:api:style YYSETSTATE . (free-form) API . re2c:YYSETACCEPT, re2c:define:YYSETACCEPT API YYSETACCEPT . re2c:YYSKIP, re2c:define:YYSKIP API YYSKIP . re2c:YYSHIFT, re2c:define:YYSHIFT API YYSHIFT . re2c:YYCOPYMTAG, re2c:define:YYCOPYMTAG API YYCOPYMTAG . re2c:YYCOPYSTAG, re2c:define:YYCOPYSTAG API YYCOPYSTAG ..TP re2c:YYSHIFTMTAG, re2c:define:YYSHIFTMTAG API YYSHIFTMTAG . re2c:YYSHIFTSTAG, re2c:define:YYSHIFTSTAG API YYSHIFTSTAG . re2c:YYSTAGN, re2c:define:YYSTAGN API YYSTAGN . re2c:YYSTAGP, re2c:define:YYSTAGP API YYSTAGP . re2c:yyaccept, re2c:variable:yyaccept API yyaccept . re2c:yybm, re2c:variable:yybm API yybm . re2c:yybm:hex, re2c:variable:yybm:hex --bit-vectors () . ( ). re2c:yych, re2c:variable:yych API yych . re2c:yych:emit, re2c:variable:yych:emit yych . . re2c:yych:conversion, re2c:variable:yych:conversion re2java yych YYCTYPE . ( ). re2c:yych:literals, re2c:variable:yych:literals yych . : char ( ) hex ( ) char_or_hex ( ). re2c:yyctable, re2c:variable:yyctable API yyctable . re2c:yynmatch, re2c:variable:yynmatch API yynmatch . re2c:yypmatch, re2c:variable:yypmatch API yypmatch . re2c:yytarget, re2c:variable:yytarget API yytarget . re2c:yystable, re2c:variable:yystable . re2c:yystate, re2c:variable:yystate API yystate . re2c:yyfill, re2c:variable:yyfill API yyfill . re2c:yyfill:check YYFILL ( YYLESSTHAN generic API YYLIMIT C pointer API) . ( ). re2c:yyfill:enable YYFILL ( ) . ( ) YYFILL ( ). ( -W) re2c:sentinel . (YYFILL ). re2c:yyfill:parameter YYFILL . . ( ). re2c:YYFILL:naked re2c:api:style . re2c:yyfn:sep YYFN ( ). re2c:yyfn:throw YYFN ( ). re2java : "foo" . 'foo' . [a-xyz], [^a-xyz] ( ). . . R \ S R S. R* R. R+ R. R? R. R{n} R n . R{n,} R n . R{n,m} R n m . (R) R . (submatch extraction) (R) --invert-captures . (!R) (!R) --invert-captures . R S : R S. R | S : R S. R / S (Lookahead): R S S . name name ( "name" Flex). {name} name Flex. @stag s-tag: @stag stag . #mtag m-tag: #mtag mtag . $ . : \a \b \f \n \r \t \v \\ \ooo \xhh \uhhhh \Uhhhhhhhh. (Actions) re2java : !entry code (Entry action) code . (start conditions) . (lexeme). !pre_rule code (Pre-rule action) code ( (start conditions)) . ( ). !post_rule code (Post-rule action) code ( (start conditions)) . (trap statements) . re2java : !use:name ; use name . . : . * $ . use use . !include file ; include : file . (Program interface) (primitives) API . . . re2java ( ) . API . re2java API : Simple API API . : YYINPUT ( ) YYCURSOR YYMARKER YYCTXMARKER YYLIMIT ( YYINPUT ). Record API Record API . --api record re2c:api = record . API yyrecord ( re2c:yyrecord ) yyinput yycursor yymarker yyctxmarker yylimit ( ). Generic API API . --api generic re2c:api = generic . : YYPEEK YYSKIP YYBACKUP YYBACKUPCTX YYSTAGP YYSTAGN YYMTAGP YYMTAGN YYRESTORE YYRESTORECTX YYRESTORETAG YYSHIFT YYSHIFTSTAG YYSHIFTMTAG YYLESSTHAN YYEND. API : YYCTYPE ( ). ASCII EBCDIC UTF-8 . UTF-16 UCS-2 . UTF-32 . YYCURSOR (l-value) ( YYINPUT). YYCURSOR . YYCURSOR . YYLIMIT (r-value) ( YYINPUT). YYLIMIT . . YYCURSOR YYLIMIT . YYMARKER (l-value) ( YYINPUT). YYCURSOR . . YYCTXMARKER (l-value) (trailing context) ( YYINPUT). . YYCTXMARKER / . YYFILL Generic API len. YYFILL len . re2c:eof len 1 YYFILL . re2c:eof YYFILL . len YYMAXFILL . YYFN --recursive-functions . re2c:fn:sep ( ) . ( void ). : ( ) ( - ). YYINPUT (r-value) ( ). YYMAXFILL YYFILL. max . YYLESSTHAN Generic API len. (r-value) len true . YYEND Generic API . (r-value) ( ) true . YYEND $ . YYLESSTHAN . YYPEEK Generic API . (r-value) YYCTYPE . YYSKIP Generic API . YYBACKUP Generic API ( YYRESTORE ). YYRESTORE Generic API YYBACKUP . YYBACKUPCTX Generic API ( YYRESTORECTX ). YYRESTORECTX Generic API YYBACKUPCTX ..TP YYRESTORETAG API tag (trailing context) tag . YYSTAGP API tag tag YYINPUT ( ). YYSTAGP tag . YYSTAGN API tag tag YYINPUT ( ). YYSTAGN tag . YYMTAGP API tag. YYMTAGP tag ( ). YYMTAGN API tag. YYMTAGN tag ( ). YYSHIFT API shift shift ( ). YYCOPYSTAG API lhs rhs s-tag rhs s-tag lhs . lhs rhs . YYCOPYMTAG API lhs rhs m-tag rhs m-tag lhs . lhs rhs . YYSHIFTSTAG API tag shift tag shift (code unit) ( ). YYSHIFTMTAG API tag shift tag shift ( ). YYMAXNMATCH POSIX . maxnmatch . YYCONDTYPE (condition enum). conditions --header . YYGETACCEPT var . var . YYSETACCEPT : var ( l-value ) val ( ). var val . YYGETCOND r-value YYCONDTYPE . YYSETCOND cond cond . YYGETSTATE r-value . -1 . YYSETSTATE state state . YYDEBUG -d --debug-output . () ( YYDEBUG ). YYDEBUG : state ( DFA -1) symbol ( ). yyaccept l-value . --storable-state . yybm ( --bitmaps). . . k- n- (true) n- k- . if switch . yych l-value YYCTYPE . -f --storable-state . yyctable ( --conditions --computed-gotos ). yyfill l-value YYFILL ( YYFILL ). yynmatch l-value POSIX . -P --posix-captures . yypmatch l-value . yynmatch * 2 ( YYMAXNMATCH * 2 ). -P --posix-captures . yystable . yystate l-value --loop-switch DFA . yytarget ( ) . . yytarget --computed-gotos . re2c . re2c . re2c . -? --help -h . --api API . simple . record (struct class ) . generic API . --bit-vectors -b . --nested-ifs . --captures, --leftmost-captures . yybmatch 2 * YYMAXNMATCH yynmatch . --captvars, --leftmost-captvars . k- yytl yytr . --case-insensitive . --case-inverted : . --case-ranges switch low ... high. C/C++ GCC Clang Tcc . Tcc . --computed-gotos -g "computed goto" ( ). re2java . cgoto:threshold . cgoto:relative . --bit-vectors . --computed-gotos-relative --computed-gotos ( ). --computed-gotos . --conditions --start-conditions -c "conditions" Flex: . re2java goto . --depfile FILE FILE Makefile : [include-file ...]. / include include . --output . --ebcdic --ecb -e EBCDIC . re2java 0 -- 0xFF . --empty-class re2java . match-empty () ( ). match-none . error . --encoding-policy re2java (Unicode surrogates). fail re2java . substitute re2java 0xFFFD . ignore () re2java . . --flex-syntax -F Flex: . . --goto-label "goto/label": DFA goto . goto . --header --type-header -t HEADER HEADER. header:on header:off . (condition enum) ( conditions ). -I PATH PATH include . include . re2java FILE include -I . --input --api. default simple ( ). custom generic . --input-encoding re2java. ascii () re2java ASCII : . utf8 re2java UTF8 . --invert-captures (capturing) . (...) (! ...) . (! ...) (...) . --lang . C D Go Haskell Java JS OCaml Python Rust Swift V Zig ( --syntax ). none . --location-format . gnu 'filename:line:column: ...' . msvc 'filename(line,column) ...' . gnu . --loop-switch "loop/switch": DFA switch switch . yystate . yystate . --nested-ifs -s if switch . . --no-debug-info -i (line directives). ( ). --no-generation-date . --no-version . --no-unsafe unsafe YYPEEK ( Rust ). YYPEEK . YYPEEK release --no-unsafe unsafe . --output -o OUTPUT OUTPUT. --posix-captures, -P POSIX. yybmatch 2 * YYMAXNMATCH yynmatch ..TP --posix-captvars POSIX. k- yytl yytr . --recursive-functions DFA . --reusable -r 2.2 ( ). --skeleton -S <<>> ("skeleton"). <<>> . . C . --storable-state -f . push . YYGETSTATE YYSETSTATE yych yyaccept state . --syntax FILE FILE . FILE ( ) ( --lang none ). --tags -T . --ucs2 --wide-chars -w UCS2 . re2java 0 -- 0xFFFF . --nested-ifs . --utf8 --utf-8 -8 UTF-8 . re2java 0 -- 0x10FFFF . --utf16 --utf-16 -x UTF16 . re2java 0 -- 0x10FFFF . --nested-ifs . --utf32 --unicode -u UTF32 . re2java 0 -- 0x10FFFF . --nested-ifs . --verbose . --vernum -V MMmmpp ( ). --version -v . --single-pass -1 . ( ). --debug-output -d YYDEBUG . . --dump-adfa : DFA ( .dot). --dump-cfg : ( .dot). --dump-closure-stats : (closure). --dump-dfa-det : DFA ( .dot). --dump-dfa-min : DFA ( .dot). --dump-dfa-tagopt : DFA ( .dot). --dump-dfa-tree : DFA ( .dot). --dump-dfa-raw : DFA ( .dot). --dump-interf : . --dump-nfa : NFA ( .dot). --emit-dot -D .dot . Graphviz ( dot -Tpng -odfa.png dfa.dot). --dfa-minimization : DFA re2java. moore ( ). table << >> ("table filling") . DFA . --eager-skip : -- . (lazy) -- . --no-lookahead . TDFA(0) . TDFA(1) TDFA(0) . . TDFA(0) TDFA(1) . --no-optimize-tags : ( ). --posix-closure : POSIX: gor1 () Goldberg-Radzik gtop << >> ("global topological order") . --posix-prectable : POSIX. complex TNFA . naive TNFA complex . --stadfa . staDFA TDFA . staDFA TDFA . --fixed-tags : (all) (none) (toplevel) . all . <<>> ("Fixed") ( <<>> "base") . . ( ). . . -W . -Werror . . -W warning. -Wno- warning. -Werror- warning ( -W ). -Wno-error- warning . . -Wcondition-order . --header conditions . -Wempty-character-class . : . re2java . --empty-class . -Wmatch-empty-string ( ) . . -Wswapped-range . . -Wundefined-control-flow ( ) . . * . -Wunreachable-rules . -Wdeprecated-eof_rule $ . $ . -Wuseless-escape (escape ) . re2java . -Wnondeterministic-tags n- n . -Wsentinel-in-midrule (sentinel) --- . . re2c:sentinel . -Wundefined-syntax-config (syntax file) --syntax . : . . re2c . -- . re2c re2c: : . ( include/syntax ) re2java . ( --lang none ). : . (Feature lists) re2java : supported_apis API simple record generic. supported_api_styles API functions free-form. supported_code_models goto-label loop-switch recursive-functions. supported_targets code dot skeleton. supported_features nested-ifs bitmaps computed-gotos case-ranges monadic unsafe tags captures captvars. (Language configurations) re2java : semicolons . backtick_quoted_strings . single_quoted_strings . indentation_sensitive . wrap_blocks_in_braces . (Inplace configurations) re2c: . . (Code templates) . : code-template :: name '=' code-exprs ';' | CODE_TEMPLATE ';' | '' ';' code-exprs :: | code-exprs code-expr code-expr :: STRING | VARIABLE | optional | list optional :: '(' CONDITIONAL '?' code-exprs ')' | '(' CONDITIONAL '?' code-exprs ':' code-exprs ')' list :: '[' VARIABLE ':' code-exprs ']' | '[' VARIABLE '{' NUMBER '}' ':' code-exprs ']' | '[' VARIABLE '{' NUMBER ',' NUMBER '}' ':' code-exprs ']' . . : re2java . ( -1 ). . ( ) ( ) . re2java . if-then-else C : code:if_then_else = [branch{0}: topindent "if " cond " {" nl indent [stmt: stmt] dedent] [branch{1:-1}: topindent "} else" (.cond ? " if " cond) " {" nl indent [stmt: stmt] dedent] topindent "}" nl; branch : branch{0} ( else ) branch{1:-1} ( ) . stmt : [stmt: stmt] . topindent indent dedent nl .cond ( ). : if x { // do something } else if y { // do something else } else { // don't do anything } re2java . . . code:var_local . : type ( ) name ( ) init ( ). : .init ( ). code:var_global code:var_local (top-level) . code:const_local . : type ( ) name ( ) init ( ). code:const_global code:const_local (top-level) . code:array_local (). : type ( ) name ( ) size ( ) row ( ) elem ( -- row ). : .const ( ). code:array_global code:array_local (top-level) . code:array_elem (). : array ( ) index ( ). code:enum (enumeration) ( ). type ( ) elem ( ) init ( ). : .init ( ). code:enum_elem ( code:enum). name ( ) type ( ). code:assign . lhs ( ) rhs ( ). code:type_int (signed integer). code:type_uint (unsigned integer). code:type_yybm yybm. code:type_yytarget yytarget. code:type_yyctable yyctable. code:cmp_eq "". code:cmp_ne "". code:cmp_lt " ". code:cmp_gt " ". code:cmp_le " ". code:cmp_ge " ". code:if_then_else if-then-else . : branch ( ) cond ( ) stmt ( ). : .cond ( ) .many ( ). code:if_then_else_oneline code:if_then_else . code:if_then_else . code:switch switch (case). : expr ( switch ) case ( case ). code:switch_cases case switch . case ( case ) stmt ( ). code:switch_cases_oneline code:switch_cases . code:switch_cases . code:switch_case_range case switch ( ). : val ( ). : .many ( ) .char_literals ( switch -- ). code:switch_case_default (default) switch. code:loop ( ). : label ( ) stmt ( ). code:continue continue. : label ( ). code:goto goto. : label ( ). code:cgoto goto (computed goto). : array ( goto ) index ( ) base ( .cgoto.relative ). code:cgoto:data goto . : label ( ) type ( ) base ( - .cgoto.relative ). code:fndecl . : name ( ) type ( ) throw ( re2c:yyfn:throw ) arg ( ) argname ( ) argtype ( ). : .type ( (non-void) ). code:fndef code:fndecl stmt . code:fncall . : name ( ) retval ( l-value ) arg ( ). : .args ( ) .retval ( ). code:tailcall (tail call). : name ( ) arg ( ). : .args ( ) .retval ( (non-void) ). code:recursive_functions --recursive-functions. : fn ( ) fndecl ( ) fndef ( )..TP code:fingerprint . : ver ( re2java ) date ( ). code:line_info ( ). : line ( ) file ( ). code:abort (abort) . code:yydebug YYDEBUG API . : YYDEBUG yyrecord yych ( re2c:) state ( DFA). code:yypeek YYPEEK API . : YYPEEK YYCTYPE YYINPUT YYCURSOR yyrecord yych ( re2c:). : .cast ( re2c:yych:conversion ). code:yyskip YYSKIP API . : YYSKIP YYCURSOR yyrecord ( re2c:). code:yybackup YYBACKUP API . : YYBACKUP YYCURSOR YYMARKER yyrecord ( re2c:). code:yybackupctx YYBACKUPCTX API . : YYBACKUPCTX YYCURSOR YYCTXMARKER yyrecord ( re2c:). code:yyskip_yypeek code:yyskip code:yypeek ( code:yyskip code:yypeek). code:yypeek_yyskip code:yypeek code:yyskip ( code:yypeek code:yyskip). code:yyskip_yybackup code:yyskip code:yybackup ( code:yyskip code:yybackup). code:yybackup_yyskip code:yybackup code:yyskip ( code:yybackup code:yyskip). code:yybackup_yypeek code:yybackup code:yypeek ( code:yybackup code:yypeek). code:yyskip_yybackup_yypeek code:yyskip code:yybackup code:yypeek ( ``code:yyskip`` code:yybackup code:yypeek). code:yybackup_yypeek_yyskip code:yybackup code:yypeek code:yyskip ( ``code:yybackup`` code:yypeek code:yyskip). code:yyrestore YYRESTORE API . : YYRESTORE YYCURSOR YYMARKER yyrecord ( re2c:). code:yyrestorectx YYRESTORECTX API . : YYRESTORECTX YYCURSOR YYCTXMARKER yyrecord ( re2c:). code:yyrestoretag YYRESTORETAG API . : YYRESTORETAG YYCURSOR yyrecord ( re2c:) tag ( ). code:yyshift YYSHIFT API . : YYSHIFT YYCURSOR yyrecord ( re2c:) offset ( ). code:yyshiftstag YYSHIFTSTAG API . : YYSHIFTSTAG yyrecord negative ( re2c:) tag ( ) offset ( ). : .nested ( -- re2c:tags:negative ). code:yyshiftmtag YYSHIFTMTAG API . : YYSHIFTMTAG ( re2c:) tag ( ) offset ( ). code:yystagp YYSTAGP API . : YYSTAGP YYCURSOR yyrecord ( re2c:) tag ( ). code:yymtagp YYMTAGP API . : YYMTAGP ( re2c:) tag ( ). code:yystagn YYSTAGN API . : YYSTAGN negative yyrecord ( re2c:) tag ( ). code:yymtagn YYMTAGN API . : YYMTAGN ( re2c:) tag ( ). code:yycopystag YYCOPYSTAG API . : YYCOPYSTAG yyrecord ( re2c:) lhs rhs ( ). code:yycopymtag YYCOPYMTAG API . : YYCOPYMTAG yyrecord ( re2c:) lhs rhs ( ). code:yygetaccept YYGETACCEPT API . : YYGETACCEPT yyrecord ( re2c:) var ( re2c:yyaccept). code:yysetaccept YYSETACCEPT API . : YYSETACCEPT yyrecord ( re2c:) var ( re2c:yyaccept) val ( ). code:yygetcond YYGETCOND API . : YYGETCOND yyrecord ( re2c:) var ( re2c:yycond). code:yysetcond YYSETCOND API . : YYSETCOND yyrecord ( re2c:) var ( re2c:yycond) val ( ). code:yygetstate YYGETSTATE API . : YYGETSTATE yyrecord ( re2c:) var ( re2c:yystate). code:yysetstate YYSETSTATE API . : YYSETSTATE yyrecord ( re2c:) var ( re2c:yystate) val ( ). code:yylessthan YYLESSTHAN API . : YYLESSTHAN YYCURSOR YYLIMIT yyrecord ( re2c:) need ( ). : .many ( need )..TP code:yyend YYEND API . : YYEND YYCURSOR YYLIMIT. code:yybm_filter yych yybm ( --bitmaps ). : yych ( re2c:yych ). code:yybm_match yybm ( --bitmaps). : yybm yych ( re2c: ) offset ( yybm yych ) mask ( ). code:yytarget_filter yych yytarget ( --computed-gotos ). : yych ( re2c:yych ). : nl (newline). indent . dedent . topindent . . : .api.simple (True) API (--api simple re2c:api = simple). .api.generic (True) API (generic) (--api generic re2c:api = generic). .api.record (True) API (--api record re2c:api = record). .api_style.functions (True) API (re2c:api-style = functions). .api_style.freeform (True) API (re2c:api-style = free-form). .case_ranges (True) case (--case-ranges re2c:case-ranges = 1). .cgoto.relative (True) goto (--computed-gotos-relative re2c:cgoto:relative = 1). .code_model.goto_label (True) goto/label (--goto-label). .code_model.loop_switch (True) loop/switch (--loop-switch). .code_model.recursive_functions (True) (--recursive-function). .date (True) (fingerprint) . .loop_label (True) re2java (re2c:label:yyloop ). .monadic (True) (re2c:monadic = 1). . .start_conditions (True) (--start-conditions). .storable_state (True) (--storable-state). .unsafe (True) re2java "unsafe" (--unsafe re2c:unsafe = 1). "unsafe" . .version (True) re2java . .yyfill.enable (True) YYFILL (re2c:yyfill:enable = 1). .yyfn.throw (True) re2c:yyfn:throw . . : o ( *) o o <<>> ("natural") : . : . . re2java . . : o Sentinel (). . <> . . sentinel : . . YYFILL re2c:yyfill:enable = 0; re2c:eof -1 . o Sentinel (Sentinel with bounds checks). : . . <> <> . sentinel ( sentinel ). . sentinel . YYFILL ( ). sentinel . . re2c:eof ( sentinel ). YYFILL . o (Bounds checks with padding). < >> . DFA (SCC) SCC SCC . . () ( ) . YYMAXFILL max . YYFILL . YYFILL re2c:eof -1 ( ). o (Custom checks). Generic API . . Generic API --api custom re2c:api = custom; re2c:yyfill:enable = 0; re2c:yyfill:check = 0; . ..SS (Sentinel) (sentinel) . null . null : (lexeme) ( ). null * (crash) ( -Wsentinel-in-midrule re2c:sentinel ). re2c:yyfill:enable = 0; YYFILL . // re2java $INPUT -o $OUTPUT class Main { // Expects a null-terminated string. static int lex(String yyinput) { int yycursor = 0; int count = 0; loop: while (true) { /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:yyfill:enable = 0; * { return -1; } [\x00] { return count; } [a-z]+ { count += 1; continue loop; } [ ]+ { continue loop; } */ } } public static void main(String []args) { assert lex("\0") == 0; assert lex("one two three\0") == 3; assert lex("f0ur\0") == -1; } }; (Sentinel with bounds checks) ( 1.2 ). (single-quoted) . null re2c:eof = 0; . sentinel null ( 'aaa\0aa'\0 'aaa\0 ). null . API YYLIMIT <= YYCURSOR YYLESSTHAN(1) . (YYFILL re2c:yyfill:enable = 0; YYFILL YYFILL with sentinel ). : $ ( *) -Wundefined-control-flow . // re2java $INPUT -o $OUTPUT class Main { // Expects a null-terminated string. static int lex(String yyinput) { int yycursor = 0; int yymarker = 0; int yylimit = yyinput.length() - 1; // yylimit points at the terminating null int count = 0; loop: while (true) { /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:yyfill:enable = 0; re2c:eof = 0; str = ['] ([^'\\] | [\\][^])* [']; * { return -1; } $ { return count; } str { count += 1; continue loop; } [ ]+ { continue loop; } */ } } public static void main(String []args) { assert lex("\0") == 0; assert lex("'qu\0tes' 'are' 'fine: \\'' \0") == 3; assert lex("'unterminated\\'\0") == -1; } }; ( ). . YYMAXFILL null YYMAXFILL max . null --- (lexeme) ( ). "stop" (null) lexer ( null ). . (YYLIMIT - YYCURSOR) < n generic API YYLESSTHAN(n) n lexer ( n ). (true) lexer YYFILL(n) n . YYFILL lexer ( ). YYFILL YYFILL with padding . // re2java $INPUT -o $OUTPUT class Main { /*!max:re2c*/ // Expects yymaxfill-padded string. static int lex(byte[] str) { // Pad string with yymaxfill zeroes at the end. byte[] yyinput = new byte[str.length + YYMAXFILL]; System.arraycopy(str, 0, yyinput, 0, str.length); int yycursor = 0; int yylimit = yyinput.length; int count = 0; loop: while (true) { /*!re2c re2c:YYCTYPE = "int"; re2c:YYPEEK = "Byte.toUnsignedInt(yyinput[yycursor])"; re2c:YYFILL = "return -1;"; str = ['] ([^'\\] | [\\][^])* [']; [\x00] { // Check that it is the sentinel, not some unexpected null. return (yycursor - 1 == str.length) ? count : -1; } str { count += 1; continue loop; } [ ]+ { continue loop; } * { return -1; } */ } } public static void main(String []args) { assert lex("".getBytes()) == 0; assert lex("'qu\0tes' 'are' 'fine: \\'' ".getBytes()) == 3; assert lex("'unterminated\\'".getBytes()) == -1; assert lex("'unexpected \00 null\\'".getBytes()) == -1; } }; (Custom checks) generic API . . sentinel null . (sentinel) YYPEEK . . YYPEEK . // re2java $INPUT -o $OUTPUT class Main { // Expects a string without terminating null. static int lex(String str) { byte[] yyinput = str.getBytes(); int yycursor = 0; int count = 0; loop: while (true) { /*!re2c re2c:api = generic; re2c:YYCTYPE = "int"; re2c:YYPEEK = "(yycursor < yyinput.length)" " ? Byte.toUnsignedInt(yyinput[yycursor]) : 0"; re2c:YYSKIP = "yycursor += 1;"; re2c:yyfill:enable = 0; * { return -1; } [\x00] { return count; } [a-z]+ { count += 1; continue loop; } [ ]+ { continue loop; } */ } } public static void main(String []args) { assert lex("") == 0; assert lex("one two three") == 3; assert lex("f0ur") == -1; } }; : ( ) . . . lexer : o (cursor): (YYCURSOR API C YYSKIP/YYPEEK API ) o (limit): (YYLIMIT API C YYLESSTHAN API ) o (marker): (YYMARKER API YYBACKUP/YYRESTORE API ) o (token): (lexeme) ( API re2java lexer ) o (context marker): (YYCTXMARKER API C YYBACKUPCTX/YYRESTORECTX API ) o (tag variables): ( stags mtags API YYSTAGP/YYSTAGN/YYMTAGP/YYMTAGN) YYFILL . token cursor token token limit . YYFILL ( lexer ). YYFILL EOF : EOF (padding) . -f --storable-state YYFILL ( storable state ). YYFILL (YYFILL with sentinel) EOF YYFILL . YYFILL YYLIMIT <= YYCURSOR API C YYLESSTHAN() API . YYFILL . YYFILL . Limit limit re2c:eof . YYFILL ( # ). <-- shift --> >-A------------B---------C-------------D#-----------E-> buffer token marker limit, cursor >-A------------B---------C-------------D------------E#-> buffer, marker cursor limit token <-- shift --> >-A------------B---------C-------------D#--E (EOF) buffer token marker limit, cursor >-A------------B---------C-------------D---E#........ buffer, marker cursor limit token input.txt EOF . // re2java $INPUT -o $OUTPUT import java.io.*; import java.nio.file.*; class Lexer { public static final int BUFSIZE = 4096; private BufferedInputStream stream; private byte[] yyinput; private int yycursor; private int yymarker; private int yylimit; private int token; private boolean eof; public Lexer(File file) throws FileNotFoundException { stream = new BufferedInputStream(new FileInputStream(file)); // Sentinel at `yylimit` offset is set to zero, which triggers YYFILL. yyinput = new byte[BUFSIZE + 1]; yycursor = yymarker = yylimit = token = BUFSIZE; eof = false; } private int fill() throws IOException { if (eof) { return -1; } // unexpected EOF // Error: lexeme too long. In real life can reallocate a larger buffer. if (token < 1) { return -2; } // Shift buffer contents (discard everything up to the current token). System.arraycopy(yyinput, token, yyinput, 0, yylimit - token); yycursor -= token; yymarker -= token; yylimit -= token; token = 0; // Fill free space at the end of buffer with new data from file. yylimit += stream.read(yyinput, yylimit, BUFSIZE - yylimit); yyinput[yylimit] = 0; // append sentinel symbol // If read less than expected, this is the end of input. eof = yylimit < BUFSIZE; return 0; } // Expects a null-terminated string. public int lex() throws IOException { int count = 0; loop: while (true) { token = yycursor; /*!re2c re2c:YYCTYPE = "int"; re2c:YYPEEK = "Byte.toUnsignedInt(yyinput[yycursor])"; re2c:YYFILL = "fill() == 0"; re2c:eof = 0; str = ['] ([^'\\] | [\\][^])* [']; * { return -1; } $ { return count; } str { count += 1; continue loop; } [ ]+ { continue loop; } */ } } public static void main(String []args) throws FileNotFoundException, IOException { String fname = "input"; String content = "'qu\0tes' 'are' 'fine: \\'' ".repeat(Lexer.BUFSIZE); // Prepare input file: a few times the size of the buffer, containing // strings with zeroes and escaped quotes. Files.writeString(Paths.get(fname), content); int count = 3 * Lexer.BUFSIZE; // number of quoted strings written to file // Prepare lexer state: all offsets are at the end of buffer. File file = new File(".", fname); Lexer lexer = new Lexer(file); // Run the lexer. int n = lexer.lex(); assert n == count; // Cleanup: remove input file. file.delete(); } }; ( EOF ) YYFILL (function-like primitive) . YYFILL (YYLIMIT - YYCURSOR) < n API C YYLESSTHAN(n) API . YYFILL . YYFILL (lexer) ( ). YYFILL (limit) YYMAXFILL ( YYFILL n ). YYFILL ( YYMAXFILL # ). <-- shift --> <-- need --> >-A------------B---------C-----D-------E---F--------G-> buffer token marker cursor limit >-A------------B---------C-----D-------E---F--------G-> buffer, marker cursor limit token <-- shift --> <-- need --> >-A------------B---------C-----D-------E-F (EOF) buffer token marker cursor limit >-A------------B---------C-----D-------E-F############### buffer, marker cursor limit token <- YYMAXFILL -> input.txt (bounds-checking) . // re2java $INPUT -o $OUTPUT import java.io.*; import java.nio.file.*; import java.util.Arrays; class Lexer { /*!max:re2c*/ public static final int BUFSIZE = 4096; private BufferedInputStream stream; private byte[] yyinput; private int yycursor; private int yylimit; private int token; private boolean eof; public Lexer(File file) throws FileNotFoundException { stream = new BufferedInputStream(new FileInputStream(file)); // Prepare lexer state: all offsets are at the end of buffer. // This immediately triggers YYFILL, as the YYLESSTHAN condition is true. yyinput = new byte[BUFSIZE + YYMAXFILL]; yycursor = yylimit = token = BUFSIZE; eof = false; } private int fill(int need) throws IOException { if (eof) { return -1; } // unexpected EOF // Error: lexeme too long. In real life can reallocate a larger buffer. if (token < need) { return -2; } // Shift buffer contents (discard everything up to the current token). System.arraycopy(yyinput, token, yyinput, 0, yylimit - token); yycursor -= token; yylimit -= token; token = 0; // Fill free space at the end of buffer with new data from file. yylimit += stream.read(yyinput, yylimit, BUFSIZE - yylimit); yyinput[yylimit] = 0; // append sentinel symbol // If read less than expected, this is the end of input. if (yylimit < BUFSIZE) { eof = true; Arrays.fill(yyinput, yylimit, yylimit + YYMAXFILL, (byte)0); yylimit += YYMAXFILL; } return 0; } // Expects a null-terminated string. public int lex() throws IOException { int count = 0; loop: while (true) { token = yycursor; /*!re2c re2c:YYCTYPE = "int"; re2c:YYPEEK = "Byte.toUnsignedInt(yyinput[yycursor])"; re2c:YYFILL = "if (fill(@@) != 0) { return -2; }"; str = ['] ([^'\\] | [\\][^])* [']; [\x00] { // Check that it is the sentinel, not some unexpected null. return (token == yylimit - YYMAXFILL) ? count : -1; } str { count += 1; continue loop; } [ ]+ { continue loop; } * { return -1; } */ } } public static void main(String []args) throws FileNotFoundException, IOException { String fname = "input"; String content = "'qu\0tes' 'are' 'fine: \\'' ".repeat(Lexer.BUFSIZE); // Prepare input file: a few times the size of the buffer, containing // strings with zeroes and escaped quotes. Files.writeString(Paths.get(fname), content); int count = 3 * Lexer.BUFSIZE; // number of quoted strings written to file // Prepare lexer state: all offsets are at the end of buffer. File file = new File(".", fname); Lexer lexer = new Lexer(file); // Run the lexer. int n = lexer.lex(); assert n == count; // Cleanup: remove input file. file.delete(); } }; (Multiple blocks) (lexer) ( ). re2java . (start conditions) . . goto ( C/C++ Go) (label) . goto . goto ( Rust) switch /switch yystate re2java . (binary) (octal) (decimal) (hexadecimal) . . . . // re2java $INPUT -o $OUTPUT class Parser { private String yyinput; private int yycursor; private int yymarker; private int number; private void add_digit(int base, int offset) throws ArithmeticException { number = Math.addExact( Math.multiplyExact(number, base), yyinput.charAt(yycursor - 1) - offset); } public int parse(String str) throws ArithmeticException, IllegalArgumentException { yyinput = str; yycursor = 0; number = 0; try { /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:yyfill:enable = 0; end = "\x00"; '0b' / [01] { return parse_bin(); } "0" { return parse_oct(); } "" / [1-9] { return parse_dec(); } '0x' / [0-9a-fA-F] { return parse_hex(); } * { throw new IllegalArgumentException("not a number"); } */ } catch (Exception e) { return -1; } } private int parse_bin() throws ArithmeticException, IllegalArgumentException { /*!re2c end { return number; } [01] { add_digit(2, 48); return parse_bin(); } * { throw new IllegalArgumentException("ill-formed binary number"); } */ } private int parse_oct() throws ArithmeticException, IllegalArgumentException { /*!re2c end { return number; } [0-7] { add_digit(8, 48); return parse_oct(); } * { throw new IllegalArgumentException("ill-formed octal number"); } */ } private int parse_dec() throws ArithmeticException, IllegalArgumentException { /*!re2c end { return number; } [0-9] { add_digit(10, 48); return parse_dec(); } * { throw new IllegalArgumentException("ill-formed decimal number"); } */ } private int parse_hex() throws ArithmeticException, IllegalArgumentException { /*!re2c end { return number; } [0-9] { add_digit(16, 48); return parse_hex(); } [a-f] { add_digit(16, 87); return parse_hex(); } [A-F] { add_digit(16, 55); return parse_hex(); } * { throw new IllegalArgumentException("ill-formed hexadecimal number"); } */ } public static void main(String []args) { Parser parser = new Parser(); assert parser.parse("1234567890\0") == 1234567890; assert parser.parse("0b1101\0") == 0b1101; assert parser.parse("0x007Fe\0") == 0x7fe; assert parser.parse("0644\0") == 0644; assert parser.parse("9999999999\0") == -1; assert parser.parse("123??\0") == -1; } }; --start-conditions . re2java . re2java . : YYGETCOND YYSETCOND . conditions --header re2c:header . re2c:cond:enumprefix . <*> . : < condition-list > regular-expression code condition-list . regular-expression code . < condition-list > regular-expression => condition code condition-list . regular-expression condition code . < condition-list > regular-expression :=> condition condition-list . regular-expression condition ( ). < condition-list > !action code code action condition-list ( actions ). code code condition-list . !pre_rule ( ). < > code "0" code . !entry ( ). < > => condition code condition . < > :=> condition condition . re2java re2java goto/label loop/switch . goto ( C/C++ Go) yyc_ cond ( re2c:cond:prefix ). goto . re2java switch YYGETSTATE . :=> switch ( re2c:cond:goto ). . goto ( Rust) re2java yystate . switch switch DFA . switch . ( YYGETCOND/YYSETCOND ). () () () () . . cond conditions . // re2java $INPUT -o $OUTPUT -c class Parser { /*!conditions:re2c*/ private String yyinput; private int yycursor; private int yymarker; private int number; private void add_digit(int base, int offset) throws ArithmeticException { number = Math.addExact( Math.multiplyExact(number, base), yyinput.charAt(yycursor - 1) - offset); } public int parse(String str) throws ArithmeticException, IllegalArgumentException { yyinput = str; yycursor = 0; int yycond = YYC_init; number = 0; try { loop: while (true) { /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:yyfill:enable = 0; <*> * { throw new IllegalArgumentException("ill-formed number"); } '0b' / [01] :=> bin "0" :=> oct "" / [1-9] :=> dec '0x' / [0-9a-fA-F] :=> hex "\x00" { return number; } [01] { add_digit(2, 48); continue loop; } [0-7] { add_digit(8, 48); continue loop; } [0-9] { add_digit(10, 48); continue loop; } [0-9] { add_digit(16, 48); continue loop; } [a-f] { add_digit(16, 87); continue loop; } [A-F] { add_digit(16, 55); continue loop; } */ } } catch (Exception e) { return -1; } } public static void main(String []args) { Parser parser = new Parser(); assert parser.parse("1234567890\0") == 1234567890; assert parser.parse("0b1101\0") == 0b1101; assert parser.parse("0x007Fe\0") == 0x7fe; assert parser.parse("0644\0") == 0644; assert parser.parse("9999999999\0") == -1; assert parser.parse("123??\0") == -1; } }; --storable-state re2java . re2java <<>> ("pull") <<>> ("pulls"). ( ). : <<>> ("push") . . . "pull" : o YYSETSTATE() YYGETSTATE(state). o yych yyaccept ( ) state . state -1 . o YYFILL . . o . o getstate YYGETSTATE . goto ( --loop-switch). o re2c:eof sentinel with bounds checks . (padding) : (lexeme) . ( ). "push" . . <<>> ("sending") . . // re2java $INPUT -o $OUTPUT -f import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.Pipe; class Lexer { enum Status { END, READY, WAITING, BIG_PACKET, BAD_PACKET }; // Use a small buffer to cover the case when a lexeme doesn't fit. // In real world use a larger buffer. public static final int BUFSIZE = 10; public static class State { Pipe.SourceChannel source; byte[] yyinput; int yycursor; int yymarker; int yylimit; int token; int yystate; int received; public State(Pipe pipe) { source = pipe.source(); // Sentinel at `yylimit` offset is set to zero, which triggers YYFILL. yyinput = new byte[BUFSIZE + 1]; yycursor = yymarker = yylimit = token = BUFSIZE; yystate = -1; received = 0; } } private static void log(String format, Object... args) { if (false) { System.out.printf(format + "\n", args); } } private static Status fill(State st) throws IOException { // Error: lexeme too long. In real life can reallocate a larger buffer. if (st.token < 1) { return Status.BIG_PACKET; } // Shift buffer contents (discard everything up to the current token). System.arraycopy(st.yyinput, st.token, st.yyinput, 0, st.yylimit - st.token); st.yycursor -= st.token; st.yymarker -= st.token; st.yylimit -= st.token; st.token = 0; // Fill free space at the end of buffer with new data from file. ByteBuffer buffer = ByteBuffer.wrap(st.yyinput, st.yylimit, BUFSIZE - st.yylimit); int have = st.source.read(buffer); if (have != -1) st.yylimit += have; // -1 means that pipe is closed st.yyinput[st.yylimit] = 0; // append sentinel symbol return Status.READY; } private static Status lex(State yyrecord) { int yych; loop: while (true) { yyrecord.token = yyrecord.yycursor; /*!re2c re2c:api = record; re2c:YYCTYPE = "int"; re2c:YYPEEK = "Byte.toUnsignedInt(yyrecord.yyinput[yyrecord.yycursor])"; re2c:YYFILL = "return Status.WAITING;"; re2c:eof = 0; packet = [a-z]+[;]; * { return Status.BAD_PACKET; } $ { return Status.END; } packet { yyrecord.received += 1; continue loop; } */ } } public static void test(String[] packets, Status expect) throws IOException { // Create a pipe. Pipe pipe = Pipe.open(); Pipe.SinkChannel sink = pipe.sink(); // Initialize lexer state Lexer.State st = new Lexer.State(pipe); // Main loop. The buffer contains incomplete data which appears packet by // packet. When the lexer needs more input it saves its internal state and // returns to the caller which should provide more input and resume lexing. int send = 0; Status status; while (true) { status = lex(st); if (status == Status.END) { log("done: got %d packets", st.received); break; } else if (status == Status.WAITING) { log("waiting..."); if (send < packets.length) { log("sent packet %d: %s", send, packets[send]); ByteBuffer buffer = ByteBuffer.wrap(packets[send].getBytes()); sink.write(buffer); send += 1; } else { sink.close(); } status = fill(st); if (status == Status.BIG_PACKET) { log("error: packet too big"); break; } assert status == Status.READY; } else { assert status == Status.BAD_PACKET; log("error: ill-formed packet"); break; } } // Check results. assert status == expect; if (status == Status.END) { assert send == st.received; } } public static void main(String []args) throws IOException { test(new String[]{}, Status.END); test(new String[]{"zero;", "one;", "two;", "three;", "four;"}, Status.END); test(new String[]{"zer0;"}, Status.BAD_PACKET); test(new String[]{"goooooooooogle;"}, Status.BIG_PACKET); } }; /*!rules:re2c[:] ... */ %{rules[:] ... %} re2java . . rules use . rules . use /*!use:re2c[:] ... */ %{use[:] ... %} . : rules ( ). use . use : rules ( ). !use:; re2java . . : . * $ ( ) . use re2java ( ). use 2.2 re2java . ( ). 2.2 -r --reusable . 1.2 . !use // re2java $INPUT -o $OUTPUT // This example shows how to combine reusable re2c blocks: two blocks // ('colors' and 'fish') are merged into one. The 'salmon' rule occurs // in both blocks; the 'fish' block takes priority because it is used // earlier. Default rule * occurs in all three blocks; the local (not // inherited) definition takes priority. /*!rules:re2c:colors * { throw new IllegalArgumentException("ah"); } "red" | "salmon" | "magenta" { return Ans.COLOR; } */ /*!rules:re2c:fish * { throw new IllegalArgumentException("oh"); } "haddock" | "salmon" | "eel" { return Ans.FISH; } */ class Main { enum Ans {COLOR, FISH, DUNNO}; static Ans lex(String yyinput) { // no-throw, as '*' rules are overridden int yycursor = 0; int yymarker = 0; /*!re2c re2c:yyfill:enable = 0; re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; !use:fish; !use:colors; * { return Ans.DUNNO; } // overrides inherited '*' rules */ } public static void main(String []args) { assert lex("salmon") == Ans.FISH; assert lex("what?") == Ans.DUNNO; } }; // re2java $INPUT -o $OUTPUT --input-encoding utf8 // This example supports multiple input encodings: UTF-8 and UTF-32. // Both lexers are generated from the same rules block, and the use // blocks add only encoding-specific configurations. /*!rules:re2c re2c:yyfill:enable = 0; re2c:YYPEEK = "yyinput[yycursor]"; re2c:indent:top = 1; "x y" { return true; } * { return false; } */ class Main { static boolean lex_utf8(int[] yyinput) { int yycursor = 0; int yymarker = 0; /*!use:re2c re2c:YYCTYPE = "int"; // Java lacks unsigned 8-bit integer type re2c:encoding:utf8 = 1; */ } static boolean lex_utf32(int[] yyinput) { int yycursor = 0; int yymarker = 0; /*!use:re2c re2c:YYCTYPE = "int"; re2c:encoding:utf32 = 1; */ } public static void main(String []args) { // we have to use `int`, because `byte`in Java cannot represent values greater than 127 int[] s_utf8 = new int[]{0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79}; assert lex_utf8(s_utf8); int[] s_utf32 = new int[]{0x2200, 0x78, 0x20, 0x2203, 0x79}; assert lex_utf32(s_utf32); } }; (Submatch extraction) re2java . Tags @stag #mtag stag mtag . -T --tags re2c:tags = 1 . : ( ) . S-tag m-tag ( ). svars mvars . (leftmost greedy) . Captures (capturing groups) . --captures re2c:captures = 1 . --leftmost-captures () --posix-captures (POSIX longest-match) . : (! ... ). --invert-captures re2c:invert-captures = 1 . yynmatch ( ) yypmatch . yynmatch yypmatch yypmatch [yynmatch * 2] . maxnmatch YYMAXNMATCH yynmatch . Captvars --captvars re2c:captvars = 1 . --captures : yynmatch yypmatch re2java yytl yytr k- ( svars ). : --leftmost-captvars re2c:leftmost-captvars = 1 ( ) --posix-captvars re2c:posix-captvars . (TDFA) . TDFA (overhead) . TDFA DFA . . TDFA : . . stags mtags . . YYFILL . S-tag : o s-tag: t = YYCURSOR API C YYSTAGP(t) API (generic API) o s-tag: t = NULL API C YYSTAGN(t) API o s-tag : t1 = t2 M-tag : o m-tag: YYMTAGP(t) API API o m-tag: YYMTAGN(t) API API o m-tag : t1 = t2 S-tag ( ) . M-tag . m-tag ( ) m-tag (prefix-tree) (v, p) v p . s-tag (semantic versions) : major minor patch ( ). YYFILL . // re2java $INPUT -o $OUTPUT import java.util.Optional; class Main { static class SemVer { int major; int minor; int patch; public SemVer(int m, int n, int k) { major = m; minor = n; patch = k; } public boolean equals(SemVer v) { return major == v.major && minor == v.minor && patch == v.patch; } }; static Optional parse(String yyinput) { int yycursor = 0; int yymarker = 0; // Final tag variables available in semantic action. /*!svars:re2c format = "int @@;"; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = "int @@ = -1;"; */ /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:yyfill:enable = 0; re2c:tags = 1; num = [0-9]+; @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\x00] { int major = Integer.valueOf(yyinput.substring(t1, t2)); int minor = Integer.valueOf(yyinput.substring(t3, t4)); int patch = (t5 == -1) ? 0 : Integer.valueOf(yyinput.substring(t5, yycursor - 1)); return Optional.of(new SemVer(major, minor, patch)); } * { return Optional.empty(); } */ } public static void main(String []args) { assert parse("23.34\0").get().equals(new SemVer(23, 34, 0)); assert parse("1.2.99999\0").get().equals(new SemVer(1, 2, 99999)); assert !parse("1.a\0").isPresent(); } }; s-tag YYFILL (semantic versions) . YYFILL . s-tag . ( m-tag ) . // re2java $INPUT -o $OUTPUT import java.io.*; import java.nio.file.*; import java.util.*; class Lexer { static class SemVer { int major; int minor; int patch; public SemVer(int m, int n, int k) { major = m; minor = n; patch = k; } public boolean equals(SemVer v) { return major == v.major && minor == v.minor && patch == v.patch; } }; public static final int BUFSIZE = 4096; private BufferedInputStream stream; private byte[] yyinput; private int yycursor; private int yymarker; private int yylimit; private int token; // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = "private int @@;\n"; */ private boolean eof; public Lexer(File file) throws FileNotFoundException { stream = new BufferedInputStream(new FileInputStream(file)); // Sentinel at `yylimit` offset is set to zero, which triggers YYFILL. yyinput = new byte[BUFSIZE + 1]; yycursor = yymarker = yylimit = token = BUFSIZE; /*!stags:re2c format = "@@ = -1;\n"; */ eof = false; } private int fill() throws IOException { if (eof) { return -1; } // unexpected EOF // Error: lexeme too long. In real life can reallocate a larger buffer. if (token < 1) { return -2; } // Shift buffer contents (discard everything up to the current token). System.arraycopy(yyinput, token, yyinput, 0, yylimit - token); yycursor -= token; yymarker -= token; yylimit -= token; /*!stags:re2c format = "if (@@ != -1) {@@ -= token;}\n"; */ token = 0; // Fill free space at the end of buffer with new data from file. yylimit += stream.read(yyinput, yylimit, BUFSIZE - yylimit); yyinput[yylimit] = 0; // append sentinel symbol // If read less than expected, this is the end of input. eof = yylimit < BUFSIZE; return 0; } private int readInt(int tag1, int tag2) { int n = 0; for (int i = tag1; i < tag2; ++i) { n = n * 10 + (yyinput[i] - 48); } return n; } public Optional> lex() throws IOException { ArrayList vers = new ArrayList(); // Final tag variables available in semantic action. /*!svars:re2c format = "int @@;"; */ loop: while (true) { token = yycursor; /*!re2c re2c:YYCTYPE = "int"; re2c:YYPEEK = "Byte.toUnsignedInt(yyinput[yycursor])"; re2c:YYFILL = "fill() == 0"; re2c:eof = 0; re2c:tags = 1; num = [0-9]+; @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\n] { int major = readInt(t1, t2); int minor = readInt(t3, t4); int patch = (t5 == -1) ? 0 : readInt(t5, yycursor - 1); vers.add(new SemVer(major, minor, patch)); continue loop; } $ { return Optional.of(vers); } * { return Optional.empty(); } */ } } public static void main(String []args) throws FileNotFoundException, IOException { String fname = "input"; String content = "1.22.333\n".repeat(Lexer.BUFSIZE); // Prepare input file: a few times the size of the buffer, containing // strings with zeroes and escaped quotes. Files.writeString(Paths.get(fname), content); // Prepare lexer state: all offsets are at the end of buffer. File file = new File(".", fname); Lexer lexer = new Lexer(file); // Run the lexer. Optional> vers = lexer.lex(); // Check results. assert vers.isPresent() && vers.get().size() == BUFSIZE; SemVer v = new SemVer(1, 22, 333); for (int i = 0; i < BUFSIZE; ++i) { assert vers.get().get(i).equals(v); } // Cleanup: remove input file. file.delete(); } }; (capturing groups) . // re2java $INPUT -o $OUTPUT import java.util.Optional; class Main { static class SemVer { int major; int minor; int patch; public SemVer(int m, int n, int k) { major = m; minor = n; patch = k; } public boolean equals(SemVer v) { return major == v.major && minor == v.minor && patch == v.patch; } }; static Optional parse(String yyinput) { int yycursor = 0; int yymarker = 0; // Final tag variables available in semantic action. /*!svars:re2c format = "int @@;"; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = "int @@ = -1;"; */ /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:yyfill:enable = 0; re2c:captvars = 1; num = [0-9]+; (num) "." (num) ("." num)? [\x00] { int major = Integer.valueOf(yyinput.substring(yytl1, yytr1)); int minor = Integer.valueOf(yyinput.substring(yytl2, yytr2)); int patch = (yytl3 == -1) ? 0 : Integer.valueOf(yyinput.substring(yytl3 + 1, yytr3)); return Optional.of(new SemVer(major, minor, patch)); } * { return Optional.empty(); } */ } public static void main(String []args) { assert parse("23.34\0").get().equals(new SemVer(23, 34, 0)); assert parse("1.2.99999\0").get().equals(new SemVer(1, 2, 99999)); assert !parse("1.a\0").isPresent(); } }; m-tag . (trie) . // re2java $INPUT -o $OUTPUT import java.util.*; class Main { static Optional parse(String yyinput) { int yycursor = 0; int yymarker = 0; // Final tag variables available in semantic action. /*!svars:re2c format = "int @@;"; */ /*!mvars:re2c format = "List @@;"; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = "int @@ = -1;"; */ /*!mtags:re2c format = "List @@ = new ArrayList<>();"; */ /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:YYMTAGP = "@@.add(yycursor);"; re2c:YYMTAGN = ""; // do nothing re2c:yyfill:enable = 0; re2c:tags = 1; num = [0-9]+; @t1 num @t2 ("." #t3 num #t4)* [\x00] { int[] vers = new int[t3.size() + 1]; vers[0] = Integer.valueOf(yyinput.substring(t1, t2)); for (int i = 0; i < t3.size(); ++i) { vers[i + 1] = Integer.valueOf(yyinput.substring(t3.get(i), t4.get(i))); } return Optional.of(vers); } * { return Optional.empty(); } */ } public static void main(String []args) { assert Arrays.equals(parse("1\0").get(), new int[]{1}); assert Arrays.equals(parse("1.2.3.4.5.6.7\0").get(), new int[]{1, 2, 3, 4, 5, 6, 7}); assert !parse("1.2.\0").isPresent(); } }; (Encoding support) (code points) (code units) . . . . . . << >> [^] ( * ). : yych YYCTYPE . . re2java . : . : o ASCII ( ). [0-255] . o EBCDIC ( --ebcdic re2c:encoding:ebcdic). [0-255] . o UCS2 ( --ucs2 re2c:encoding:ucs2). [0-0xFFFF] . o UTF8 ( --utf8 re2c:encoding:utf8). . . . o UTF16 ( --utf16 re2c:encoding:utf16). . . . o UTF32 ( --utf32 re2c:encoding:utf32). [0-0x10FFFF] . include/unicode_blocks.re include/unicode_categories.re include/unicode_properties.re re2java . --input-encoding . --input-encoding utf8 re2java UTF8 ( --utf8 ). --encoding-policy re2java ( [0xD800-0xDFFF]) . UTF8 . // re2java $INPUT -o $OUTPUT --utf8 -s /*!include:re2c "unicode_categories.re" */ class Main { static boolean lex(String yyinput) { int yycursor = 0; int yymarker = 0; /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:yyfill:enable = 0; // Simplified "Unicode Identifier and Pattern Syntax" // (see https://unicode.org/reports/tr31) id_start = L | Nl | [$_]; id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3]; identifier = id_start+; // It should be `id_start id_continue*`, but that causes `error: code too large` identifier { return true; } * { return false; } */ } public static void main(String []args) { assert lex("_\0"); } }; re2java /*!include:re2c FILE */ %{include FILE %} !include FILE ; FILE . re2java include -I . / include re2java #include C/C++ : FILE / . include . --depfile . re2java include/ . ( ) re2java . . (definitions.java) /*!re2c number = [1-9][0-9]*; */ (extra_rules.re.inc) // floating-point numbers frac = [0-9]* "." [0-9]+ | [0-9]+ "."; exp = 'e' [+-]? [0-9]+; float = frac exp? | [0-9]+ exp; float { return Num.FLOAT; } (Input file) // re2java $INPUT -o $OUTPUT /*!include:re2c "definitions.java" */ class Main { enum Num {INT, FLOAT, NAN}; static Num lex(String yyinput) { int yycursor = 0; int yymarker = 0; /*!re2c re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyinput.charAt(yycursor)"; re2c:yyfill:enable = 0; * { return Num.NAN; } number { return Num.INT; } !include "extra_rules.re.inc"; */ } public static void main(String []args) { assert lex("123\0") == Num.INT; assert lex("123.4567\0") == Num.FLOAT; } }; re2java .re --header re2c:header /*!header:re2c:on*/ /*!header:re2c:off*/ %{header:on%} %{header:off%} . . re2java --header re2c:header ( stdout) . re2java (translation units) . (lexer state) (tag variables) ( ). (Input file) // re2java $INPUT -o $OUTPUT --header lexer/state.java package headers; import headers.lexer.State; /*!header:re2c:on*/ package headers.lexer; public class State { public String yyinput; public int yycursor; /*!stags:re2c format = "public int @@;\n"; */ public State(String str) { yyinput = str; yycursor = 0; /*!stags:re2c format = "@@ = 0;\n"; */ } }; /*!header:re2c:off*/ class Main { static int lex(String str) { State yyrecord = new State(str); int t; /*!re2c re2c:api = record; re2c:tags = 1; re2c:yyfill:enable = 0; re2c:YYCTYPE = "char"; re2c:YYPEEK = "yyrecord.yyinput.charAt(yyrecord.yycursor)"; re2c:header = "lexer/state.java"; [a]* @t [b]* { return t; } */ } public static void main(String []args) { assert lex("ab\0") == 1; } }; (Header file) // Generated by re2c package headers.lexer; public class State { public String yyinput; public int yycursor; public int yyt1; public State(String str) { yyinput = str; yycursor = 0; yyt1 = 0; } }; -S, --skeleton re2java re2java C . (lexer) . DFA ( ) re2java : .input DFA .keys . .input . : o re2java ( DFA ). o (benchmarking) . o ( ). ( ). re2java DFA . . DFA . ( ASCII UTF-8 EBCDIC) DFA . DFA : re2java DFA . . ( ) re2java . (re2java ). (Visualization and debug) -D, --emit-dot re2java DFA DOT (dump) . Graphviz DFA . DFA . --dump-nfa --dump-dfa-raw ( ). (SEE ALSO) re2c : http://re2c.org. flex(1) lex(1) quex(http://quex.sourceforge.net). (AUTHORS) re2java Peter Bumbulis (peter@csg.uwaterloo.ca) . Marcus Boerger Dan Nuffer . : Brian Young (bayoung@acm.org) Marcus Boerger Dan Nuffer (nuffer@users.sourceforge.net) Ulya Trofimovich (skvadrik@gmail.com) Serghei Iakovlev Sergei Trofimovich Petr Skocik ligfx raekye PolarGoose. RE2JAVA(1)