RE2OCAML(1) RE2OCAML(1) " Man page generated from reStructuredText. (NAME) re2ocaml - OCaml (SYNOPSIS) re2ocaml [ OPTIONS ] [ WARNINGS ] INPUT - stdin . (INTRODUCTION) re2ocaml . ( OCaml ) / . . re2ocaml . OCaml . : (* re2ocaml -o -i *) open String type state = { yyinput: string; mutable yycursor: int; } %{ re2c:YYFN = ["lex;bool", "yyrecord;state"]; re2c:yyfill:enable = 0; [1-9][0-9]* { true } * { false } %} let main () = let st = {yyinput = "1234"; yycursor = 0} in if not (lex st) then raise (Failure "error") let _ = main () re2ocaml : (* Generated by re2ocaml *) (* re2ocaml -o -i *) open String type state = { yyinput: string; mutable yycursor: int; } let rec yy0 (yyrecord : state) : bool = let yych = unsafe_get yyrecord.yyinput yyrecord.yycursor in yyrecord.yycursor <- yyrecord.yycursor + 1; match yych with | '1'..'9' -> (yy2 [@tailcall]) yyrecord | _ -> (yy1 [@tailcall]) yyrecord and yy1 (yyrecord : state) : bool = false and yy2 (yyrecord : state) : bool = let yych = unsafe_get yyrecord.yyinput yyrecord.yycursor in match yych with | '0'..'9' -> yyrecord.yycursor <- yyrecord.yycursor + 1; (yy2 [@tailcall]) yyrecord | _ -> (yy3 [@tailcall]) yyrecord and yy3 (yyrecord : state) : bool = true and lex (yyrecord : state) : bool = (yy0 [@tailcall]) yyrecord let main () = let st = {yyinput = "1234"; yycursor = 0} in if not (lex st) then raise (Failure "error") let _ = main () (BASICS) re2ocaml . : name = regular-expression ; name regular-expression . - . % re2ocaml . name . . . . configuration = value ; re2ocaml . re2ocaml % . value - . . . regular-expression code regular-expression ( := ). regular-expression code . . . : * $. ( % ). % . % . !action code code ( ). % . !directive ; . . % . (Blocks) /*!re2c */ %{ %} ( ). 2.2 . : /*!re2c[:] ... */ %{[:] ... %} . re2ocaml . . % . /*!local:re2c[:] ... */ %{local[:] ... %} ( ). /*!rules:re2c[:] ... */ %{rules[:] ... %} - - . ( % ). 2.2 re2ocaml -r --reusable . /*!use:re2c[:] ... */ %{use[:] ... %} . re2ocaml . ( ). use . 2.2 re2ocaml use -r --reusable . /*!max:re2c[:[:...]] ... */ %{max[:[:...]] ... %} YYMAXFILL . YYMAXFILL ( ). C (#define YYMAXFILL ) Go (var YYMAXFILL int = ) . format @@{max} ( @@) YYMAXFILL . /*!maxnmatch:re2c[:[:...]] ... */ %{maxnmatch[:[:...]] ... %} YYMAXNMATCH ( -P --posix-captures ). YYMAXNMATCH ( ). C (#define YYMAXNMATCH ) Go (var YYMAXNMATCH int = ) . format @@{max} ( @@) YYMAXNMATCH . /*!stags:re2c[:[:...]] ... */ /*!mtags:re2c[:[:...]] ... */ %{stags[:[:...]] ... %} %{mtags[:[:...]] ... %{ s-tag/m-tag re2ocaml . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!svars:re2c[:[:...]] ... */ /*!mvars:re2c[:[:...]] ... */ %{svars[:[:...]] ... %} %{mvars[:[:...]] ... %{ s-tag/m-tag ( - -tags) re2ocaml ( - -captvars - -posix-captvars). ( ). : format separator. format @@{tag} ( @@) . separator format . /*!getstate:re2c[:[:...]] ... */ %{getstate[:[:...]] ... %} ( - -storable-state ). . . . - -loop-switch goto . /*!conditions:re2c[:[:...]] ... */ /*!types:re2c... */ %{conditions[:[:...]] ... %} %{types... %} ( - -conditions ). ( ). YYCONDTYPE . format separator . format @@{cond} ( @@) @@{num} . separator format . /*!include:re2c */ %{include %} . #include C/C++ . - -depfile . /*!header:re2c:on*/ %{header:on %} . header:off re2ocaml -t - -type-header . /*!header:re2c:off*/ %{header:off %} header:on*/ . /*!ignore:re2c ... */ %{ignore ... %} . (Configurations) re2ocaml : re2c:api, re2c:input --api. re2c:api:sigil ("sigil") API . @@ . . $ ${name} . API . API re2c:YYFILL@len YYFILL. re2c:api:style API . functions ( C) free-form ( Go Rust). functions API . ( YYFILL) C/C++ (closures) Go Rust . free-form API : @@{var} @@ ( ). API re2c:YYFILL:naked YYFILL . re2c:bit-vectors, re2c:flags:bit-vectors, re2c:flags:b --bit-vectors . re2c:captures, re2c:leftmost-captures --leftmost-captures . re2c:captvars, re2c:leftmost-captvars --leftmost-captvars . re2c:case-insensitive, re2c:flags:case-insensitive --case-insensitive . re2c:case-inverted, re2c:flags:case-inverted --case-inverted . re2c:case-ranges, re2c:flags:case-ranges --case-ranges . re2c:computed-gotos, re2c:flags:computed-gotos, re2c:flags:g --computed-gotos . re2c:computed-gotos:relative, re2c:cgoto:relative --computed-gotos-relative . re2c:computed-gotos:threshold, re2c:cgoto:threshold goto if . 9 . re2c:cond:abort (abort) . re2c:cond:goto :=> . goto @@; . @@ ( re2c:api:sigil re2c:cond:goto@cond ). re2c:cond:goto@cond re2c:cond:goto . @@ . re2c:api:sigil . re2c:cond:divider . /* *********************************** */ . ( re2c:api;sigil re2c:cond:divider@cond ). re2c:cond:divider@cond re2c:cond:divider . @@ . re2c:api:sigil . re2c:cond:prefix, re2c:condprefix . yyc_ . re2c:cond:enumprefix, re2c:condenumprefix . yyc . re2c:debug-output, re2c:flags:debug-output, re2c:flags:d --debug-output . re2c:empty-class, re2c:flags:empty-class --empty-class . re2c:encoding:ebcdic, re2c:flags:ecb, re2c:flags:e --ebcdic . re2c:encoding:ucs2, re2c:flags:wide-chars, re2c:flags:w --ucs2 . re2c:encoding:utf8, re2c:flags:utf-8, re2c:flags:8 --utf8 . re2c:encoding:utf16, re2c:flags:utf-16, re2c:flags:x --utf16 . re2c:encoding:utf32, re2c:flags:unicode, re2c:flags:u --utf32 . re2c:encoding-policy, re2c:flags:encoding-policy --encoding-policy . re2c:eof $ . -1 ( $ ). . . re2c:header, re2c:flags:type-header, re2c:flags:t . --header . re2c:indent:string . "\t" . . . re2c:indent:top . . . re2c:invert-captures --invert-captures . re2c:label:prefix, re2c:labelprefix DFA . yy . re2c:label:start, re2c:startlabel . . . . ( ). re2c:label:yyFillLabel YYFILL re2c:eof . re2c:label:yyloop --loop-switch . yyloop . re2c:label:yyNext YYGETSTATE ( re2c:state:nextlabel) . yyNext . re2c:lookahead, re2c:flags:lookahead ( --no-lookahead ). re2c:monadic ( Haskell ). re2c:nested-ifs, re2c:flags:nested-ifs, re2c:flags:s --nested-ifs . re2c:posix-captures, re2c:flags:posix-captures, re2c:flags:P --posix-captures . re2c:posix-captvars --posix-captvars . re2c:tags, re2c:flags:tags, re2c:flags:T --tags . re2c:tags:expression . re2ocaml yyt . struct . @@{tag} @@ . re2c:tags:expression = "s.@@"; s.yyt . re2c:api:sigil ..TP re2c:tags:negative ( -1 (null pointer) ). re2c:tags:prefix . yyt . re2c:sentinel ( re2c:yyfill:enable = 0; re2c:eof ). : . -1` ( re2ocaml ). . re2c:state:abort (abort) -1 . re2c:state:nextlabel switch YYGETSTATE yyNext ( ). re2c:label:start getstate switch YYGETSTATE . re2c:unsafe, re2c:flags:unsafe --no-unsafe . unsafe YYPEEK . ( ). Rust . re2c:YYBACKUP, re2c:define:YYBACKUP API YYBACKUP . re2c:YYBACKUPCTX, re2c:define:YYBACKUPCTX API YYBACKUPCTX . re2c:YYCONDTYPE, re2c:define:YYCONDTYPE API YYCONDTYPE . re2c:YYCTYPE, re2c:define:YYCTYPE API YYCTYPE . re2c:YYCTXMARKER, re2c:define:YYCTXMARKER API YYCTXMARKER . re2c:YYCURSOR, re2c:define:YYCURSOR API YYCURSOR . re2c:YYDEBUG, re2c:define:YYDEBUG API YYDEBUG . re2c:YYFILL, re2c:define:YYFILL API YYFILL . re2c:YYFILL@len, re2c:define:YYFILL@len (sigil) YYFILL . @@ . re2c:api:sigil . re2c:YYFILL:naked, re2c:define:YYFILL:naked re2c:api:style YYFILL . API (free-form) . re2c:YYFN API YYFN . re2c:YYINPUT API YYINPUT . re2c:YYGETCOND, re2c:define:YYGETCONDITION API YYGETCOND . re2c:YYGETCOND:naked, re2c:define:YYGETCONDITION:naked re2c:api:style YYGETCOND . API . re2c:YYGETSTATE, re2c:define:YYGETSTATE API YYGETSTATE . re2c:YYGETSTATE:naked, re2c:define:YYGETSTATE:naked re2c:api:style YYGETSTATE . API . re2c:YYGETACCEPT, re2c:define:YYGETACCEPT API YYGETACCEPT . re2c:YYLESSTHAN, re2c:define:YYLESSTHAN API YYLESSTHAN . re2c:YYLIMIT, re2c:define:YYLIMIT API YYLIMIT . re2c:YYMARKER, re2c:define:YYMARKER API YYMARKER . re2c:YYMTAGN, re2c:define:YYMTAGN API YYMTAGN . re2c:YYMTAGP, re2c:define:YYMTAGP API YYMTAGP . re2c:YYPEEK, re2c:define:YYPEEK API YYPEEK . re2c:YYRESTORE, re2c:define:YYRESTORE API YYRESTORE . re2c:YYRESTORECTX, re2c:define:YYRESTORECTX API YYRESTORECTX . re2c:YYRESTORETAG, re2c:define:YYRESTORETAG API YYRESTORETAG . re2c:YYSETCOND, re2c:define:YYSETCONDITION API YYSETCOND . re2c:YYSETCOND@cond, re2c:define:YYSETCONDITION@cond (sigil) YYSETCOND . @@ . re2c:api:sigil . re2c:YYSETCOND:naked, re2c:define:YYSETCONDITION:naked re2c:api:style YYSETCOND . API . re2c:YYSETSTATE, re2c:define:YYSETSTATE API YYSETSTATE . re2c:YYSETSTATE@state, re2c:define:YYSETSTATE@state (sigil) YYSETSTATE . @@ . re2c:api:sigil . re2c:YYSETSTATE:naked, re2c:define:YYSETSTATE:naked re2c:api:style YYSETSTATE . API . re2c:YYSETACCEPT, re2c:define:YYSETACCEPT API YYSETACCEPT . re2c:YYSKIP, re2c:define:YYSKIP API YYSKIP . re2c:YYSHIFT, re2c:define:YYSHIFT API YYSHIFT . re2c:YYCOPYMTAG, re2c:define:YYCOPYMTAG API YYCOPYMTAG . re2c:YYCOPYSTAG, re2c:define:YYCOPYSTAG API YYCOPYSTAG . re2c:YYSHIFTMTAG, re2c:define:YYSHIFTMTAG API YYSHIFTMTAG . re2c:YYSHIFTSTAG, re2c:define:YYSHIFTSTAG API YYSHIFTSTAG . re2c:YYSTAGN, re2c:define:YYSTAGN API YYSTAGN . re2c:YYSTAGP, re2c:define:YYSTAGP API YYSTAGP . re2c:yyaccept, re2c:variable:yyaccept API yyaccept . re2c:yybm, re2c:variable:yybm API yybm . re2c:yybm:hex, re2c:variable:yybm:hex (bitmaps) --bit-vectors . ( ). re2c:yych, re2c:variable:yych API yych . re2c:yych:emit, re2c:variable:yych:emit yych . . re2c:yych:conversion, re2c:variable:yych:conversion re2ocaml yych YYCTYPE . ( ). re2c:yych:literals, re2c:variable:yych:literals yych . : char ( ) hex ( ) char_or_hex ( ). re2c:yyctable, re2c:variable:yyctable API yyctable ..TP re2c:yynmatch, re2c:variable:yynmatch API yynmatch . re2c:yypmatch, re2c:variable:yypmatch API yypmatch . re2c:yytarget, re2c:variable:yytarget API yytarget . re2c:yystable, re2c:variable:yystable . re2c:yystate, re2c:variable:yystate API yystate . re2c:yyfill, re2c:variable:yyfill API yyfill . re2c:yyfill:check YYFILL ( YYLESSTHAN API YYLIMIT API C) . ( ). re2c:yyfill:enable YYFILL ( ) . ( ) YYFILL ( (sentinel) ). ( -W) re2c:sentinel (lexer) . (YYFILL ). re2c:yyfill:parameter YYFILL . . ( ). re2c:YYFILL:naked re2c:api:style . re2c:yyfn:sep YYFN ( - ). re2c:yyfn:throw YYFN ( ). (Regular expressions) re2ocaml : "foo" . 'foo' . [a-xyz], [^a-xyz] ( ). . (newline). R \ S R S. R* R. R+ R. R? R . R{n} R n . R{n,} R n . R{n,m} R n m . (R) R . (submatch) --invert-captures (R) . (!R) --invert-captures (!R) . R S : R S. R | S : R S. R / S (Lookahead): R S S . name name ( "name" Flex). {name} name Flex. @stag s-tag: @stag stag . #mtag m-tag: #mtag mtag . $ . : \a \b \f \n \r \t \v \\ \ooo \xhh \uhhhh \Uhhhhhhhh..SS " (Actions)" re2ocaml : !entry code (Entry action) code . start conditions . (lexeme). !pre_rule code (Pre-rule action) code ( start conditions) . ( ) . !post_rule code (Post-rule action) code ( start conditions) . (trap statements) . (Directives) re2ocaml : !use:name ; use name . . : . * $ . use use . !include file ; include : file . (Program interface) (primitives) API . . . re2ocaml ( ) . API . re2ocaml API : Record API Record API OCaml . yyrecord ( re2c:yyrecord ) record _yyinput _yycursor _yymarker _yyctxmarker _yylimit . . Generic API API . --api generic re2c:api = generic . : YYPEEK YYSKIP YYBACKUP YYBACKUPCTX YYSTAGP YYSTAGN YYMTAGP YYMTAGN YYRESTORE YYRESTORECTX YYRESTORETAG YYSHIFT YYSHIFTSTAG YYSHIFTMTAG YYLESSTHAN YYEND. API . YYCTYPE ( ). ASCII EBCDIC UTF-8 . UTF-16 UCS-2 . UTF-32 . YYCURSOR l-value ( YYINPUT). YYCURSOR . YYCURSOR . YYLIMIT r-value ( YYINPUT). YYLIMIT . . YYCURSOR YYLIMIT . YYMARKER l-value ( YYINPUT). YYCURSOR . . YYCTXMARKER l-value (trailing context) ( YYINPUT). . YYCTXMARKER / . YYFILL API len. YYFILL len . re2c:eof len 1 YYFILL . re2c:eof YYFILL . len YYMAXFILL . YYFN --recursive-functions . re2c:fn:sep ( ) . ( ). : ( ) ( - ). YYINPUT r-value ( ). YYMAXFILL YYFILL. max . YYLESSTHAN API len. r-value len true . YYEND API . r-value ( ) true . YYEND $ . YYLESSTHAN . YYPEEK API . r-value YYCTYPE . YYSKIP API . YYBACKUP API ( YYRESTORE ). YYRESTORE API YYBACKUP . YYBACKUPCTX API ( YYRESTORECTX ). YYRESTORECTX API YYBACKUPCTX . YYRESTORETAG API tag tag . YYSTAGP API tag tag YYINPUT ( ). YYSTAGP tag . YYSTAGN API tag tag YYINPUT ( ). YYSTAGN tag . YYMTAGP API tag. YYMTAGP tag ( ). YYMTAGN API tag. YYMTAGN tag ( ). YYSHIFT API shift shift ( ). YYCOPYSTAG API lhs rhs s-tag rhs s-tag lhs . lhs rhs . YYCOPYMTAG API lhs rhs m-tag rhs m-tag lhs . lhs rhs . YYSHIFTSTAG API tag shift tag shift ( ). YYSHIFTMTAG API tag shift tag shift ( ). YYMAXNMATCH POSIX . maxnmatch . YYCONDTYPE . conditions --header . YYGETACCEPT var . var . YYSETACCEPT : var ( l-value ) val ( ). var val . YYGETCOND r-value YYCONDTYPE . YYSETCOND cond cond . YYGETSTATE r-value . -1 . YYSETSTATE state state . YYDEBUG -d --debug-output . ( YYDEBUG ). YYDEBUG : state ( DFA -1) symbol ( ). yyaccept l-value . --storable-state . yybm ( --bitmaps). . . k- n- n- k- . if switch . yych l-value YYCTYPE . -f --storable-state . yyctable ( --conditions --computed-gotos). yyfill l-value YYFILL ( YYFILL ). yynmatch l-value POSIX . -P --posix-captures . yypmatch l-value . yynmatch * 2 ( YYMAXNMATCH * 2 ). -P --posix-captures . yystable . yystate l-value --loop-switch DFA. yytarget ( ) . . yytarget --computed-gotos . (OPTIONS) re2ocaml . re2ocaml . re2ocaml . -? --help -h . --api API . simple . record (struct class ) . generic API . --bit-vectors -b . --nested-ifs . --captures, --leftmost-captures . yybmatch 2 * YYMAXNMATCH yynmatch . --captvars, --leftmost-captvars . yytl yytr k- . --case-insensitive . --case-inverted : . --case-ranges switch low ... high. C/C++ GCC Clang Tcc . Tcc . --computed-gotos -g "computed goto" ( ). re2ocaml . cgoto:threshold . cgoto:relative . --bit-vectors . --computed-gotos-relative --computed-gotos ( ). --computed-gotos . --conditions --start-conditions -c "conditions" Flex: . re2ocaml goto . --depfile FILE FILE Makefile : [include-file ...]. / include include . --output . --ebcdic --ecb -e EBCDIC . re2ocaml 0 -- 0xFF . --empty-class re2ocaml . match-empty () ( ). match-none . error . --encoding-policy re2ocaml . fail re2ocaml . substitute re2ocaml 0xFFFD . ignore () re2ocaml . . --flex-syntax -F Flex: - . . --goto-label "goto/label": DFA goto . goto . --header --type-header -t HEADER HEADER. header:on header:off . enum ( conditions ). -I PATH PATH include . include . re2ocaml FILE include -I . --input --api. default simple ( ). custom generic . --input-encoding re2ocaml. ascii () re2ocaml ASCII : . utf8 re2ocaml UTF8 . --invert-captures . (...) (! ...) . (! ...) (...) . --lang . C D Go Haskell Java JS OCaml Python Rust Swift V Zig ( --syntax ). none . --location-format . gnu 'filename:line:column: ...' . msvc 'filename(line,column) ...' . gnu . --loop-switch "loop/switch": DFA switch switch . yystate . yystate case ..TP --nested-ifs -s if switch . . --no-debug-info -i (line directives) . ( diff ). --no-generation-date . --no-version . --no-unsafe unsafe YYPEEK ( Rust ). YYPEEK (bounds-checking) (lexer) . YYPEEK release --no-unsafe unsafe . --output -o OUTPUT OUTPUT . --posix-captures, -P (submatch) POSIX . yybmatch 2 * YYMAXNMATCH yynmatch . --posix-captvars POSIX . yytl yytr k- . --recursive-functions (co-recursive) DFA . --reusable -r 2.2 ( ). --skeleton -S "skeleton" () . "skeleton" . . C . --storable-state -f . push . YYGETSTATE YYSETSTATE yych yyaccept state . --syntax FILE FILE (syntax) . FILE ( ) ( --lang none ). --tags -T (tags) . --ucs2 --wide-chars -w UCS2 . re2ocaml 0 -- 0xFFFF . --nested-ifs . --utf8 --utf-8 -8 UTF-8 . re2ocaml 0 -- 0x10FFFF . --utf16 --utf-16 -x UTF16 . re2ocaml 0 -- 0x10FFFF . --nested-ifs . --utf32 --unicode -u UTF32 . re2ocaml 0 -- 0x10FFFF . --nested-ifs . --verbose . --vernum -V MMmmpp ( ). --version -v . --single-pass -1 . ( ). --debug-output -d YYDEBUG . . --dump-adfa : DFA (tunneling) ( .dot). --dump-cfg : ( .dot). --dump-closure-stats : (closure). --dump-dfa-det : DFA (determinization) ( .dot). --dump-dfa-min : DFA (minimization) ( .dot). --dump-dfa-tagopt : DFA ( .dot). --dump-dfa-tree : DFA ( .dot). --dump-dfa-raw : DFA ( .dot). --dump-interf : (liveness analysis) . --dump-nfa : NFA ( .dot). --emit-dot -D .dot . Graphviz ( dot -Tpng -odfa.png dfa.dot). --dfa-minimization : DFA re2ocaml. moore (Moore) ( ). table " " (table filling) . DFA . --eager-skip : (eagerly) -- . (lazily) -- -- . --no-lookahead . TDFA(0) . TDFA(1) TDFA(0) (one-symbol lookahead) . . TDFA(0) TDFA(1) . --no-optimize-tags : ( ). --posix-closure : (epsilon-closure) POSIX: gor1 () - (Goldberg-Radzik) gtop " " (global topological order) . --posix-prectable : POSIX. complex TNFA . naive TNFA complex ..TP --stadfa . staDFA TDFA . staDFA TDFA . --fixed-tags : (fixed-tag) (all) (none) (toplevel). all . "" ( "") . . (no-match) ( no-match ). . (Warnings) . -W . -Werror . . -W warning. -Wno- warning. -Werror- warning ( -W ). -Wno-error- warning . . -Wcondition-order . --header conditions . -Wempty-character-class . : . re2ocaml . --empty-class . -Wmatch-empty-string ( ). . -Wswapped-range . . -Wundefined-control-flow ( ). . * . -Wunreachable-rules (shadowed) . -Wdeprecated-eof_rule $ . $ . -Wuseless-escape ( escape ). re2ocaml (escape sequence) . -Wnondeterministic-tags n n . -Wsentinel-in-midrule --- . . re2ocaml:sentinel . -Wundefined-syntax-config --syntax . : . . re2ocaml syntax files ( ) . -- . re2ocaml (Syntax files) re2c . -- . re2c re2c: : . ( include/syntax ) re2ocaml . ( --lang none ). : . (Feature lists) re2ocaml : supported_apis API simple record generic. supported_api_styles API functions free-form. supported_code_models goto-label loop-switch recursive-functions. supported_targets code dot skeleton. supported_features nested-ifs bitmaps computed-gotos case-ranges monadic unsafe tags captures captvars. (Language configurations) re2ocaml : semicolons (semicolon) . backtick_quoted_strings (backtick) . single_quoted_strings (single-quote) . indentation_sensitive . wrap_blocks_in_braces . (Inplace configurations) re2c: . . (Code templates) . - : code-template :: name '=' code-exprs ';' | CODE_TEMPLATE ';' | '' ';' code-exprs :: | code-exprs code-expr code-expr :: STRING | VARIABLE | optional | list optional :: '(' CONDITIONAL '?' code-exprs ')' | '(' CONDITIONAL '?' code-exprs ':' code-exprs ')' list :: '[' VARIABLE ':' code-exprs ']' | '[' VARIABLE '{' NUMBER '}' ':' code-exprs ']' | '[' VARIABLE '{' NUMBER ',' NUMBER '}' ':' code-exprs ']' . . : re2ocaml . ( -1 ). . ( ) ( ) . re2ocaml . if-then-else C : code:if_then_else = [branch{0}: topindent "if " cond " {" nl indent [stmt: stmt] dedent] [branch{1:-1}: topindent "} else" (.cond ? " if " cond) " {" nl indent [stmt: stmt] dedent] topindent "}" nl; branch : branch{0} ( else ) branch{1:-1} ( ) . stmt : [stmt: stmt] . topindent indent dedent nl .cond ( ). : if x { // do something } else if y { // do something else } else { // don't do anything } re2ocaml . . . code:var_local . : type ( ) name ( ) init ( ). : .init ( ). code:var_global code:var_local top-level . code:const_local . : type ( ) name ( ) init ( ). code:const_global code:const_local top-level . code:array_local () . : type ( ) name ( ) size ( ) row ( ) elem ( -- row ). : .const ( ). code:array_global code:array_local top-level . code:array_elem (). : array ( ) index ( ). code:enum enum ( ). : type ( ) elem ( ) init ( ). : .init ( ). code:enum_elem ( code:enum). : name ( ) type ( ). code:assign . : lhs ( ) rhs ( ). code:type_int (signed). code:type_uint (unsigned). code:type_yybm yybm. code:type_yytarget yytarget. code:type_yyctable yyctable. code:cmp_eq " " (equals). code:cmp_ne " " (not equals). code:cmp_lt " " (less than). code:cmp_gt " " (greater than). code:cmp_le " " (less or equal). code:cmp_ge " " (greater or equal). code:if_then_else if-then-else . : branch ( ) cond ( ) stmt ( ). : .cond ( ) .many ( ). code:if_then_else_oneline code:if_then_else . code:if_then_else . code:switch switch (case). : expr ( switch) case ( case ). code:switch_cases switch . : case ( ) stmt ( ). code:switch_cases_oneline code:switch_cases . code:switch_cases . code:switch_case_range switch ( ). : val ( ). : .many ( ) .char_literals ( switch -- ). code:switch_case_default (default) switch. code:loop ( ). : label ( ) stmt ( ). code:continue continue. : label ( ). code:goto goto. : label ( ). code:cgoto goto (computed goto). : array ( goto ) index ( ) base ( .cgoto.relative ). code:cgoto:data goto . : label ( ) type ( ) base ( - .cgoto.relative ). code:fndecl . : name ( ) type ( ) throw ( re2c:yyfn:throw ) arg ( ) argname ( ) argtype ( ). : .type ( void ). code:fndef code:fndecl stmt . code:fncall . : name ( ) retval ( l-value ) arg ( ). : .args ( ) .retval ( ). code:tailcall (tail call). : name ( ) arg ( ). : .args ( ) .retval ( void ). code:recursive_functions --recursive-functions. : fn ( ) fndecl ( ) fndef ( ). code:fingerprint (fingerprint) . : ver ( re2ocaml ) date ( ). code:line_info (line directives) ( ). : line ( ) file ( ). code:abort (abort) . code:yydebug YYDEBUG (API) . : YYDEBUG yyrecord yych ( re2c:) state ( DFA)..TP code:yypeek YYPEEK API . : YYPEEK YYCTYPE YYINPUT YYCURSOR yyrecord yych ( re2c: ). : .cast ( re2c:yych:conversion ). code:yyskip YYSKIP API . : YYSKIP YYCURSOR yyrecord ( re2c: ). code:yybackup YYBACKUP API . : YYBACKUP YYCURSOR YYMARKER yyrecord ( re2c: ). code:yybackupctx YYBACKUPCTX API . : YYBACKUPCTX YYCURSOR YYCTXMARKER yyrecord ( re2c: ). code:yyskip_yypeek code:yyskip code:yypeek ( code:yyskip code:yypeek ). code:yypeek_yyskip code:yypeek code:yyskip ( code:yypeek code:yyskip ). code:yyskip_yybackup code:yyskip code:yybackup ( code:yyskip code:yybackup ). code:yybackup_yyskip code:yybackup code:yyskip ( code:yybackup code:yyskip ). code:yybackup_yypeek code:yybackup code:yypeek ( code:yybackup code:yypeek ). code:yyskip_yybackup_yypeek code:yyskip code:yybackup code:yypeek ( ``code:yyskip`` code:yybackup code:yypeek ). code:yybackup_yypeek_yyskip code:yybackup code:yypeek code:yyskip ( ``code:yybackup`` code:yypeek code:yyskip ). code:yyrestore YYRESTORE API . : YYRESTORE YYCURSOR YYMARKER yyrecord ( re2c: ). code:yyrestorectx YYRESTORECTX API . : YYRESTORECTX YYCURSOR YYCTXMARKER yyrecord ( re2c: ). code:yyrestoretag YYRESTORETAG API . : YYRESTORETAG YYCURSOR yyrecord ( re2c: ) tag ( ). code:yyshift YYSHIFT API . : YYSHIFT YYCURSOR yyrecord ( re2c: ) offset ( ). code:yyshiftstag YYSHIFTSTAG API . : YYSHIFTSTAG yyrecord negative ( re2c: ) tag ( ) offset ( ). : .nested ( -- re2c:tags:negative ). code:yyshiftmtag YYSHIFTMTAG API . : YYSHIFTMTAG ( re2c: ) tag ( ) offset ( ). code:yystagp YYSTAGP API . : YYSTAGP YYCURSOR yyrecord ( re2c: ) tag ( ). code:yymtagp YYMTAGP API . : YYMTAGP ( re2c: ) tag ( ). code:yystagn YYSTAGN API . : YYSTAGN negative yyrecord ( re2c: ) tag ( ). code:yymtagn YYMTAGN API . : YYMTAGN ( re2c: ) tag ( ). code:yycopystag YYCOPYSTAG API . : YYCOPYSTAG yyrecord ( re2c: ) lhs rhs ( ). code:yycopymtag YYCOPYMTAG API . : YYCOPYMTAG yyrecord ( re2c: ) lhs rhs ( ). code:yygetaccept YYGETACCEPT API . : YYGETACCEPT yyrecord ( re2c: ) var ( re2c:yyaccept ). code:yysetaccept YYSETACCEPT API . : YYSETACCEPT yyrecord ( re2c: ) var ( re2c:yyaccept ) val ( ). code:yygetcond YYGETCOND API . : YYGETCOND yyrecord ( re2c: ) var ( re2c:yycond ). code:yysetcond YYSETCOND API . : YYSETCOND yyrecord ( re2c: ) var ( re2c:yycond ) val ( ). code:yygetstate YYGETSTATE API . : YYGETSTATE yyrecord ( re2c: ) var ( re2c:yystate ). code:yysetstate YYSETSTATE API . : YYSETSTATE yyrecord ( re2c: ) var ( re2c:yystate ) val ( ). code:yylessthan YYLESSTHAN API . : YYLESSTHAN YYCURSOR YYLIMIT yyrecord ( re2c: ) need ( ). : .many ( need ). code:yyend YYEND API . : YYEND YYCURSOR YYLIMIT. code:yybm_filter yych yybm ( --bitmaps ). : yych ( re2c:yych ). code:yybm_match yybm ( --bitmaps). : yybm yych ( re2c: ) offset ( yybm yych ) mask ( ). code:yytarget_filter yych yytarget ( --computed-gotos ). : yych ( re2c:yych ). (syntax files) : nl (newline). indent ..TP dedent . topindent . . (syntax) : .api.simple API (--api simple re2c:api = simple). .api.generic API (--api generic re2c:api = generic). .api.record API (--api record re2c:api = record). .api_style.functions API (re2c:api-style = functions). .api_style.freeform API (free-form) (re2c:api-style = free-form). .case_ranges case (--case-ranges re2c:case-ranges = 1). .cgoto.relative goto (--computed-gotos-relative re2c:cgoto:relative = 1). .code_model.goto_label goto/label (--goto-label). .code_model.loop_switch loop/switch (--loop-switch). .code_model.recursive_functions (--recursive-function). .date (fingerprint) . .loop_label re2ocaml (re2c:label:yyloop ). .monadic (re2c:monadic = 1). . .start_conditions (start conditions) (--start-conditions). .storable_state (--storable-state). .unsafe re2ocaml "unsafe" (--unsafe re2c:unsafe = 1). "unsafe" . .version (fingerprint) re2ocaml . .yyfill.enable YYFILL (re2c:yyfill:enable = 1). .yyfn.throw re2c:yyfn:throw . (HANDLING THE END OF INPUT) . : o ( *) o o : . : . . re2ocaml . . : o (Sentinel). . <<>> . . : . . YYFILL re2c:yyfill:enable = 0; re2c:eof -1 . o (Sentinel with bounds checks). : . . <<>> <<>> . ( ). . . YYFILL ( ). . . re2c:eof ( ). YYFILL . o (Bounds checks with padding). << >> . DFA (SCC) SCC SCC . . (padding) ( ) . YYMAXFILL max . YYFILL . YYFILL re2c:eof -1 ( ). o (Custom checks). (Generic API) . . API --api custom re2c:api = custom; re2c:yyfill:enable = 0; re2c:yyfill:check = 0; . . (Sentinel) . null . null : ( ). null * ( %-Wsentinel-in-midrule re2c:sentinel ). re2c:yyfill:enable = 0; YYFILL . (* re2ocaml -o *) open String type state = { yyinput: string; mutable yycursor: int; } (* expect a null-terminated string *) %{ re2c:YYFN = ["lex;int", "yyrecord;state", "count;int"]; re2c:yyfill:enable = 0; * { -1 } [\x00] { count } [a-z]+ { lex yyrecord (count + 1) } [ ]+ { lex yyrecord count } %} let test(yyinput, count) = let st = {yyinput = yyinput; yycursor = 0} in if not (lex st 0 = count) then raise (Failure "error") let main () = test("", 0); test("one two three", 3); test("f0ur", -1) let _ = main () (Sentinel with bounds checks) ( 1.2 ). . null re2c:eof = 0; . % null ( 'aaa\0aa'\0 'aaa\0 ). null . API YYLIMIT <= YYCURSOR YYLESSTHAN(1) . (YYFILL re2c:yyfill:enable = 0; YYFILL %YYFILL ). : $ ( *) %-Wundefined-control-flow . (* re2ocaml -o *) open String type state = { yyinput: string; mutable yycursor: int; mutable yymarker: int; yylimit: int; } (* expect a null-terminated string *) %{ re2c:YYFN = ["lex;int", "yyrecord;state", "count;int"]; re2c:yyfill:enable = 0; re2c:eof = 0; str = ['] ([^'\\] | [\\][^])* [']; * { -1 } $ { count } str { lex yyrecord (count + 1) } [ ]+ { lex yyrecord count } %} let test(str, count) = let st = { yyinput = str; yycursor = 0; yymarker = 0; yylimit = length str - 1; (* terminating null not included *) } in if not (lex st 0 = count) then raise (Failure "error") let main () = test("", 0); test("'qu\x00tes' 'are' 'fine: \\'' ", 3); test("'unterminated\\'", -1) let _ = main () (Bounds checks with padding) () ( ). . null YYMAXFILL YYMAXFILL max . null -- ( ). <<>> (null) ( null ). . API (YYLIMIT - YYCURSOR) < n YYLESSTHAN(n) n ( n ). YYFILL(n) n . YYFILL ( ). YYFILL %YYFILL . (* re2ocaml -o *) open String exception Fill type state = { yyinput: string; mutable yycursor: int; yylimit: int; } %{max %} %{ re2c:YYFN = ["lex;int", "yyrecord;state", "count;int"]; re2c:YYFILL = "raise Fill;"; str = ['] ([^'\\] | [\\][^])* [']; [\x00] { (* check that it is the sentinel, not some unexpected null *) if yyrecord.yycursor = length yyrecord.yyinput - yymaxfill + 1 then count else -1 } str { lex yyrecord (count + 1) } [ ]+ { lex yyrecord count } * { -1 } %} let test(str, count) = let buf = cat str (make yymaxfill '\x00') in let st = {yyinput = buf; yycursor = 0; yylimit = length buf} in let result = try lex st 0 with Fill -> -1 in if not (result = count) then raise (Failure "error") let main () = test("", 0); test("'unterminated\\'", -1); test("'qu\x00tes' 'are' 'fine: \\'' ", 3); test("'unexpected \x00 null", -1) let _ = main () (Custom checks) API . . % null . YYPEEK . . YYPEEK . (* re2ocaml -o *) type state = { str: string; mutable cur: int; lim: int; } (* expect a string without terminating null *) %{ re2c:api = generic; re2c:YYFN = ["lex;int", "st;state", "count;int"]; re2c:YYPEEK = "if st.cur < st.lim then st.str.[st.cur] else '\\x00'"; re2c:YYSKIP = "st.cur <- st.cur + 1;"; re2c:yyfill:enable = 0; * { -1 } [\x00] { count } [a-z]+ { lex st (count + 1) } [ ]+ { lex st count } %} let test(str, count) = let st = {str = str; cur = 0; lim = String.length str} in if not (lex st 0 = count) then raise (Failure "error") let main () = test("", 0); test("one two three", 3); test("f0ur", -1) let _ = main () (BUFFER REFILLING) : ( ) . . . : o (cursor): (YYCURSOR API C YYSKIP/YYPEEK API ) o (limit): (YYLIMIT API C YYLESSTHAN API ) o (marker): (YYMARKER API YYBACKUP/YYRESTORE API ) o (token): ( API re2ocaml ) o (context marker): (YYCTXMARKER API C YYBACKUPCTX/YYRESTORECTX API ) o (tag variables): ( stags mtags API YYSTAGP/YYSTAGN/YYMTAGP/YYMTAGN) YYFILL . token cursor token token limit . YYFILL ( ). YYFILL EOF : EOF . -f --storable-state YYFILL ( ). YYFILL (YYFILL with sentinel) EOF YYFILL . YYFILL YYLIMIT <= YYCURSOR API C YYLESSTHAN() API . YYFILL . YYFILL . Limit limit re2c:eof . YYFILL ( # ). <-- shift --> >-A------------B---------C-------------D#-----------E-> buffer token marker limit, cursor >-A------------B---------C-------------D------------E#-> buffer, marker cursor limit token <-- shift --> >-A------------B---------C-------------D#--E (EOF) buffer token marker limit, cursor >-A------------B---------C-------------D---E#........ buffer, marker cursor limit token Here is an example of a program that reads input file input.txt in chunks of 4096 bytes and uses EOF rule. (* re2ocaml $INPUT -o $OUTPUT *) open Bytes let bufsize = 4096 type state = { file: in_channel; yyinput: bytes; mutable yycursor: int; mutable yymarker: int; mutable yylimit: int; mutable token: int; mutable eof: bool; } type status = Ok | Eof | LongLexeme let fill(st: state) : status = if st.eof then Eof else (* Error: lexeme too long. In real life could reallocate a larger buffer. *) if st.token < 1 then LongLexeme else ( (* Shift buffer contents (discard everything up to the current token). *) blit st.yyinput st.token st.yyinput 0 (st.yylimit - st.token); st.yycursor <- st.yycursor - st.token; st.yymarker <- st.yymarker - st.token; st.yylimit <- st.yylimit - st.token; st.token <- 0; (* Fill free space at the end of buffer with new data from file. *) let n = input st.file st.yyinput st.yylimit (bufsize - st.yylimit - 1) in (* -1 for sentinel *) st.yylimit <- st.yylimit + n; if n = 0 then st.eof <- true; (* end of file *) set st.yyinput st.yylimit '\x00'; (* append sentinel *) Ok) %{ re2c:YYFN = ["lex;int", "yyrecord;state", "count;int"]; re2c:YYFILL = "fill yyrecord = Ok"; re2c:eof = 0; str = ['] ([^'\\] | [\\][^])* [']; * { -1 } $ { count } str { lex_loop yyrecord (count + 1) } [ ]+ { lex_loop yyrecord count } %} and lex_loop st count = st.token <- st.yycursor; lex st count let main () = let fname = "input" in (* Prepare input file. *) Out_channel.with_open_bin fname (fun oc -> for i = 1 to bufsize do output_string oc "'qu\x00tes' 'are' 'fine: \\'' " done); (* Run lexer on the prepared file. *) In_channel.with_open_bin fname (fun ic -> let yylimit = bufsize - 1 in let st = { file = ic; yyinput = create bufsize; yycursor = yylimit; yymarker = yylimit; yylimit = yylimit; token = yylimit; eof = false; } in if not (lex_loop st 0 = 3 * bufsize) then raise (Failure "error")); (* Cleanup. *) Sys.remove fname let _ = main () YYFILL with padding In the default case (when EOF rule is not used) YYFILL is a function-like primitive that accepts a single argument and does not return any value. YYFILL invocation is triggered by condition (YYLIMIT - YYCURSOR) < n in C pointer API and YYLESSTHAN(n) in generic API. The argument passed to YYFILL is the minimal number of characters that must be supplied. If it fails to do so, YYFILL must not return to the lexer (for that reason it is best implemented as a macro that returns from the calling function on failure). In case of a successful YYFILL invocation the limit position must be set either to one after the last input position in buffer, or to the end of YYMAXFILL padding (in case YYFILL has successfully read at least n characters, but not enough to fill the entire buffer). The pictures below show the relative locations of input positions in buffer before and after YYFILL invocation (YYMAXFILL padding on the second picture is marked with # symbols). <-- shift --> <-- need --> >-A------------B---------C-----D-------E---F--------G-> buffer token marker cursor limit >-A------------B---------C-----D-------E---F--------G-> buffer, marker cursor limit token <-- shift --> <-- need --> >-A------------B---------C-----D-------E-F (EOF) buffer token marker cursor limit >-A------------B---------C-----D-------E-F############### buffer, marker cursor limit token <- YYMAXFILL -> Here is an example of a program that reads input file input.txt in chunks of 4096 bytes and uses bounds-checking with padding. (* re2ocaml $INPUT -o $OUTPUT *) open Bytes %{max %} let bufsize = 4096 exception Fill type state = { file: in_channel; yyinput: bytes; mutable yycursor: int; mutable yymarker: int; mutable yylimit: int; mutable token: int; mutable eof: bool; } type status = Ok | Eof | LongLexeme let fill (st: state) (need: int) : status = if st.eof then Eof else (* Error: lexeme too long. In real life could reallocate a larger buffer. *) if st.token < need then LongLexeme else ( (* Shift buffer contents (discard everything up to the current token). *) blit st.yyinput st.token st.yyinput 0 (st.yylimit - st.token); st.yycursor <- st.yycursor - st.token; st.yymarker <- st.yymarker - st.token; st.yylimit <- st.yylimit - st.token; st.token <- 0; (* Fill free space at the end of buffer with new data from file. *) let n = input st.file st.yyinput st.yylimit (bufsize - st.yylimit - 1) in (* -1 for sentinel *) st.yylimit <- st.yylimit + n; (* If read zero characters, this is end of input => add zero padding so that the lexer can access characters at the end of buffer. *) if n = 0 then st.eof <- true; (* end of file *) for i = 0 to (yymaxfill - 1) do set st.yyinput (st.yylimit + i) '\x00'; st.yylimit <- st.yylimit + yymaxfill done; Ok) %{ re2c:YYFN = ["lex;int", "yyrecord;state", "count;int"]; re2c:YYFILL = "if not (fill yyrecord @@ = Ok) then raise Fill;"; str = ['] ([^'\\] | [\\][^])* [']; [\x00] { (* check that it is the sentinel, not some unexpected null *) if yyrecord.token = yyrecord.yylimit - yymaxfill then count else -1 } str { lex_loop yyrecord (count + 1) } [ ]+ { lex_loop yyrecord count } * { -1 } %} and lex_loop st count = st.token <- st.yycursor; try lex st count with Fill -> -1 let main () = let fname = "input" in (* Prepare input file. *) Out_channel.with_open_bin fname (fun oc -> for i = 1 to bufsize do output_string oc "'qu\x00tes' 'are' 'fine: \\'' " done); (* Run lexer on the prepared file. *) In_channel.with_open_bin fname (fun ic -> let yylimit = bufsize - yymaxfill in let st = { file = ic; yyinput = create bufsize; yycursor = yylimit; yymarker = yylimit; yylimit = yylimit; token = yylimit; eof = false; } in if not (lex_loop st 0 = 3 * bufsize) then raise (Failure "error")); (* Cleanup. *) Sys.remove fname let _ = main () input.txt EOF . YYFILL (YYFILL with padding) ( EOF ) YYFILL . YYFILL (YYLIMIT - YYCURSOR) < n API C YYLESSTHAN(n) API . YYFILL . YYFILL ( ). YYFILL limit YYMAXFILL ( YYFILL n ). YYFILL ( YYMAXFILL # ). input.txt . (FEATURES) (Multiple blocks) ( ). re2ocaml . % (start conditions) . . goto ( C/C++ Go) (label) . goto . goto switch . ( ) ( ) ( ) ( ) . . . . (* re2ocaml $INPUT -o $OUTPUT -i *) open Int64 open Option open String type state = { yyinput: string; mutable yycursor: int; mutable yymarker: int; } let add (num: int option) (dgt: int) (base: int) : int option = match num with | None -> None | Some n -> let n' = add (mul (of_int n) (of_int base)) (of_int dgt) in if n' > (of_int32 Int32.max_int) then None else Some (to_int n') %{ re2c:yyrecord = "st"; re2c:yyfill:enable = 0; %} %{local re2c:YYFN = ["parse_bin;int option", "st;state", "num;int option"]; [01] { parse_bin st (add num (Char.code st.yyinput.[st.yycursor - 1] - 48) 2) } * { num } %} %{local re2c:YYFN = ["parse_oct;int option", "st;state", "num;int option"]; [0-7] { parse_oct st (add num (Char.code st.yyinput.[st.yycursor - 1] - 48) 8) } * { num } %} %{local re2c:YYFN = ["parse_dec;int option", "st;state", "num;int option"]; [0-9] { parse_dec st (add num (Char.code st.yyinput.[st.yycursor - 1] - 48) 10) } * { num } %} %{local re2c:YYFN = ["parse_hex;int option", "st;state", "num;int option"]; [0-9] { parse_hex st (add num (Char.code st.yyinput.[st.yycursor - 1] - 48) 16) } [a-f] { parse_hex st (add num (Char.code st.yyinput.[st.yycursor - 1] - 87) 16) } [A-F] { parse_hex st (add num (Char.code st.yyinput.[st.yycursor - 1] - 55) 16) } * { num } %} %{local re2c:YYFN = ["parse;int option", "st;state"]; '0b' / [01] { parse_bin st (Some 0) } "0" { parse_oct st (Some 0) } "" / [1-9] { parse_dec st (Some 0) } '0x' / [0-9a-fA-F] { parse_hex st (Some 0) } * { None } %} let test (yyinput: string) (result: int option) = let st = {yyinput = yyinput; yycursor = 0; yymarker = 0} in if not (parse st = result) then raise (Failure "error") let main () = test "\x00" None; test "1234567890\x00" (Some 1234567890); test "0b1101\x00" (Some 13); test "0x7Fe\x00" (Some 2046); test "0644\x00" (Some 420); test "9999999999\x00" None let _ = main () (Start conditions) --start-conditions . re2ocaml . re2ocaml . : YYGETCOND YYSETCOND . conditions --header re2c:header . re2c:cond:enumprefix . (< >) <*> . : < condition-list > regular-expression code condition-list . regular-expression code . < condition-list > regular-expression => condition code condition-list . regular-expression condition code . < condition-list > regular-expression :=> condition condition-list . regular-expression condition ( ). < condition-list > !action code code action condition-list ( %actions ). code code condition-list . !pre_rule ( ). < > code "0" code . !entry ( ). < > => condition code condition . < > :=> condition condition . re2ocaml re2ocaml goto/label loop/switch . goto ( C/C++ Go) yyc_ cond ( re2c:cond:prefix ). goto . re2ocaml switch YYGETSTATE . :=> switch ( re2c:cond:goto ). . goto ( Rust) re2ocaml yystate . switch DFA . switch . ( YYGETCOND/YYSETCOND ). . . cond conditions . (* re2ocaml $INPUT -o $OUTPUT -ci *) open Int64 open Option open String %{conditions %} type state = { yyinput: string; mutable yycursor: int; mutable yymarker: int; mutable yycond: yycondtype; } let add (num: int option) (dgt: int) (base: int) : int option = match num with | None -> None | Some n -> let n' = add (mul (of_int n) (of_int base)) (of_int dgt) in if n' > (of_int32 Int32.max_int) then None else Some (to_int n') %{ re2c:YYFN = ["parse;int option", "st;state", "num;int option"]; re2c:yyrecord = "st"; re2c:yyfill:enable = 0; '0b' / [01] :=> bin "0" :=> oct "" / [1-9] :=> dec '0x' / [0-9a-fA-F] :=> hex * { None } [01] { yyfnbin st (add num (Char.code st.yyinput.[st.yycursor - 1] - 48) 2) } [0-7] { yyfnoct st (add num (Char.code st.yyinput.[st.yycursor - 1] - 48) 8) } [0-9] { yyfndec st (add num (Char.code st.yyinput.[st.yycursor - 1] - 48) 10) } [0-9] { yyfnhex st (add num (Char.code st.yyinput.[st.yycursor - 1] - 48) 16) } [a-f] { yyfnhex st (add num (Char.code st.yyinput.[st.yycursor - 1] - 87) 16) } [A-F] { yyfnhex st (add num (Char.code st.yyinput.[st.yycursor - 1] - 55) 16) } * { num } %} let test (yyinput: string) (result: int option) = let st = {yyinput = yyinput; yycursor = 0; yymarker = 0; yycond = YYC_init} in if not (parse st (Some 0) = result) then raise (Failure "error") let main () = test "\x00" None; test "1234567890\x00" (Some 1234567890); test "0b1101\x00" (Some 13); test "0x7Fe\x00" (Some 2046); test "0644\x00" (Some 420); test "9999999999\x00" None let _ = main () (Storable state) --storable-state re2ocaml . re2ocaml <<>> ("pull") <<>> ( ). ( (parser) ). : <<>> ("push") . . . <<>> : o YYSETSTATE() YYGETSTATE(state) . o yych yyaccept ( ) state (persistent) . state -1 . o YYFILL . . o . o getstate YYGETSTATE . goto ( --loop-switch). o re2c:eof %sentinel with bounds checks ( ) . (padding) : (lexeme) . ( ). <<>> . . <<>> . . (* re2ocaml $INPUT -o $OUTPUT -fi *) open Bytes (* Use a small buffer to cover the case when a lexeme doesn't fit. In real world use a larger buffer. *) let bufsize = 10 let debug = false let log format = (if debug then Printf.eprintf else Printf.ifprintf stderr) format type state = { file: in_channel; yyinput: bytes; mutable yycursor: int; mutable yymarker: int; mutable yylimit: int; mutable token: int; mutable yystate: int; mutable recv: int; } type status = End | Ready | Waiting | BadPacket | BigPacket let fill(st: state) : status = (* Error: lexeme too long. In real life could reallocate a larger buffer. *) if st.token < 1 then BigPacket else ( (* Shift buffer contents (discard everything up to the current token). *) blit st.yyinput st.token st.yyinput 0 (st.yylimit - st.token); st.yycursor <- st.yycursor - st.token; st.yymarker <- st.yymarker - st.token; st.yylimit <- st.yylimit - st.token; st.token <- 0; (* Fill free space at the end of buffer with new data from file. *) let n = In_channel.input st.file st.yyinput st.yylimit (bufsize - st.yylimit - 1) in st.yylimit <- st.yylimit + n; set st.yyinput st.yylimit '\x00'; (* append sentinel *) Ready) %{ re2c:YYFN = ["lex;status", "yyrecord;state"]; re2c:YYFILL = "Waiting"; re2c:eof = 0; packet = [a-z]+[;]; * { BadPacket } $ { End } packet { yyrecord.recv <- yyrecord.recv + 1; lex_loop yyrecord } %} and lex_loop st = st.token <- st.yycursor; lex st let test (packets: string list) (sts: status) = let fname = "pipe" in let oc = Out_channel.open_bin fname in let ic = In_channel.open_bin fname in let yylimit = bufsize - 1 in let st = { file = ic; (* Sentinel (at `yylimit` offset) is set to null, which triggers YYFILL. *) yyinput = create bufsize; yycursor = yylimit; yymarker = yylimit; yylimit = yylimit; token = yylimit; yystate = -1; recv = 0; } in let rec loop packets = match lex_loop st with | End -> log "done: got %d packets\n" st.recv; End | Waiting -> log "waiting...\n"; let packets' = match packets with | [] -> [] | p :: ps -> log "sent packet '%s'\n" p; Out_channel.output_string oc p; Out_channel.flush oc; (* without `flush` write happens too late *) ps in (match fill st with | BigPacket -> log "error: packet too big\n"; BigPacket | Ready -> loop packets' | _ -> raise (Failure "unexpected status after fill")) | BadPacket -> log "error: ill-formed packet\n"; BadPacket | _ -> raise (Failure "unexpected status") in if not (loop packets = sts) then raise (Failure "error"); In_channel.close ic; Out_channel.close oc; Sys.remove fname let main () = test [] End; test ["zero;"; "one;"; "two;"; "three;"; "four;"] End; test ["zer0;"] BadPacket; test ["goooooooooogle;"] BigPacket let _ = main () (Reusable blocks) /*!rules:re2c[:] ... */ %{rules[:] ... %} re2ocaml . . (rules block) use . . use /*!use:re2c[:] ... */ %{use[:] ... %} . : ( ). use . use (lexer) : ( ). !use:; re2ocaml . . : ( ). * $ ( ) . use re2ocaml ( ). use 2.2 re2ocaml . ( ). 2.2 -r --reusable . 1.2 . !use (* re2ocaml $INPUT -o $OUTPUT *) (* This example shows how to combine reusable re2c blocks: two blocks ('colors' and 'fish') are merged into one. The 'salmon' rule occurs in both blocks; the 'fish' block takes priority because it is used earlier. Default rule * occurs in all three blocks; the local (not inherited) definition takes priority. *) open String type answer = Color | Fish | Dunno type state = { yyinput: string; mutable yycursor: int; mutable yymarker: int; } %{rules:colors * { raise (Failure "ah"); } "red" | "salmon" | "magenta" { Color } %} %{rules:fish * { raise (Failure "oh"); } "haddock" | "salmon" | "eel" { Fish } %} %{ re2c:YYFN = ["lex;answer", "yyrecord;state"]; re2c:yyfill:enable = 0; !use:fish; !use:colors; * { Dunno } // overrides inherited '*' rules %} let test(str, ans) = let st = {yyinput = str; yycursor = 0; yymarker = 0} in if not (lex st = ans) then raise (Failure "error") let main () = test("salmon", Fish); test("what?", Dunno) let _ = main () /*!use:re2c ... */ (* re2ocaml $INPUT -o $OUTPUT --input-encoding utf8 *) (* This example supports multiple input encodings: UTF-8 and UTF-32. Both lexers are generated from the same rules block, and the use blocks add only encoding-specific configurations. *) open Array type 'a state = { yyinput: 'a array; mutable yycursor: int; mutable yymarker: int; } %{rules re2c:yyfill:enable = 0; "x y" { Some yyrecord.yycursor } * { None } %} %{use re2c:YYFN = ["lex8;int option", "yyrecord;char state"]; re2c:encoding:utf8 = 1; %} %{use re2c:YYFN = ["lex32;int option", "yyrecord;int state"]; re2c:encoding:utf32 = 1; %} let main() = let st8 = { yyinput = [|'\xe2'; '\x08'; '\x80'; '\x78'; '\x20'; '\xe2'; '\x88'; '\x83'; '\x79'|]; yycursor = 0; yymarker = 0; } in if not (lex8 st8 = Some (Array.length st8.yyinput)) then raise (Failure "error"); let st32 = { yycursor = 0; yymarker = 0; yyinput = [|0x2200; 0x78; 0x20; 0x2203; 0x79|]; } in if not (lex32 st32 = Some (Array.length st32.yyinput)) then raise (Failure "error"); (Submatch extraction) re2ocaml . Tags tags () @stag #mtag stag mtag . -T --tags re2c:tags = 1 . : ( ) . S-tags m-tags ( ). svars mvars. (leftmost greedy) . Captures capturing groups ( ) . --captures re2c:captures = 1 . --leftmost-captures () --posix-captures POSIX . (bang) : (! ... ). --invert-captures re2c:invert-captures = 1 . yynmatch ( ) yypmatch . yynmatch yypmatch yypmatch [yynmatch * 2] . maxnmatch YYMAXNMATCH yynmatch . Captvars --captvars re2c:captvars = 1 . --captures : yynmatch yypmatch re2ocaml yytl yytr k- ( svars ). : --leftmost-captvars re2c:leftmost-captvars = 1 ( ) --posix-captvars re2c:posix-captvars POSIX. Tagged Deterministic Finite Automata with Lookahead . TDFA (overhead) . ( ) TDFA DFA . . TDFA tag variables ( ) : . . stags mtags . . YYFILL . S-tags : o s-tag: t = YYCURSOR API C YYSTAGP(t) API (generic) o s-tag: t = NULL API C YYSTAGN(t) API o s-tag : t1 = t2 M-tags : o m-tag: YYMTAGP(t) API o m-tag: YYMTAGN(t) API o m-tag : t1 = t2 S-tag ( ) . M-tag . m-tag ( ) m-tag (prefix-tree) (v, p) v p . s-tag (semantic versions) : major minor patch ( ). YYFILL . (* re2ocaml $INPUT -o $OUTPUT *) open String type state = { yyinput: string; mutable yycursor: int; mutable yymarker: int; (* Final tag variables available in semantic action. *) %{svars format = "\n\tmutable @@{tag}: int;"; %} (* Intermediate tag variables used by the lexer (must be autogenerated). *) %{stags format = "\n\tmutable @@{tag}: int;"; %} } type semver = { major: int; minor: int; patch: int; } let s2n (str: string) (i1: int) (i2: int) : int = let rec f s i j n = if i >= j then n else f s (i + 1) j (n * 10 + Char.code s.[i] - 48) in f str i1 i2 0 %{local re2c:YYFN = ["parse;semver option", "st;state"]; re2c:yyrecord = "st"; re2c:tags = 1; re2c:yyfill:enable = 0; num = [0-9]+; @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\x00] { Some { major = s2n st.yyinput st.t1 st.t2; minor = s2n st.yyinput st.t3 st.t4; patch = if st.t5 = -1 then 0 else s2n st.yyinput st.t5 (st.yycursor - 1) } } * { None } %} let test (str: string) (result: semver option) = let st = { yyinput = str; yycursor = 0; yymarker = 0; %{svars format = "\n\t\t@@{tag} = -1;"; %} %{stags format = "\n\t\t@@{tag} = -1;"; %} } in if not (parse st = result) then raise (Failure "error") let main () = test "23.34" (Some {major = 23; minor = 34; patch = 0}); test "1.2.99999" (Some {major = 1; minor = 2; patch = 99999}); test "1.a" None let _ = main () s-tag YYFILL . YYFILL . s-tag . ( m-tag ) . (* re2ocaml $INPUT -o $OUTPUT *) open Bytes let bufsize = 4096 type state = { file: in_channel; yyinput: bytes; mutable yycursor: int; mutable yymarker: int; mutable yylimit: int; mutable token: int; mutable eof: bool; (* Final tag variables available in semantic action. *) %{svars format = "\n\tmutable @@{tag}: int;"; %} (* Intermediate tag variables used by the lexer (must be autogenerated). *) %{stags format = "\n\tmutable @@{tag}: int;"; %} } type status = Ok | Eof | LongLexeme type semver = { major: int; minor: int; patch: int; } let s2n (str: bytes) (i1: int) (i2: int) : int = let rec f s i j n = if i >= j then n else f s (i + 1) j (n * 10 + Char.code (get s i) - 48) in f str i1 i2 0 let fill(st: state) : status = if st.eof then Eof else (* Error: lexeme too long. In real life could reallocate a larger buffer. *) if st.token < 1 then LongLexeme else ( (* Shift buffer contents (discard everything up to the current token). *) blit st.yyinput st.token st.yyinput 0 (st.yylimit - st.token); st.yycursor <- st.yycursor - st.token; st.yymarker <- st.yymarker - st.token; st.yylimit <- st.yylimit - st.token; %{stags format = "\n\tst.@@ <- if st.@@ = -1 then -1 else st.@@ - st.token;"; %} st.token <- 0; (* Fill free space at the end of buffer with new data from file. *) let n = input st.file st.yyinput st.yylimit (bufsize - st.yylimit - 1) in (* -1 for sentinel *) st.yylimit <- st.yylimit + n; if n = 0 then st.eof <- true; (* end of file *) set st.yyinput st.yylimit '\x00'; (* append sentinel *) Ok) %{ re2c:YYFN = ["lex;(semver list) option", "st;state", "vers;semver list"]; re2c:YYFILL = "fill st = Ok"; re2c:yyrecord = "st"; re2c:tags = 1; re2c:eof = 0; num = [0-9]+; @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\n] { let ver = { major = s2n st.yyinput st.t1 st.t2; minor = s2n st.yyinput st.t3 st.t4; patch = if st.t5 = -1 then 0 else s2n st.yyinput st.t5 (st.yycursor - 1) } in lex_loop st (ver :: vers) } $ { Some (List.rev vers) } * { None } %} and lex_loop st vers = st.token <- st.yycursor; lex st vers let main () = let fname = "input" in (* Prepare input file. *) Out_channel.with_open_bin fname (fun oc -> for i = 1 to bufsize do output_string oc "1.22.333\n" done); (* Construct the expected result to compare against. *) let expect = Some (List.init bufsize (fun _ -> {major = 1; minor = 22; patch = 333;})) in (* Run lexer on the prepared file. *) In_channel.with_open_bin fname (fun ic -> let yylimit = bufsize - 1 in let st = { file = ic; yyinput = create bufsize; yycursor = yylimit; yymarker = yylimit; yylimit = yylimit; token = yylimit; eof = false; %{svars format = "\n\t\t@@{tag} = -1;"; %} %{stags format = "\n\t\t@@{tag} = -1;"; %} } in if (lex_loop st [] <> expect) then raise (Failure "error")); (* Cleanup. *) Sys.remove fname let _ = main () (capturing groups) (semantic versions) . (* re2ocaml $INPUT -o $OUTPUT *) open String type state = { yyinput: string; mutable yycursor: int; mutable yymarker: int; (* Final tag variables available in semantic action. *) %{svars format = "\n\tmutable @@{tag}: int;"; %} (* Intermediate tag variables used by the lexer (must be autogenerated). *) %{stags format = "\n\tmutable @@{tag}: int;"; %} } type semver = { major: int; minor: int; patch: int; } let s2n (str: string) (i1: int) (i2: int) : int = let rec f s i j n = if i >= j then n else f s (i + 1) j (n * 10 + Char.code s.[i] - 48) in f str i1 i2 0 %{local re2c:YYFN = ["parse;semver option", "st;state"]; re2c:yyrecord = "st"; re2c:captvars = 1; re2c:yyfill:enable = 0; num = [0-9]+; (num) "." (num) ("." num)? [\x00] { Some { major = s2n st.yyinput st.yytl1 st.yytr1; minor = s2n st.yyinput st.yytl2 st.yytr2; patch = if st.yytl3 = -1 then 0 else s2n st.yyinput (st.yytl3 + 1) st.yytr3 } } * { None } %} let test (str: string) (result: semver option) = let st = { yyinput = str; yycursor = 0; yymarker = 0; %{svars format = "\n\t\t@@{tag} = -1;"; %} %{stags format = "\n\t\t@@{tag} = -1;"; %} } in if not (parse st = result) then raise (Failure "error") let main () = test "23.34" (Some {major = 23; minor = 34; patch = 0}); test "1.2.99999" (Some {major = 1; minor = 2; patch = 99999}); test "1.a" None let _ = main () m-tags . trie . (* re2ocaml $INPUT -o $OUTPUT *) open String type state = { yyinput: string; mutable yycursor: int; mutable yymarker: int; (* Final tag variables available in semantic action. *) %{svars format = "\n\tmutable @@{tag}: int;"; %} %{mvars format = "\n\tmutable @@{tag}: int list;"; %} (* Intermediate tag variables used by the lexer (must be autogenerated). *) %{stags format = "\n\tmutable @@{tag}: int;"; %} %{mtags format = "\n\tmutable @@{tag}: int list;"; %} } let s2n (str: string) (i1: int) (i2: int) : int = let rec f s i j n = if i >= j then n else f s (i + 1) j (n * 10 + Char.code s.[i] - 48) in f str i1 i2 0 %{local re2c:YYFN = ["parse;(int list) option", "st;state"]; re2c:YYMTAGP = "@@ <- st.yycursor :: @@;"; re2c:YYMTAGN = ""; // alternatively could add `-1` to the list re2c:yyrecord = "st"; re2c:tags = 1; re2c:yyfill:enable = 0; num = [0-9]+; @t1 num @t2 ("." #t3 num #t4)* [\x00] { let x = s2n st.yyinput st.t1 st.t2 in let xs = List.rev (List.map2 (fun x y -> s2n st.yyinput x y) st.t3 st.t4) in Some (x :: xs) } * { None } %} let test (str: string) (result: (int list) option) = let st = { yyinput = str; yycursor = 0; yymarker = 0; %{svars format = "\n\t\t@@{tag} = -1;"; %} %{mvars format = "\n\t\t@@{tag} = [];"; %} %{stags format = "\n\t\t@@{tag} = -1;"; %} %{mtags format = "\n\t\t@@{tag} = [];"; %} } in if not (parse st = result) then raise (Failure "error") let main () = test "1" (Some [1]); test "1.2.3.4.5.6.7" (Some [1; 2; 3; 4; 5; 6; 7;]); test "1.2." None let _ = main () (Encoding support) (code points) (code units) . . . . . . "any" [^] ( * ). : yych YYCTYPE . . re2ocaml . : . : o ASCII ( ). [0-255] - . o EBCDIC ( --ebcdic re2c:encoding:ebcdic). [0-255] - . o UCS2 ( --ucs2 re2c:encoding:ucs2). [0-0xFFFF] - . o UTF8 ( --utf8 re2c:encoding:utf8). Unicode . . 1 -- 4 . o UTF16 ( --utf16 re2c:encoding:utf16). Unicode . . 1 -- 2 . o UTF32 ( --utf32 re2c:encoding:utf32). Unicode [0-0x10FFFF] - . include/unicode_blocks.re include/unicode_categories.re include/unicode_properties.re re2ocaml Unicode . --input-encoding Unicode . --input-encoding utf8 re2ocaml UTF8 ( --utf8 ). --encoding-policy (Unicode surrogates [0xD800-0xDFFF]) re2ocaml . Unicode UTF8 . (* re2ocaml $INPUT -o $OUTPUT --utf8 -i *) open String %{include "unicode_categories.re" %} type state = { yyinput: string; mutable yycursor: int; mutable yymarker: int; mutable yyaccept: int; } %{ re2c:YYFN = ["lex;bool", "yyrecord;state"]; re2c:yyfill:enable = 0; // Simplified "Unicode Identifier and Pattern Syntax" // (see https://unicode.org/reports/tr31) id_start = L | Nl | [$_]; id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3]; identifier = id_start id_continue*; identifier { true } * { false } %} let main () = let st = { yyinput = "_"; yycursor = 0; yymarker = 0; yyaccept = 0; } in if not (lex st) then raise (Failure "error") let _ = main () (Include files) re2ocaml /*!include:re2c FILE */ %{include FILE %} !include FILE ; FILE . re2ocaml include ( -I ) . / re2ocaml #include C/C++ : FILE / . include . --depfile . re2ocaml include/ . ( Unicode) re2ocaml . . (definitions.ml) type number = Int | Float | NaN %{ number = [1-9][0-9]*; %} (extra_rules.re.inc) // floating-point numbers frac = [0-9]* "." [0-9]+ | [0-9]+ "."; exp = 'e' [+-]? [0-9]+; float = frac exp? | [0-9]+ exp; float { Float } (Input file) (* re2ocaml $INPUT -o $OUTPUT -i *) open String %{include "definitions.ml" %} type state = { yyinput: string; mutable yycursor: int; mutable yymarker: int; mutable yyaccept: int; } %{ re2c:YYFN = ["lex;number", "yyrecord;state"]; re2c:yyfill:enable = 0; * { NaN } number { Int } !include "extra_rules.re.inc"; %} let test(str, num) = let st = {yyinput = str; yycursor = 0; yymarker = 0; yyaccept = 0} in if not (lex st = num) then raise (Failure "error") let main () = test("123", Int); test("123.4567", Float) let _ = main () (Header files) re2ocaml --header re2c:header /*!header:re2c:on*/ /*!header:re2c:off*/ %{header:on%} %{header:off%} . . re2ocaml --header re2c:header ( stdout ). re2ocaml . (tag variables) ( ). (Input file) (* re2ocaml $INPUT -o $OUTPUT --header lexer/state.ml -i *) open State open String %{header:on %} type state = { yyinput: string; mutable yycursor: int; mutable tag: int; %{stags format = "mutable @@: int;"; %} } %{header:off %} %{ re2c:YYFN = ["lex;int", "yyrecord;State.state"]; re2c:tags = 1; re2c:yyfill:enable = 0; re2c:header = "lexer/state.ml"; [a]* @tag [b]* { yyrecord.tag } %} let main () = let st = { yyinput = "ab"; yycursor = 0; tag = 0; %{stags format = "\n\t@@ = 0;"; %} } in if not (lex st = 1) then raise (Failure "error") let _ = main () (Header file) (* Generated by re2c *) type state = { yyinput: string; mutable yycursor: int; mutable tag: int; mutable yyt1: int; } (Skeleton programs) -S, --skeleton re2ocaml re2ocaml C . . DFA ( ) re2ocaml : &.input DFA &.keys . &.input . : o re2ocaml ( DFA ). o (benchmarking) . o ( ). ( ). re2ocaml DFA . : DFA . ( ASCII UTF-8 EBCDIC) DFA . DFA : re2ocaml DFA . . re2ocaml ( ) . (re2ocaml ). (Visualization and debug) -D, --emit-dot re2ocaml . DFA DOT . Graphviz DFA . DFA . --dump-nfa --dump-dfa-raw ( ). (SEE ALSO) re2c : %http://re2c.org. flex(1) lex(1) quex(%http://quex.sourceforge.net). (AUTHORS) re2ocaml Peter Bumbulis (%peter@csg.uwaterloo.ca) . Marcus Boerger Dan Nuffer . : Brian Young (%bayoung@acm.org) %Marcus Boerger Dan Nuffer (%nuffer@users.sourceforge.net) %Ulya Trofimovich (%skvadrik@gmail.com) %Serghei Iakovlev %Sergei Trofimovich %Petr Skocik %ligfx %raekye %PolarGoose. RE2OCAML(1)