RE2SWIFT(1) RE2SWIFT(1) (NAME) re2swift - Swift (SYNOPSIS) re2swift [ OPTIONS ] [ WARNINGS ] INPUT - stdin . (INTRODUCTION) re2swift . ( Swift ) / . . re2swift Swift . : // re2swift $INPUT -o $OUTPUT -i func lex(_ yyinput: UnsafePointer) -> Bool { var yycursor = 0 /*!re2c re2c:yyfill:enable = 0; [1-9][0-9]* { return true } * { return false } */ } assert(lex("1234")) re2swift : /* Generated by re2swift */ // re2swift $INPUT -o $OUTPUT -i func lex(_ yyinput: UnsafePointer) -> Bool { var yycursor = 0 var yych: UInt8 = 0 var yystate: UInt = 0 yyl: while true { switch yystate { case 0: yych = yyinput[yycursor] yycursor += 1 switch yych { case 0x31...0x39: yystate = 2 continue yyl default: yystate = 1 continue yyl } case 1: return false case 2: yych = yyinput[yycursor] switch yych { case 0x30...0x39: yycursor += 1 yystate = 2 continue yyl default: yystate = 3 continue yyl } case 3: return true default: fatalError("internal lexer error") } } } assert(lex("1234")) re2swift . : name = regular-expression ; name regular-expression . - . regular expressions re2swift . name . . (global scope) . . configuration = value ; re2swift . re2swift configurations . value - . . . regular-expression code regular-expression ( := ). regular-expression code . . . : * $. (code unit) ( encoding support ). handling the end of input . start conditions . !action code (code) ( ). actions . !directive ; . . directives . (Blocks) /*!re2c */ %{ %} ( ). 2.2 . : /*!re2c[:] ... */ or %{[:] ... %} . re2swift . . options . /*!local:re2c[:] ... */ or %{local[:] ... %} ( ). /*!rules:re2c[:] ... */ or %{rules[:] ... %} -- . ( reusable blocks). 2.2 re2swift -r --reusable . /*!use:re2c[:] ... */ or %{use[:] ... %} use . re2swift . ( ) . use . 2.2 re2swift use -r --reusable . /*!max:re2c[:[:...]] ... */ or %{max[:[:...]] ... %} YYMAXFILL . YYMAXFILL ( ). C (#define YYMAXFILL ) Go (var YYMAXFILL int = ) . format @@{max} ( @@) YYMAXFILL . /*!maxnmatch:re2c[:[:...]] ... */ or %{maxnmatch[:[:...]] ... %} YYMAXNMATCH ( -P --posix-captures ). YYMAXNMATCH ( ). C (#define YYMAXNMATCH ) Go (var YYMAXNMATCH int = ) . format @@{max} ( @@) YYMAXNMATCH . /*!stags:re2c[:[:...]] ... */, /*!mtags:re2c[:[:...]] ... */ or %{stags[:[:...]] ... %}, %{mtags[:[:...]] ... %{ s-tag/m-tag re2swift . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!svars:re2c[:[:...]] ... */, /*!mvars:re2c[:[:...]] ... */ or %{svars[:[:...]] ... %}, %{mvars[:[:...]] ... %{ s-tag/m-tag ( --tags) re2swift ( --captvars --posix-captvars) . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!getstate:re2c[:[:...]] ... */ or %{getstate[:[:...]] ... %} lexer ( --storable-state ). . . . --loop-switch goto . /*!conditions:re2c[:[:...]] ... */, /*!types:re2c... */ or %{conditions[:[:...]] ... %}, %{types... %} (condition enumeration) ( --conditions ). ( ). YYCONDTYPE . format separator . format @@{cond} ( @@) @@{num} . separator format . /*!include:re2c */ or %{include %} . #include C/C++ . --depfile . /*!header:re2c:on*/ or %{header:on %} (header) . header:off re2swift -t --type-header . /*!header:re2c:off*/ or %{header:off %} header:on*/ . /*!ignore:re2c ... */ or %{ignore ... %} . re2swift : re2c:api, re2c:input --api . re2c:api:sigil ("sigil") API . @@ . sigil . sigil $ ${name} . API . API re2c:YYFILL@len YYFILL. re2c:api:style API . functions ( C) free-form ( Go Rust). functions API . ( YYFILL) C/C++ Go Rust . free-form API : @@{var} @@ ( ). API re2c:YYFILL:naked YYFILL . re2c:bit-vectors, re2c:flags:bit-vectors, re2c:flags:b --bit-vectors . re2c:captures, re2c:leftmost-captures --leftmost-captures . re2c:captvars, re2c:leftmost-captvars --leftmost-captvars . re2c:case-insensitive, re2c:flags:case-insensitive --case-insensitive . re2c:case-inverted, re2c:flags:case-inverted --case-inverted . re2c:case-ranges, re2c:flags:case-ranges --case-ranges . re2c:computed-gotos, re2c:flags:computed-gotos, re2c:flags:g --computed-gotos . re2c:computed-gotos:relative, re2c:cgoto:relative --computed-gotos-relative . re2c:computed-gotos:threshold, re2c:cgoto:threshold goto if . 9 . re2c:cond:abort . re2c:cond:goto :=> . goto @@; . @@ ( re2c:api:sigil re2c:cond:goto@cond ). re2c:cond:goto@cond re2c:cond:goto . @@ . re2c:api:sigil . re2c:cond:divider . /* *********************************** */ . ( re2c:api;sigil re2c:cond:divider@cond ). re2c:cond:divider@cond re2c:cond:divider . @@ . re2c:api:sigil . re2c:cond:prefix, re2c:condprefix . yyc_ . re2c:cond:enumprefix, re2c:condenumprefix . yyc . re2c:debug-output, re2c:flags:debug-output, re2c:flags:d --debug-output . re2c:empty-class, re2c:flags:empty-class --empty-class . re2c:encoding:ebcdic, re2c:flags:ecb, re2c:flags:e --ebcdic . re2c:encoding:ucs2, re2c:flags:wide-chars, re2c:flags:w --ucs2 . re2c:encoding:utf8, re2c:flags:utf-8, re2c:flags:8 --utf8 . re2c:encoding:utf16, re2c:flags:utf-16, re2c:flags:x --utf16 . re2c:encoding:utf32, re2c:flags:unicode, re2c:flags:u --utf32 . re2c:encoding-policy, re2c:flags:encoding-policy --encoding-policy . re2c:eof $ . -1 ( $ ). . . re2c:header, re2c:flags:type-header, re2c:flags:t . --header . re2c:indent:string . "\t" . . . re2c:indent:top . . . re2c:invert-captures --invert-captures . re2c:label:prefix, re2c:labelprefix DFA . yy . re2c:label:start, re2c:startlabel . . . . ( ). re2c:label:yyFillLabel YYFILL re2c:eof . re2c:label:yyloop --loop-switch . yyloop . re2c:label:yyNext YYGETSTATE ( re2c:state:nextlabel ). yyNext . re2c:lookahead, re2c:flags:lookahead ( --no-lookahead ). re2c:monadic ( Haskell ). re2c:nested-ifs, re2c:flags:nested-ifs, re2c:flags:s --nested-ifs . re2c:posix-captures, re2c:flags:posix-captures, re2c:flags:P --posix-captures . re2c:posix-captvars --posix-captvars . re2c:tags, re2c:flags:tags, re2c:flags:T --tags . re2c:tags:expression . re2swift yyt . struct . @@{tag} @@ . re2c:tags:expression = "s.@@"; s.yyt . re2c:api:sigil ..TP re2c:tags:negative ( -1 ). re2c:tags:prefix . yyt . re2c:sentinel (sentinel) ( re2c:yyfill:enable = 0; re2c:eof ). : . -1` ( re2swift ). . re2c:state:abort (default case) (abort) -1 . re2c:state:nextlabel switch YYGETSTATE yyNext ( ). re2c:label:start getstate switch YYGETSTATE . re2c:unsafe, re2c:flags:unsafe --no-unsafe . (wrappers) unsafe YYPEEK . ( ). Rust . re2c:YYBACKUP, re2c:define:YYBACKUP API YYBACKUP . re2c:YYBACKUPCTX, re2c:define:YYBACKUPCTX API YYBACKUPCTX . re2c:YYCONDTYPE, re2c:define:YYCONDTYPE API YYCONDTYPE . re2c:YYCTYPE, re2c:define:YYCTYPE API YYCTYPE . re2c:YYCTXMARKER, re2c:define:YYCTXMARKER API YYCTXMARKER . re2c:YYCURSOR, re2c:define:YYCURSOR API YYCURSOR . re2c:YYDEBUG, re2c:define:YYDEBUG API YYDEBUG . re2c:YYFILL, re2c:define:YYFILL API YYFILL . re2c:YYFILL@len, re2c:define:YYFILL@len (sigil) YYFILL . @@ . re2c:api:sigil . re2c:YYFILL:naked, re2c:define:YYFILL:naked re2c:api:style YYFILL . API (free-form) . re2c:YYFN API YYFN . re2c:YYINPUT API YYINPUT . re2c:YYGETCOND, re2c:define:YYGETCONDITION API YYGETCOND . re2c:YYGETCOND:naked, re2c:define:YYGETCONDITION:naked re2c:api:style YYGETCOND . API . re2c:YYGETSTATE, re2c:define:YYGETSTATE API YYGETSTATE . re2c:YYGETSTATE:naked, re2c:define:YYGETSTATE:naked re2c:api:style YYGETSTATE . API . re2c:YYGETACCEPT, re2c:define:YYGETACCEPT API YYGETACCEPT . re2c:YYLESSTHAN, re2c:define:YYLESSTHAN API YYLESSTHAN . re2c:YYLIMIT, re2c:define:YYLIMIT API YYLIMIT . re2c:YYMARKER, re2c:define:YYMARKER API YYMARKER . re2c:YYMTAGN, re2c:define:YYMTAGN API YYMTAGN . re2c:YYMTAGP, re2c:define:YYMTAGP API YYMTAGP . re2c:YYPEEK, re2c:define:YYPEEK API YYPEEK . re2c:YYRESTORE, re2c:define:YYRESTORE API YYRESTORE . re2c:YYRESTORECTX, re2c:define:YYRESTORECTX API YYRESTORECTX . re2c:YYRESTORETAG, re2c:define:YYRESTORETAG API YYRESTORETAG . re2c:YYSETCOND, re2c:define:YYSETCONDITION API YYSETCOND . re2c:YYSETCOND@cond, re2c:define:YYSETCONDITION@cond (sigil) YYSETCOND . @@ . re2c:api:sigil . re2c:YYSETCOND:naked, re2c:define:YYSETCONDITION:naked re2c:api:style YYSETCOND . API . re2c:YYSETSTATE, re2c:define:YYSETSTATE API YYSETSTATE . re2c:YYSETSTATE@state, re2c:define:YYSETSTATE@state (sigil) YYSETSTATE . @@ . re2c:api:sigil . re2c:YYSETSTATE:naked, re2c:define:YYSETSTATE:naked re2c:api:style YYSETSTATE . API . re2c:YYSETACCEPT, re2c:define:YYSETACCEPT API YYSETACCEPT . re2c:YYSKIP, re2c:define:YYSKIP API YYSKIP . re2c:YYSHIFT, re2c:define:YYSHIFT API YYSHIFT . re2c:YYCOPYMTAG, re2c:define:YYCOPYMTAG API YYCOPYMTAG . re2c:YYCOPYSTAG, re2c:define:YYCOPYSTAG API YYCOPYSTAG . re2c:YYSHIFTMTAG, re2c:define:YYSHIFTMTAG API YYSHIFTMTAG . re2c:YYSHIFTSTAG, re2c:define:YYSHIFTSTAG API YYSHIFTSTAG . re2c:YYSTAGN, re2c:define:YYSTAGN API YYSTAGN . re2c:YYSTAGP, re2c:define:YYSTAGP API YYSTAGP . re2c:yyaccept, re2c:variable:yyaccept API yyaccept . re2c:yybm, re2c:variable:yybm API yybm . re2c:yybm:hex, re2c:variable:yybm:hex (bitmaps) --bit-vectors (hexadecimal) . ( ). re2c:yych, re2c:variable:yych API yych . re2c:yych:emit, re2c:variable:yych:emit yych . . re2c:yych:conversion, re2c:variable:yych:conversion re2swift yych YYCTYPE . ( ). re2c:yych:literals, re2c:variable:yych:literals yych . : char ( ) hex ( ) char_or_hex ( ). re2c:yyctable, re2c:variable:yyctable API yyctable ..TP re2c:yynmatch, re2c:variable:yynmatch API yynmatch . re2c:yypmatch, re2c:variable:yypmatch API yypmatch . re2c:yytarget, re2c:variable:yytarget API yytarget . re2c:yystable, re2c:variable:yystable . re2c:yystate, re2c:variable:yystate API yystate . re2c:yyfill, re2c:variable:yyfill API yyfill . re2c:yyfill:check YYFILL ( YYLESSTHAN API YYLIMIT API C) . ( ). re2c:yyfill:enable YYFILL ( ) . ( ) YYFILL ( / sentinel ). ( -W) re2c:sentinel . (YYFILL ). re2c:yyfill:parameter YYFILL . . ( ). re2c:YYFILL:naked re2c:api:style . re2c:yyfn:sep YYFN ( - ). re2c:yyfn:throw YYFN ( ). (Regular expressions) re2swift : "foo" . 'foo' . [a-xyz], [^a-xyz] ( ). . . R \ S R S. R* R. R+ R. R? R . R{n} R n . R{n,} R n . R{n,m} R n m . (R) R . (R) --invert-captures . (!R) (!R) --invert-captures . R S : R S. R | S : R S. R / S : R S S . name name ( "name" Flex). {name} name Flex. @stag s-tag: @stag stag . #mtag m-tag: #mtag mtag . $ . : \a \b \f \n \r \t \v \\ \ooo \xhh \uhhhh \Uhhhhhhhh..SS " (Actions)" re2swift : !entry code (Entry action) code . start conditions . (lexeme) . !pre_rule code (Pre-rule action) code ( start conditions ) . ( ) . !post_rule code (Post-rule action) code ( start conditions ) . (trap) . (Directives) re2swift : !use:name ; use name . . : . * $ . use use . !include file ; include : file . (Program interface) (primitives) API . . . re2swift ( ) . API . re2swift (flavor) API : Simple API backend (Swift) . API : YYINPUT ( ) YYCURSOR YYMARKER YYCTXMARKER YYLIMIT ( YYINPUT ). Record API Record API . --api record re2c:api = record . API yyrecord ( re2c:YYRECORD ) (struct) (class) yyinput yycursor yymarker yyctxmarker yylimit ( ). Generic API API . --api generic re2c:api = generic . API : YYPEEK YYSKIP YYBACKUP YYBACKUPCTX YYSTAGP YYSTAGN YYMTAGP YYMTAGN YYRESTORE YYRESTORECTX YYRESTORETAG YYSHIFT YYSHIFTSTAG YYSHIFTMTAG YYLESSTHAN YYEND. buffer: [UInt8] cursor limit marker ctxmarker Int -1 Generic API : /*!re2c re2c:YYPEEK = "buffer[cursor]"; re2c:YYSKIP = "cursor++"; re2c:YYBACKUP = "marker = cursor"; re2c:YYRESTORE = "cursor = marker"; re2c:YYBACKUPCTX = "ctxmarker = cursor"; re2c:YYRESTORECTX = "cursor = ctxmarker"; re2c:YYRESTORETAG = "cursor = ${tag}"; re2c:YYLESSTHAN = "limit - cursor < @@{len}"; re2c:YYEND = "limit == cursor"; re2c:YYSTAGP = "@@{tag} = cursor"; re2c:YYSTAGN = "@@{tag} = -1"; re2c:YYSHIFT = "cursor += @@{shift}"; re2c:YYSHIFTSTAG = "@@{tag} += @@{shift}"; */ API : YYCTYPE ( ). ASCII EBCDIC UTF-8 . UTF-16 UCS-2 . UTF-32 . YYCURSOR (l-value) ( YYINPUT). YYCURSOR . YYCURSOR . YYLIMIT (r-value) ( YYINPUT). YYLIMIT . . YYCURSOR YYLIMIT . YYMARKER (l-value) ( YYINPUT). YYCURSOR (rollback) . . YYCTXMARKER (l-value) (trailing context) ( YYINPUT). . YYCTXMARKER (lookahead) / . YYFILL Generic API len. YYFILL len . re2c:eof len 1 YYFILL . re2c:eof YYFILL . len YYMAXFILL . YYFN (prototype) --recursive-functions . re2c:fn:sep ( ) . ( void ). : ( ) ( - ). YYINPUT (r-value) ( ) . YYMAXFILL YYFILL. max . YYLESSTHAN Generic API len. (r-value) len true . YYEND Generic API . (r-value) ( /sentinel) true . YYEND $ . YYLESSTHAN . YYPEEK (primitive) API . r-value YYCTYPE . YYSKIP API . YYBACKUP API ( YYRESTORE ). YYRESTORE API YYBACKUP . YYBACKUPCTX API (trailing context) ( YYRESTORECTX ). YYRESTORECTX API YYBACKUPCTX . YYRESTORETAG API tag tag . YYSTAGP API tag tag YYINPUT ( ). YYSTAGP tag . YYSTAGN API tag tag YYINPUT ( ). YYSTAGN tag . YYMTAGP API tag. YYMTAGP tag ( ). YYMTAGN API tag. YYMTAGN tag ( ). YYSHIFT API shift shift ( ). YYCOPYSTAG API lhs rhs s-tag (rhs) s-tag (lhs) . lhs rhs . YYCOPYMTAG API lhs rhs m-tag (rhs) m-tag (lhs) . lhs rhs . YYSHIFTSTAG API tag shift tag shift ( ). YYSHIFTMTAG API tag shift tag shift ( ). YYMAXNMATCH POSIX . maxnmatch . YYCONDTYPE (condition enum). conditions --header . YYGETACCEPT var . var . YYSETACCEPT : var ( l-value ) val ( ). var val . YYGETCOND r-value YYCONDTYPE . YYSETCOND cond cond . YYGETSTATE r-value (lexer) . -1 . YYSETSTATE state state . YYDEBUG -d --debug-output . () ( YYDEBUG ). YYDEBUG : state ( DFA -1) symbol ( ). yyaccept l-value . --storable-state . yybm ( --bitmaps ). . . k- n- (true) n- k- . if switch . yych l-value YYCTYPE . -f --storable-state . yyctable ( --conditions --computed-gotos ). yyfill l-value YYFILL ( YYFILL ). yynmatch l-value POSIX . -P --posix-captures . yypmatch l-value . yynmatch * 2 ( YYMAXNMATCH * 2 ). -P --posix-captures . yystable . yystate l-value --loop-switch DFA . yytarget ( ) . . yytarget --computed-gotos . re2c . () re2c . re2c . -? --help -h . --api API . simple (lexer) . record (struct class ) . generic API . --bit-vectors -b . --nested-ifs . --captures, --leftmost-captures (leftmost greedy). yybmatch 2 * YYMAXNMATCH yynmatch . --captvars, --leftmost-captvars (leftmost greedy). yytl yytr k- . --case-insensitive . --case-inverted : . --case-ranges case switch low ... high. C/C++ GCC Clang Tcc . case Tcc . --computed-gotos -g "computed goto" ( ). re2swift . cgoto:threshold . cgoto:relative . --bit-vectors . --computed-gotos-relative --computed-gotos ( ). --computed-gotos . --conditions --start-conditions -c "" Flex: . re2swift goto . --depfile FILE FILE Makefile : [include-file ...]. / include include . --output . --ebcdic --ecb -e EBCDIC . re2swift 0 -- 0xFF . --empty-class re2swift . match-empty () ( ). match-none . error . --encoding-policy re2swift (surrogates). fail re2swift . substitute re2swift 0xFFFD . ignore () re2swift . . --flex-syntax -F Flex: . . --goto-label "goto/label": DFA goto . goto . --header --type-header -t HEADER HEADER. header:on header:off . (enum) ( conditions ). -I PATH PATH include . include . re2swift FILE include -I . --input --api. default simple ( ). custom generic . --input-encoding re2swift. ascii () re2swift ASCII : . utf8 re2swift UTF-8 . --invert-captures . (...) (! ...) . (! ...) (...) . --lang . C D Go Haskell Java JS OCaml Python Rust Swift V Zig ( --syntax ). none . --location-format . gnu 'filename:line:column: ...' . msvc 'filename(line,column) ...' . gnu . --loop-switch "loop/switch": DFA switch case switch . yystate . yystate case . --nested-ifs -s if switch . . --no-debug-info -i (line directives) . ( diff ). --no-generation-date . --no-version . --no-unsafe unsafe YYPEEK ( Rust ). YYPEEK (bounds- checking) . YYPEEK (release) --no-unsafe unsafe . --output -o OUTPUT OUTPUT. --posix-captures, -P POSIX. yybmatch 2 * YYMAXNMATCH yynmatch . --posix-captvars POSIX. yytl yytr k- . --recursive-functions (co- recursive) DFA . --reusable -r 2.2 ( ). --skeleton -S "" (skeleton). "" . . C ..TP --storable-state -f . push . YYGETSTATE YYSETSTATE yych yyaccept state . --syntax FILE FILE . FILE ( ) ( --lang none ). --tags -T . --ucs2 --wide-chars -w UCS2 . re2swift 0 -- 0xFFFF . --nested-ifs . --utf8 --utf-8 -8 UTF-8 . re2swift 0 -- 0x10FFFF . --utf16 --utf-16 -x UTF16 . re2swift 0 -- 0x10FFFF . --nested-ifs . --utf32 --unicode -u UTF32 . re2swift 0 -- 0x10FFFF . --nested-ifs . --verbose . --vernum -V MMmmpp ( ). --version -v . --single-pass -1 . ( ). --debug-output -d YYDEBUG . . --dump-adfa : DFA ( .dot). --dump-cfg : ( .dot). --dump-closure-stats : . --dump-dfa-det : DFA ( .dot). --dump-dfa-min : DFA ( .dot). --dump-dfa-tagopt : DFA ( .dot). --dump-dfa-tree : DFA ( .dot). --dump-dfa-raw : DFA ( .dot). --dump-interf : . --dump-nfa : NFA ( .dot). --emit-dot -D .dot . Graphviz ( dot -Tpng -odfa.png dfa.dot). --dfa-minimization : DFA re2swift. moore ( ). table " " . DFA . --eager-skip : -- . -- -- . --no-lookahead . TDFA(0) . TDFA(1) TDFA(0) . . TDFA(0) TDFA(1) . --no-optimize-tags : ( ). --posix-closure : POSIX gor1 () - (Goldberg-Radzik) gtop " " . --posix-prectable : POSIX. complex TNFA . naive TNFA complex . --stadfa . staDFA TDFA . staDFA TDFA . --fixed-tags : (all) (none) (toplevel). all . "" ( "") . . (alternative) ( ). . . -W . -Werror . . -W warning. -Wno- warning. -Werror- warning ( -W ). -Wno-error- warning . . -Wcondition-order . --header conditions . -Wempty-character-class . : . re2swift . --empty-class . -Wmatch-empty-string ( ). . -Wswapped-range . . -Wundefined-control-flow ( ). . * . -Wunreachable-rules . -Wdeprecated-eof_rule $ . $ . -Wuseless-escape . re2swift . -Wnondeterministic-tags n- n . -Wsentinel-in-midrule --- . . re2c:sentinel . -Wundefined-syntax-config --syntax . : . . (Syntax files) re2c . -- . re2c re2c: : . ( include/syntax ) re2swift . ( --lang none ). : . (Feature lists) re2swift : supported_apis API simple record generic. supported_api_styles API functions free-form. supported_code_models goto-label loop-switch recursive-functions. supported_targets code dot skeleton. supported_features nested-ifs bitmaps computed-gotos case-ranges monadic unsafe tags captures captvars. (Language configurations) re2swift : semicolons . backtick_quoted_strings . single_quoted_strings . indentation_sensitive . wrap_blocks_in_braces . (Inplace configurations) re2c: . . (Code templates) . : code-template :: name '=' code-exprs ';' | CODE_TEMPLATE ';' | '' ';' code-exprs :: | code-exprs code-expr code-expr :: STRING | VARIABLE | optional | list optional :: '(' CONDITIONAL '?' code-exprs ')' | '(' CONDITIONAL '?' code-exprs ':' code-exprs ')' list :: '[' VARIABLE ':' code-exprs ']' | '[' VARIABLE '{' NUMBER '}' ':' code-exprs ']' | '[' VARIABLE '{' NUMBER ',' NUMBER '}' ':' code-exprs ']' . . : re2swift . ( -1 ). . ( ) ( ) . re2swift . if-then-else C- : code:if_then_else = [branch{0}: topindent "if " cond " {" nl indent [stmt: stmt] dedent] [branch{1:-1}: topindent "} else" (.cond ? " if " cond) " {" nl indent [stmt: stmt] dedent] topindent "}" nl; branch : branch{0} ( else ) branch{1:-1} ( ) . stmt : [stmt: stmt] . topindent indent dedent nl .cond ( ). : if x { // do something } else if y { // do something else } else { // don't do anything } re2swift . . . code:var_local . : type ( ) name ( ) init ( ). : .init ( true). code:var_global code:var_local (top-level) . code:const_local . : type ( ) name ( ) init ( ). code:const_global code:const_local (top-level) . code:array_local (). : type ( ) name ( ) size ( ) row ( ) elem ( -- row ). : .const ( true). code:array_global code:array_local (top-level) . code:array_elem (). : array ( ) index ( ). code:enum ( ). type ( ) elem ( ) init ( ). : .init ( true). code:enum_elem ( code:enum). name ( ) type ( ). code:assign . lhs ( ) rhs ( ). code:type_int (signed integer). code:type_uint (unsigned integer). code:type_yybm yybm. code:type_yytarget yytarget. code:type_yyctable yyctable. code:cmp_eq "equals" (). code:cmp_ne "not equals" (). code:cmp_lt "less than" (). code:cmp_gt "greater than" (). code:cmp_le "less or equal" ( ). code:cmp_ge "greater or equal" ( ). code:if_then_else If-then-else . : branch ( ) cond ( ) stmt ( ). : .cond ( true) .many ( true). code:if_then_else_oneline code:if_then_else . code:if_then_else . code:switch switch (case). : expr ( switch ) case ( case ). code:switch_cases switch . case ( ) stmt ( ). code:switch_cases_oneline code:switch_cases . code:switch_cases . code:switch_case_range switch ( ). : val ( ). : .many ( true) .char_literals ( switch true -- ). code:switch_case_default (default) switch. code:loop ( ). : label ( ) stmt ( ). code:continue continue. : label ( ). code:goto goto. : label ( ). code:cgoto goto (computed goto). : array ( goto ) index ( ) base ( .cgoto.relative true ). code:cgoto:data goto . : label ( ) type ( ) base ( - .cgoto.relative true ). code:fndecl . : name ( ) type ( ) throw ( re2c:yyfn:throw ) arg ( ) argname ( ) argtype ( ). : .type ( void true). code:fndef code:fndecl stmt . code:fncall . : name ( ) retval ( l-value ) arg ( ). : .args ( true) .retval ( true). code:tailcall (tail call). : name ( ) arg ( ). : .args ( true) .retval ( void true). code:recursive_functions --recursive-functions. : fn ( ) fndecl ( ) fndef ( ). code:fingerprint (fingerprint) . : ver ( re2swift ) date ( ). code:line_info ( ). : line ( ) file ( ). code:abort (abort) . code:yydebug YYDEBUG API . : YYDEBUG yyrecord yych ( re2c: ) state ( DFA)..TP code:yypeek YYPEEK API . : YYPEEK YYCTYPE YYINPUT YYCURSOR yyrecord yych ( re2c:). : .cast ( re2c:yych:conversion ). code:yyskip YYSKIP API . : YYSKIP YYCURSOR yyrecord ( re2c:). code:yybackup YYBACKUP API . : YYBACKUP YYCURSOR YYMARKER yyrecord ( re2c:). code:yybackupctx YYBACKUPCTX API . : YYBACKUPCTX YYCURSOR YYCTXMARKER yyrecord ( re2c:). code:yyskip_yypeek code:yyskip code:yypeek ( code:yyskip code:yypeek). code:yypeek_yyskip code:yypeek code:yyskip ( code:yypeek code:yyskip). code:yyskip_yybackup code:yyskip code:yybackup ( code:yyskip code:yybackup). code:yybackup_yyskip code:yybackup code:yyskip ( code:yybackup code:yyskip). code:yybackup_yypeek code:yybackup code:yypeek ( code:yybackup code:yypeek). code:yyskip_yybackup_yypeek code:yyskip code:yybackup code:yypeek ( ``code:yyskip`` code:yybackup code:yypeek). code:yybackup_yypeek_yyskip code:yybackup code:yypeek code:yyskip ( ``code:yybackup`` code:yypeek code:yyskip). code:yyrestore YYRESTORE API . : YYRESTORE YYCURSOR YYMARKER yyrecord ( re2c:). code:yyrestorectx YYRESTORECTX API . : YYRESTORECTX YYCURSOR YYCTXMARKER yyrecord ( re2c:). code:yyrestoretag YYRESTORETAG API . : YYRESTORETAG YYCURSOR yyrecord ( re2c:) tag ( ). code:yyshift YYSHIFT API . : YYSHIFT YYCURSOR yyrecord ( re2c:) offset ( ). code:yyshiftstag YYSHIFTSTAG API . : YYSHIFTSTAG yyrecord negative ( re2c:) tag ( ) offset ( ). : .nested ( -- re2c:tags:negative ). code:yyshiftmtag YYSHIFTMTAG API . : YYSHIFTMTAG ( re2c:) tag ( ) offset ( ). code:yystagp YYSTAGP API . : YYSTAGP YYCURSOR yyrecord ( re2c:) tag ( ). code:yymtagp YYMTAGP API . : YYMTAGP ( re2c:) tag ( ). code:yystagn YYSTAGN API . : YYSTAGN negative yyrecord ( re2c:) tag ( ). code:yymtagn YYMTAGN API . : YYMTAGN ( re2c:) tag ( ). code:yycopystag YYCOPYSTAG API . : YYCOPYSTAG yyrecord ( re2c:) lhs rhs ( ). code:yycopymtag YYCOPYMTAG API . : YYCOPYMTAG yyrecord ( re2c:) lhs rhs ( ). code:yygetaccept YYGETACCEPT API . : YYGETACCEPT yyrecord ( re2c:) var ( re2c:yyaccept). code:yysetaccept YYSETACCEPT API . : YYSETACCEPT yyrecord ( re2c:) var ( re2c:yyaccept) val ( ). code:yygetcond YYGETCOND API . : YYGETCOND yyrecord ( re2c:) var ( re2c:yycond). code:yysetcond YYSETCOND API . : YYSETCOND yyrecord ( re2c:) var ( re2c:yycond) val ( ). code:yygetstate YYGETSTATE API . : YYGETSTATE yyrecord ( re2c:) var ( re2c:yystate). code:yysetstate YYSETSTATE API . : YYSETSTATE yyrecord ( re2c:) var ( re2c:yystate) val ( ). code:yylessthan YYLESSTHAN API . : YYLESSTHAN YYCURSOR YYLIMIT yyrecord ( re2c:) need ( ). : .many ( need ). code:yyend YYEND API . : YYEND YYCURSOR YYLIMIT. code:yybm_filter yych yybm ( --bitmaps ). : yych ( re2c:yych). code:yybm_match yybm ( --bitmaps). : yybm yych ( re2c:) offset ( yybm yych ) mask ( ) code:yytarget_filter yych yytarget ( --computed-gotos ). : yych ( re2c:yych). : nl . indent ..TP dedent . topindent . . : .api.simple API (--api simple re2c:api = simple). .api.generic API (--api generic re2c:api = generic). .api.record API (--api record re2c:api = record). .api_style.functions API (re2c:api-style = functions). .api_style.freeform API (re2c:api-style = free-form). .case_ranges case (--case-ranges re2c:case-ranges = 1). .cgoto.relative goto (--computed-gotos-relative re2c:cgoto:relative = 1). .code_model.goto_label goto/label (--goto-label). .code_model.loop_switch loop/switch (--loop-switch). .code_model.recursive_functions (--recursive-function). .date . .loop_label re2swift (re2c:label:yyloop ). .monadic (re2c:monadic = 1). . .start_conditions (--start-conditions). .storable_state (--storable-state). .unsafe re2swift "unsafe" (--unsafe re2c:unsafe = 1). "unsafe" . .version re2swift . .yyfill.enable YYFILL (re2c:yyfill:enable = 1). .yyfn.throw re2c:yyfn:throw . (HANDLING THE END OF INPUT) (lexer) . : o ( *) o o <<>> : . : . . re2swift . . : o (Sentinel). . <<>> . . : . . YYFILL re2c:yyfill:enable = 0; re2c:eof -1 . o (Sentinel with bounds checks). (generic) : . . <<>> <<>> . ( ). . . YYFILL ( ). . . re2c:eof ( ). YYFILL . o (Bounds checks with padding). << >> . DFA (SCC) SCC SCC . (lexemes) . ( ) . YYMAXFILL max . YYFILL . YYFILL re2c:eof -1 ( ). o (Custom checks). Generic API . . API --api custom re2c:api = custom; re2c:yyfill:enable = 0; re2c:yyfill:check = 0; . ..SS (Sentinel) (sentinel character) . null . null : (lexeme) ( ). null (syntax error) * (crash) ( -Wsentinel-in-midrule re2c:sentinel ). re2c:yyfill:enable = 0; (bounds checks) YYFILL . // re2swift $INPUT -o $OUTPUT // Expects a null-terminated string. func lex(_ yyinput: UnsafePointer) -> Int? { var yycursor = 0, count = 0 loop: while true { /*!re2c re2c:yyfill:enable = 0; [a-z]+ { count += 1 continue loop } [ ]+ { continue loop } [\x00] { return count } * { return nil } */ } } func test(_ str: StaticString, _ expect: Int?) { assert(lex(str.utf8Start) == expect) } test("", 0) test("one two three", 3) test("f0ur", nil) (Sentinel with bounds checks) ( 1.2 ). (single-quoted) . null re2c:eof = 0; . sentinel null ( 'aaa\0aa'\0 'aaa\0 ). null . API YYLIMIT <= YYCURSOR YYLESSTHAN(1) . (true) ( YYFILL re2c:yyfill:enable = 0; YYFILL YYFILL with sentinel ). : $ ( *) -Wundefined-control-flow . // re2swift $INPUT -o $OUTPUT // Expects a null-terminated string. func lex(_ yyinput: UnsafePointer, _ length: Int) -> Int? { var yycursor = 0, yylimit = length, yymarker = 0 var count = 0 loop: while true { /*!re2c re2c:eof = 0; re2c:yyfill:enable = 0; str = ['] ([^'\\] | [\\][^])* [']; str { count += 1 continue loop } [ ]+ { continue loop } * { return nil } $ { return count } */ } } func test(_ str: StaticString, _ expect: Int?) { assert(lex(str.utf8Start, str.utf8CodeUnitCount) == expect) } test("", 0); test("'qu\0tes' 'are' 'fine: \\'' ", 3); test("'unterminated\\'", nil); (Bounds checks with padding) (padding) ( ). . null YYMAXFILL YYMAXFILL max . null --- ( ). ("stop") (null) ( null ). () . API (YYLIMIT - YYCURSOR) < n YYLESSTHAN(n) n ( n ). YYFILL(n) n . YYFILL ( ). YYFILL YYFILL with padding . // re2swift $INPUT -o $OUTPUT /*!max:re2c*/ func lex(_ str: UnsafeBufferPointer) -> Int? { let length = str.count // Make a copy of the string with YYMAXFILL zeroes at the end. let yyinput = ContiguousArray(str) + repeatElement(0, count: yymaxfill) var yycursor = 0, yylimit = length + yymaxfill var count = 0 loop: while true { /*!re2c re2c:YYFILL = "return nil"; str = ['] ([^'\\] | [\\][^])* [']; str { count += 1 continue loop } [\x00] { // Check that it's the sentinel, not some unexpected null. if yycursor - 1 == length { return count } else { return nil } } [ ]+ { continue loop } * { return nil } */ } } func test(_ str: StaticString, _ expect: Int?) { str.withUTF8Buffer { assert(lex($0) == expect) } } test("", 0) test("'qu\0tes' 'are' 'fine: \\'' ", 3) test("'unterminated\\'", nil) test("'unexpected \0 null\\'", nil) (end-of-input) API (generic API) . (single-quoted) . sentinel (null-terminated ). (sentinel) YYPEEK (bounds check) . . YYPEEK . // re2swift $INPUT -o $OUTPUT import Foundation func lex(_ str: Data) -> Int? { var cursor = 0, limit = str.count var count = 0 loop: while true { /*!re2c re2c:api = generic; re2c:yyfill:enable = 0; re2c:YYSKIP = "cursor += 1"; re2c:YYPEEK = "cursor < limit ? str[cursor] : 0"; // fake null [a-z]+ { count += 1 continue loop } [\x00] { return count } [ ]+ { continue loop } * { return nil} */ } } func test(_ str: String, _ expect: Int?) { // For the sake of example create a string without terminating null. assert(lex(Data(str.utf8)) == expect) } test("", 0) test("one two three ", 3) test("f0ur", nil) (BUFFER REFILLING) : ( ). (chunks) . . (lexer) : o (cursor): (YYCURSOR API C YYSKIP/YYPEEK generic API) o (limit): (YYLIMIT API C YYLESSTHAN generic API) o (marker): (YYMARKER API YYBACKUP/YYRESTORE generic API) o (token): (lexeme) ( re2swift API ) o (context marker): (YYCTXMARKER API C YYBACKUPCTX/YYRESTORECTX generic API) o (tag variables): ( stags mtags generic API YYSTAGP/YYSTAGN/YYMTAGP/YYMTAGN) YYFILL . token cursor token token limit . YYFILL ( ). YYFILL EOF : EOF (padding) . -f --storable-state YYFILL ( ). YYFILL (YYFILL with sentinel) EOF YYFILL . YYFILL YYLIMIT <= YYCURSOR API C YYLESSTHAN() generic API . YYFILL . YYFILL . Limit limit re2c:eof . YYFILL ( # ). <-- shift --> >-A------------B---------C------------D#-----------E-> buffer token marker limit, cursor >-A------------B---------C------------D------------E#-> buffer, marker cursor limit token <-- shift --> >-A------------B---------C------------D#--E (EOF) buffer token marker limit, cursor >-A------------B---------C------------D---E#........ buffer, marker cursor limit token input.txt EOF . // re2swift $INPUT -o $OUTPUT import Foundation struct Input { static let bufferSize = 4095 var file: FileHandle var yyinput = ContiguousArray(repeating: 0, count: Self.bufferSize + 1) var yylimit = Self.bufferSize var yycursor = Self.bufferSize var yymarker = Self.bufferSize var token = -1 var eof = false } extension Input { mutating func lex() -> Int? { var count = 0 loop: while true { self.token = self.yycursor /*!re2c re2c:api = record; re2c:eof = 0; re2c:yyrecord = "self"; re2c:YYFILL = "self.fill() == .ok"; str = ['] ([^'\\] | [\\][^])* [']; str { count += 1 continue loop } [ ]+ { continue loop } $ { return count } * { return nil } */ } } mutating func fill() -> FillStatus { guard !self.eof else { return .eof } let shift = self.token let used = self.yylimit - self.token let free = Self.bufferSize - used // Error: Lexeme too long. In the real world we can reallocate a larger buffer. if self.token < 1 { return .longLexeme } // Shift buffer contents, discarding everything up to the current lexeme. self.yyinput.replaceSubrange(.. ( EOF ) YYFILL . YYFILL (YYLIMIT - YYCURSOR) < n API C YYLESSTHAN(n) API . YYFILL . YYFILL (lexer) ( ). YYFILL (limit) YYMAXFILL ( YYFILL n ). YYFILL ( YYMAXFILL # ). <-- shift --> <-- need --> >-A------------B---------C-----D-------E---F--------G-> buffer token marker cursor limit >-A------------B---------C-----D-------E---F--------G-> buffer, marker cursor limit token <-- shift --> <-- need --> >-A------------B---------C-----D-------E-F (EOF) buffer token marker cursor limit >-A------------B---------C-----D-------E-F############### buffer, marker cursor limit token <- YYMAXFILL -> input.txt (bounds-checking) . // re2swift $INPUT -o $OUTPUT import Foundation /*!max:re2c*/ struct Input { static let bufferSize = 4096 - yymaxfill var file: FileHandle var yyinput = ContiguousArray(repeating: 0, count: Self.bufferSize + yymaxfill) var yylimit = Self.bufferSize var yycursor = Self.bufferSize var token = -1 var eof = false } extension Input { mutating func lex() -> Int? { var count = 0 loop: while true { self.token = self.yycursor /*!re2c re2c:api = record; re2c:yyrecord = "self"; re2c:YYFILL = "if self.fill(@@) != .ok { return nil }"; str = ['] ([^'\\] | [\\][^])* [']; [\x00] { // Check that it is the sentinel, not some unexpected null. return self.token == self.yylimit - yymaxfill ? count : nil; } str { count += 1 continue loop } [ ]+ { continue loop } * { return nil } */ } } mutating func fill(_ need: Int) -> FillStatus { guard !self.eof else { return .eof } let shift = self.token let used = self.yylimit - self.token let free = Self.bufferSize - used // Error: Lexeme too long. In the real world we can reallocate a larger buffer. if self.token < need { return .longLexeme } // Shift buffer contents, discarding everything up to the current lexeme. self.yyinput.replaceSubrange(.. add zero padding // so that the lexer can access characters at the end of buffer. self.yylimit += read if read < free { self.eof = true self.yyinput.withUnsafeMutableBytes { _ = memset($0.baseAddress! + self.yylimit, 0, yymaxfill) } self.yylimit += yymaxfill } return .ok } enum FillStatus { case ok, eof, longLexeme } } let fileName = "input" let content = "'qu\0tes' 'are' 'fine: \\'' "; // Prepare input file: a few times the size of the buffer, // containing strings with zeroes and escaped quotes. guard FileManager.default.createFile( atPath: fileName, contents: Data(String(repeating: content, count: Input.bufferSize).utf8) ) else { fatalError("failed to write file \"\(fileName)\"") } // Number of quoted strings written to file let count = 3 * Input.bufferSize // Initialize lexer state // This immediately triggers YYFILL, as the check `in.yycursor < in.yylimit` fails. guard let file = FileHandle(forReadingAtPath: fileName) else { throw NSError(domain: NSCocoaErrorDomain, code: CocoaError.fileReadNoSuchFile.rawValue) } var `in` = Input(file: file) // Run the lexer assert(`in`.lex() == count) // Cleanup: remove input file try file.close() try FileManager.default.removeItem(atPath: fileName) (Multiple blocks) (lexer) ( ). re2swift . (start conditions) . . goto ( C/C++ Go) . goto . goto ( Rust) switch /switch yystate re2swift . (binary) (octal) (decimal) (hexadecimal) . . . . // re2swift $INPUT -o $OUTPUT func parseUInt32(_ yyinput: UnsafePointer) -> UInt32? { var yycursor = 0, yymarker = 0 /*!re2c re2c:yyfill:enable = 0; "0b" / [01] { return parseBin() } "0" { return parseOct() } "" / [1-9] { return parseDec() } "0x" / [0-9a-fA-F] { return parseHex() } * { return nil } */ func add(_ accum: inout UInt64, _ charOrigin: UnicodeScalar, _ base: UInt64, _ offset: UInt64 = 0) { let digit = yyinput[yycursor - 1] - UInt8(ascii: charOrigin) accum = min(accum * base + UInt64(digit) + offset, UInt64(UInt32.max) + 1) } func parseBin() -> UInt32? { var accum: UInt64 = 0 parse: while true { /*!re2c [01] { add(&accum, "0", 2); continue parse } * { return UInt32(exactly: accum) } */ } } func parseOct() -> UInt32? { var accum: UInt64 = 0 parse: while true { /*!re2c [0-7] { add(&accum, "0", 8); continue parse } * { return UInt32(exactly: accum) } */ } } func parseDec() -> UInt32? { var accum: UInt64 = 0 parse: while true { /*!re2c [0-9] { add(&accum, "0", 10); continue parse } * { return UInt32(exactly: accum) } */ } } func parseHex() -> UInt32? { var accum: UInt64 = 0 parse: while true { /*!re2c [0-9] { add(&accum, "0", 16); continue parse } [a-f] { add(&accum, "a", 16, 10); continue parse } [A-F] { add(&accum, "A", 16, 10); continue parse } * { return UInt32(exactly: accum) } */ } } } assert(parseUInt32("") == nil) assert(parseUInt32("1234567890") == 1234567890) assert(parseUInt32("0b1101") == 13) assert(parseUInt32("0x7Fe") == 2046) assert(parseUInt32("0644") == 420) assert(parseUInt32("9999999999") == nil) (Start conditions) --start-conditions . () re2swift . re2swift . : YYGETCOND YYSETCOND . conditions --header re2c:header . re2c:cond:enumprefix . <*> . : < condition-list > regular-expression code condition-list . regular-expression code . < condition-list > regular-expression => condition code condition-list . regular-expression condition code . < condition-list > regular-expression :=> condition condition-list . regular-expression condition ( ). < condition-list > !action code code action condition-list ( actions ). code code condition-list . !pre_rule ( ). < > code "0" code . !entry ( ). < > => condition code condition . < > :=> condition condition . re2swift re2swift goto/label loop/switch . goto ( C/C++ Go) yyc_ cond ( re2c:cond:prefix ). goto . re2swift switch YYGETSTATE . :=> switch ( re2c:cond:goto ). . goto ( Rust) re2swift yystate . switch switch DFA . switch . ( YYGETCOND/YYSETCOND ). . . cond conditions . // re2swift $INPUT -o $OUTPUT -c /*!conditions:re2c*/ func parseUInt32(_ yyinput: UnsafePointer) -> UInt32? { var yycursor = 0, yymarker = 0, yycond = yycinit var accum: UInt64 = 0 loop: while true { /*!re2c re2c:yyfill:enable = 0; '0b' / [01] :=> bin "0" :=> oct "" / [1-9] :=> dec '0x' / [0-9a-fA-F] :=> hex [01] { add("0", 2); continue loop } [0-7] { add("0", 8); continue loop } [0-9] { add("0", 10); continue loop } [0-9] { add("0", 16); continue loop } [a-f] { add("a", 16, 10); continue loop } [A-F] { add("A", 16, 10); continue loop } "\x00" { return UInt32(exactly: accum) } <*> * { return nil } */ } func add(_ charOrigin: UnicodeScalar, _ base: UInt64, _ offset: UInt64 = 0) { let digit = yyinput[yycursor - 1] - UInt8(ascii: charOrigin) accum = min(accum * base + UInt64(digit) + offset, UInt64(UInt32.max) + 1) } } assert(parseUInt32("") == nil) assert(parseUInt32("1234567890") == 1234567890) assert(parseUInt32("0b1101") == 13) assert(parseUInt32("0x7Fe") == 2046) assert(parseUInt32("0644") == 420) assert(parseUInt32("9999999999") == nil) --storable-state re2swift (lexer) . re2swift "pull" () <<>>. ( (parser) ). : "push" () . . . "pull" : o YYSETSTATE() YYGETSTATE(state). o yych yyaccept ( ) state (persistent) . state -1 . o YYFILL . . o . o getstate YYGETSTATE . goto ( --loop-switch). o re2c:eof sentinel with bounds checks . (padding) : (greedy lexeme) . ( ). "push" . . <<>> . . // re2swift $INPUT -o $OUTPUT -f import Foundation func log(_ items: Any..., separator: String = " ", terminator: String = "\n") { #if DEBUG var stderr = FileHandle.standardError print(items, separator: separator, terminator: terminator, to: &stderr) #endif } extension FileHandle: TextOutputStream { public func write(_ string: String) { self.write(Data(string.utf8)) } } struct State { // Use a small buffer to cover the case when a lexeme doesn't fit, // in a real world use case use a larger buffer. static let bufferSize = 10 let file: FileHandle // Extra '\0' byte on buffer acts as terminator. var yyinput = ContiguousArray(repeating: 0, count: Self.bufferSize + 1) var yylimit = Self.bufferSize var yycursor = Self.bufferSize var yymarker = Self.bufferSize var token = Self.bufferSize var yystate = -1 } extension State { mutating func lex(recv: inout Int) -> Status { var yych: UInt8 = 0 lex: while true { self.token = self.yycursor /*!re2c re2c:api = record; re2c:eof = 0; re2c:variable:yyrecord = "self"; re2c:YYFILL = "return .waiting"; packet = [a-z]+[;]; * { return .badPacket } $ { return .end } packet { recv += 1 continue lex } */ } } mutating func fill() -> Status { let used = self.yylimit - self.token let free = Self.bufferSize - used // Error: No space. In the real world we can reallocate a larger buffer. if free < 1 { return .bigPacket } // Shift buffer contents, discarding everything up to the current lexeme. let shift = self.token self.yyinput.replaceSubrange(.. State.Status { // Create a "socket" (open the same file for reading and writing). let fname: String = "pipe" guard FileManager.default.createFile(atPath: fname, contents: nil), let fw = FileHandle(forWritingAtPath: fname) else { fatalError("cannot open '\(fname)'") } guard let fr = FileHandle(forReadingAtPath: fname) else { fatalError("cannot read file '\(fname)'") } defer { // Cleanup: remove input file. do { try fw.close() try fr.close() try FileManager.default.removeItem(atPath: fname) } catch { fatalError("error on file cleanup: \(error.localizedDescription)") } } var state = State(file: fr) // Main loop. The buffer contains incomplete data which appears packet by // packet. When the lexer needs more input it saves its internal state and // returns to the caller which should provide more input and resume lexing. var send = 0, recv = 0 while true { switch state.lex(recv: &recv) { case .end: log("done: got \(recv)") assert(recv == send) return .end case .waiting: log("waiting...") if send < packets.count { log("sent packet \(send)") packets[send].withUTF8Buffer { do { try fw.write(contentsOf: $0) } catch { fatalError("cannot write to \(fname): \(error.localizedDescription)") } } send += 1 } let status = state.fill() state.yyinput.withUnsafeBytes { let buf = $0.bindMemory(to: CChar.self) log("queue: '\(String(utf8String: buf.baseAddress!) ?? "")'") } if status == .bigPacket { log("error: packet too big") return .bigPacket } assert(status == .ready) case .badPacket: log("error: ill-formed packet") return .badPacket default: fatalError("unreachable") } } } assert(test([]) == .end) assert(test([ "zero;", "one;", "two;", "three;", "four;" ]) == .end) assert(test([ "zer0;" ]) == .badPacket) assert(test([ "looooooooooong;" ]) == .bigPacket) log("all tests completed successfully") /*!rules:re2c[:] ... */ %{rules[:] ... %} re2swift . . use . . use /*!use:re2c[:] ... */ %{use[:] ... %} . : ( ). use . use (lexer) : ( ). !use:; re2swift . . : . * $ ( ) . use re2swift ( ). 2.2 re2swift . ( ). 2.2 -r --reusable . 1.2 . !use // re2swift $INPUT -o $OUTPUT // This example shows how to combine reusable re2c blocks: two blocks // ('colors' and 'fish') are merged into one. The 'salmon' rule occurs // in both blocks; the 'fish' block takes priority because it is used // earlier. Default rule * occurs in all three blocks; the local (not // inherited) definition takes priority. /*!rules:re2c:colors * { fatalError("unreachable") } "red" | "salmon" | "magenta" { return .color } */ /*!rules:re2c:fish * { fatalError("unreachable") } "haddock" | "salmon" | "eel" { return .fish } */ func lex(_ yyinput: UnsafePointer) -> What { var yycursor = 0, yymarker = 0 /*!re2c re2c:yyfill:enable = 0; !use:fish; !use:colors; * { return .dunno } // Overrides inherited '*' rules */ } enum What { case color, fish, dunno } assert(lex("salmon") == .fish) assert(lex("what?") == .dunno) /*!use:re2c ... */ // re2swift $INPUT -o $OUTPUT --input-encoding utf8 // This example supports multiple input encodings: UTF-8 and UTF-32. // Both lexers are generated from the same rules block, and the use // blocks add only encoding-specific configurations. /*!rules:re2c re2c:yyfill:enable = 0; "x y" { return true } * { return false } */ func lexUTF8(_ yyinput: [UInt8]) -> Bool { var yycursor = 0, yymarker = 0 /*!use:re2c re2c:YYCTYPE = UInt8; re2c:encoding:utf8 = 1; */ } func lexUTF32(_ yyinput: [UInt32]) -> Bool { var yycursor = 0, yymarker = 0 /*!use:re2c re2c:YYCTYPE = UInt32; re2c:encoding:utf32 = 1; */ } assert(lexUTF8([ 0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79 ])) // UTF-8 assert(lexUTF32([ 0x00002200, 0x00000078, 0x00000020, 0x00002203, 0x00000079 ])) // UTF-32 (Submatch extraction) re2swift . Tags (tags) @stag #mtag stag mtag . -T --tags re2c:tags = 1 . : ( ) . S m ( ). svars mvars . (leftmost greedy) . Captures (capturing groups) . --captures re2c:captures = 1 . --leftmost-captures () --posix-captures POSIX . : (! ... ). --invert-captures re2c:invert-captures = 1 . yynmatch ( ) yypmatch . yynmatch yypmatch yypmatch [yynmatch * 2] . maxnmatch YYMAXNMATCH yynmatch . Captvars --captvars re2c:captvars = 1 . --captures : yynmatch yypmatch re2swift yytl yytr k- ( svars ). : --leftmost-captvars re2c:leftmost-captvars = 1 ( ) --posix-captvars re2c:posix-captvars POSIX. (Tagged Deterministic Finite Automata with Lookahead TDFA) . TDFA . TDFA DFA . . TDFA : . . stags mtags . . YYFILL . S : o s: t = YYCURSOR API C YYSTAGP(t) API o s: t = NULL API C YYSTAGN(t) API o s : t1 = t2 M : o m: YYMTAGP(t) API API o m: YYMTAGN(t) API API o m : t1 = t2 S ( ) . M . m ( ) m (v, p) v p . s : major minor patch ( ). YYFILL . // re2swift $INPUT -o $OUTPUT struct SemVer: Equatable { var major: Int, minor: Int, patch: Int } func lex(_ yyinput: UnsafePointer) -> SemVer? { let tagNone = -1 var yycursor = 0, yymarker = 0 // Final tag variables available in semantic action. /*!svars:re2c format = "var @@ = tagNone\n"; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = 'var @@ = tagNone\n'; */ /*!re2c re2c:yyfill:enable = 0; re2c:tags = 1; num = [0-9]+; @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\x00] { let s2n = { (range: Range) in range.reduce(0) { accum, idx in accum * 10 + Int(yyinput[idx] - UInt8(ascii: "0")) } } return SemVer( major: s2n(t1.. s YYFILL . YYFILL . s . ( m ) . // re2swift $INPUT -o $OUTPUT --api generic --tags import Foundation struct SemVer: Equatable { var major: Int, minor: Int, patch: Int } struct Input { static let bufferSize = 4095 static let tagNone = -1 var yyinput = ContiguousArray(repeating: 0, count: Self.bufferSize + 1) var yylimit = Self.bufferSize var yycursor = Self.bufferSize var yymarker = Self.bufferSize var token = Self.bufferSize // Intermediate tag variables must be part of the lexer state passed to YYFILL. // They don't correspond to tags and should be autogenerated by re2c. /*!stags:re2c format = " var @@ = Self.tagNone\n"; */ var eof = false let file: FileHandle } extension Input { mutating func lex() -> [SemVer]? { var semVers = [SemVer]() semVers.reserveCapacity(Self.bufferSize) // Final tag variables available in semantic action. /*!svars:re2c format = " var @@: Int\n"; */ parse: while true { self.token = self.yycursor /*!re2c re2c:api = record; re2c:eof = 0; re2c:tags = 1; re2c:yyrecord = "self"; re2c:YYFILL = "self.fill() == .ok"; num = [0-9]+; num @t1 "." @t2 num @t3 ("." @t4 num)? [\n] { semVers.append(SemVer( major: self.s2n(self.token..) -> Int { self.yyinput[range].reduce(0) { accum, digit in accum * 10 + Int(digit - UInt8(ascii: "0")) } } mutating func fill() -> FillStatus { guard !self.eof else { return .eof } let shift = self.token let used = self.yylimit - self.token let free = Self.bufferSize - used // Error: Lexeme too long. In the real world we could reallocate a larger buffer. guard self.token >= 1 else { return .longLexeme } // Shift buffer contents, discarding everything up to the current token. self.yyinput.replaceSubrange(.. . // re2swift $INPUT -o $OUTPUT struct SemVer: Equatable { var major: Int, minor: Int, patch: Int } func lex(_ yyinput: UnsafePointer) -> SemVer? { let tagNone = -1 var yycursor = 0, yymarker = 0 // Final tag variables available in semantic action. /*!svars:re2c format = " var @@: Int\n"; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = " var @@ = tagNone\n"; */ /*!re2c re2c:yyfill:enable = 0; re2c:captvars = 1; num = [0-9]+; (num) "." (num) ("." num)? [\x00] { let _ = yytl0; _ = yytr0 // Some variables are unused. return SemVer( major: s2n(yytl1..) -> Int { range.reduce(0) { accum, idx in accum * 10 + Int(yyinput[idx] - UInt8(ascii: "0")) } } } assert(lex("23.34") == SemVer(major: 23, minor: 34, patch: 0)) assert(lex("1.2.999") == SemVer(major: 1, minor: 2, patch: 999)) assert(lex("1.a") == nil) m . (trie) . // re2swift $INPUT -o $OUTPUT import Foundation // Unbounded number of version components. struct Ver: Equatable { let components: [Int] } func parse(_ yyinput: UnsafePointer) -> Ver? { let tagNone = -1 var yycursor = 0, yymarker = 0 var mTagPool = [MTag]() // Final tag variables available in semantic action. /*!svars:re2c format = " let @@: Int\n"; */ /*!mvars:re2c format = " let @@: MTag?\n"; */ // Intermediate tag variables used by the lexer (must be autogenerated). /*!stags:re2c format = " var @@ = tagNone\n"; */ /*!mtags:re2c format = " var @@: MTag? = nil\n"; */ /*!re2c re2c:tags = 1; re2c:yyfill:enable = 0; re2c:YYSTAGP = "@@ = yycursor"; re2c:YYSTAGN = "@@ = tagNone"; re2c:YYMTAGP = "@@ = add(tag: @@, value: yycursor)"; re2c:YYMTAGN = "@@ = add(tag: @@, value: tagNone)"; num = [0-9]+; @t1 num @t2 ("." #t3 num #t4)* [\x00] { var components = [s2n(t1..) -> Int { range.reduce(0) { accum, idx in accum * 10 + Int(yyinput[idx] - UInt8(ascii: "0")) } } // Append a single value to an m-tag history. func add(tag prevTag: MTag?, value: Int) -> MTag { let tag = MTag(prev: prevTag, element: value) mTagPool.append(tag) return tag } } // An m-tag tree is a way to store histories with an O(1) copy operation. // Histories naturally form a tree, as they have common start and fork at some // point. The tree is stored as an array of pairs (tag value, link to parent). // An m-tag is represented with a single link in the tree (array index). class MTag { weak var prev: MTag? // Link of predecessor node or root (nil) var elem: Int // Tag value init(prev: MTag?, element: Int) { self.prev = prev self.elem = element } } assert(parse("1") == Ver(components: [1])) assert(parse("1.2.3.4.5.6.7") == Ver(components: [1, 2, 3, 4, 5, 6, 7])) assert(parse("1.2.") == nil) (code points) (code units) . . . . (fixed-length) . (variable-length) . << >> ("any") [^] ( * ). (lexer) : yych YYCTYPE . . re2swift . : . : o ASCII ( ). [0-255] . o EBCDIC ( --ebcdic re2c:encoding:ebcdic ). [0-255] . o UCS2 ( --ucs2 re2c:encoding:ucs2 ). [0-0xFFFF] . o UTF8 ( --utf8 re2c:encoding:utf8 ). . . . o UTF16 ( --utf16 re2c:encoding:utf16 ). . . . o UTF32 ( --utf32 re2c:encoding:utf32 ). [0-0x10FFFF] . include include/unicode_blocks.re include/unicode_categories.re include/unicode_properties.re re2swift Unicode . --input-encoding . --input-encoding utf8 re2swift UTF8 ( --utf8 ). --encoding-policy re2swift ( [0xD800-0xDFFF]) . UTF8 . // re2swift $INPUT -o $OUTPUT -8 -i /*!include:re2c "unicode_categories.re" */ func lex(_ yyinput: UnsafePointer) -> Bool { var yycursor = 0, yymarker = 0 /*!re2c re2c:yyfill:enable = 0; // Simplified "Unicode Identifier and Pattern Syntax" // (see https://unicode.org/reports/tr31) id_start = L | Nl | [$_]; id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3]; identifier = id_start id_continue*; identifier { return true } * { return false } */ } assert(lex("_")) assert(lex("")) assert(!lex("+")) (Include files) re2swift /*!include:re2c FILE */ %{include FILE %} !include FILE ; FILE . re2swift include include -I . / include re2swift #include C/C++ : FILE / . include include . include --depfile . re2swift include include/ . ( Unicode) re2swift . include . (definitions.swift) enum Result { case ok, fail } /*!re2c number = [1-9][0-9]*; */ // floating-point numbers frac = [0-9]* "." [0-9]+ | [0-9]+ "."; exp = 'e' [+-]? [0-9]+; float = frac exp? | [0-9]+ exp; float { return .ok } (Input file) // re2swift $INPUT -o $OUTPUT -i /*!include:re2c "definitions.swift" */ func lex(_ yyinput: UnsafePointer) -> Result { var yycursor = 0, yymarker = 0 /*!re2c re2c:yyfill:enable = 0; * { return .fail } number { return .ok } !include "extra_rules.re.inc"; */ } assert(lex("123") == .ok) assert(lex("123.4567") == .ok) (Header files) re2swift --header re2c:header /*!header:re2c:on*/ /*!header:re2c:off*/ %{header:on%} %{header:off%} .re . . re2swift --header re2c:header ( stdout ). re2swift (translation units) . (tag variables) ( ). (Input file) // re2swift $INPUT -o $OUTPUT -i --header lexer/state.swift /*!header:re2c:on*/ struct LexerState { var yyinput: UnsafePointer var yycursor: Int /*!stags:re2c format = "var @@ = -1"; */ init(str: UnsafePointer, cur: Int = 0) { self.yyinput = str self.yycursor = cur } } /*!header:re2c:off*/ extension LexerState { mutating func lex() -> Int { let t: Int /*!re2c re2c:header = "lexer/state.swift"; re2c:api = record; re2c:tags = 1; re2c:yyfill:enable = 0; re2c:yyrecord = "self"; [a]* @t [b]* { return t } */ } } @main struct Program { static func main() { let str: StaticString = "ab" var state = LexerState(str: str.utf8Start) assert(state.lex() == 1) } } (Header file) /* Generated by re2swift */ struct LexerState { var yyinput: UnsafePointer var yycursor: Int var yyt1 = -1 init(str: UnsafePointer, cur: Int = 0) { self.yyinput = str self.yycursor = cur } } -S, --skeleton re2swift re2swift C . (lexer) . DFA ( ) re2swift : .input DFA .keys . .input . : o re2swift ( DFA ). o (benchmarking) . o (minimized test cases) ( ). ( ). re2swift DFA . : DFA . 1- ( ASCII UTF-8 EBCDIC) DFA . DFA : re2swift DFA . . re2swift ( ) . 1 (re2swift ). (Visualization and debug) -D, --emit-dot re2swift DFA DOT (dump) . Graphviz DFA . DFA . --dump-nfa --dump-dfa-raw ( ). (SEE ALSO) re2c : http://re2c.org. flex(1) lex(1) quex(http://quex.sourceforge.net). (AUTHORS) re2swift Peter Bumbulis (peter@csg.uwaterloo.ca) . Marcus Boerger Dan Nuffer . : Brian Young (bayoung@acm.org) Marcus Boerger Dan Nuffer (nuffer@users.sourceforge.net) Ulya Trofimovich (skvadrik@gmail.com) Serghei Iakovlev Sergei Trofimovich Petr Skocik ligfx raekye PolarGoose. RE2SWIFT(1)