RE2HS(1) RE2HS(1) (NAME) re2hs - Haskell (SYNOPSIS) re2hs [ OPTIONS ] [ WARNINGS ] INPUT - stdin . (INTRODUCTION) re2hs . ( Haskell ) / . . re2hs . Haskell . : -- re2hs $INPUT -o $OUTPUT -i {-# LANGUAGE OverloadedStrings #-} {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} import Data.ByteString (ByteString, index) data State = State { _yyinput :: ByteString, _yycursor :: Int } %{ re2c:YYFN = ["lexer;Bool", "State{..};State"]; re2c:yyfill:enable = 0; [1-9][0-9]* { True } * { False } %} main :: IO () main = case lexer State{_yyinput = "1234\0", _yycursor = 0} of True -> return () False -> error "lexer failed!" re2hs : -- Generated by re2hs {-# LANGUAGE RecordWildCards #-} -- re2hs $INPUT -o $OUTPUT -i {-# LANGUAGE OverloadedStrings #-} {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} import Data.ByteString (ByteString, index) data State = State { _yyinput :: ByteString, _yycursor :: Int } yy0 :: State -> Bool yy0 State{..} = let yych = index _yyinput _yycursor in let __ = _yycursor + 1 in let _yycursor = __ in case yych of _c | 0x31 <= _c && _c <= 0x39 -> yy2 State{..} _c | True -> yy1 State{..} yy1 :: State -> Bool yy1 State{..} = False yy2 :: State -> Bool yy2 State{..} = let yych = index _yyinput _yycursor in case yych of _c | 0x30 <= _c && _c <= 0x39 -> let __ = _yycursor + 1 in let _yycursor = __ in yy2 State{..} _c | True -> yy3 State{..} yy3 :: State -> Bool yy3 State{..} = True lexer :: State -> Bool lexer State{..} = yy0 State{..} main :: IO () main = case lexer State{_yyinput = "1234\0", _yycursor = 0} of True -> return () False -> error "lexer failed!" re2hs . : name = regular-expression ; name regular-expression . - . regular expressions re2hs . name . ( ) . (global scope) . . configuration = value ; re2hs . re2hs configurations . value - . . . regular-expression code regular-expression ( := ) . regular-expression code . . . : * $. ( encoding support ). handling the end of input . start conditions . !action code code ( ). actions . !directive ; . . directives . (Blocks) /*!re2c */ %{ %} ( ). 2.2 . : /*!re2c[:] ... */ or %{[:] ... %} . re2hs (DFA) . . options . /*!local:re2c[:] ... */ or %{local[:] ... %} ( ). /*!rules:re2c[:] ... */ or %{rules[:] ... %} -- . ( reusable blocks). 2.2 re2hs -r --reusable . /*!use:re2c[:] ... */ or %{use[:] ... %} use . re2hs . ( ). use . 2.2 re2hs use -r --reusable . /*!max:re2c[:[:...]] ... */ or %{max[:[:...]] ... %} YYMAXFILL . YYMAXFILL ( ). C (#define YYMAXFILL ) Go (var YYMAXFILL int = ) . format @@{max} ( @@) YYMAXFILL . /*!maxnmatch:re2c[:[:...]] ... */ or %{maxnmatch[:[:...]] ... %} YYMAXNMATCH ( -P --posix-captures ). YYMAXNMATCH ( ). C (#define YYMAXNMATCH ) Go (var YYMAXNMATCH int = ) . format @@{max} ( @@) YYMAXNMATCH . /*!stags:re2c[:[:...]] ... */, /*!mtags:re2c[:[:...]] ... */ or %{stags[:[:...]] ... %}, %{mtags[:[:...]] ... %{ s-tag/m-tag re2hs . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!svars:re2c[:[:...]] ... */, /*!mvars:re2c[:[:...]] ... */ or %{svars[:[:...]] ... %}, %{mvars[:[:...]] ... %{ s-tag/m-tag ( --tags) re2hs ( --captvars --posix-captvars) . ( ). : format separator. format @@{tag} ( @@) . separator format . /*!getstate:re2c[:[:...]] ... */ or %{getstate[:[:...]] ... %} (conditional dispatch) ( --storable-state ). . . . --loop-switch (cross-block) goto . /*!conditions:re2c[:[:...]] ... */, /*!types:re2c... */ or %{conditions[:[:...]] ... %}, %{types... %} (condition enumeration) ( --conditions ). ( ). YYCONDTYPE . format separator . format @@{cond} ( @@) @@{num} . separator format . /*!include:re2c */ or %{include %} . #include C/C++ . --depfile (build system) . /*!header:re2c:on*/ or %{header:on %} . header:off re2hs -t --type-header . /*!header:re2c:off*/ or %{header:off %} header:on*/ . /*!ignore:re2c ... */ or %{ignore ... %} . re2hs : re2c:api, re2c:input --api. re2c:api:sigil ("sigil") API . @@ . . $ ${name} . API . API re2c:YYFILL@len YYFILL . re2c:api:style API . functions ( C) free-form ( Go Rust) . functions API . ( YYFILL) (lexer) C/C++ (closures) Go Rust . free-form API : (interpolated) @@{var} @@ ( ). API re2c:YYFILL:naked YYFILL . re2c:bit-vectors, re2c:flags:bit-vectors, re2c:flags:b --bit-vectors . re2c:captures, re2c:leftmost-captures --leftmost-captures . re2c:captvars, re2c:leftmost-captvars --leftmost-captvars . re2c:case-insensitive, re2c:flags:case-insensitive --case-insensitive . re2c:case-inverted, re2c:flags:case-inverted --case-inverted . re2c:case-ranges, re2c:flags:case-ranges --case-ranges . re2c:computed-gotos, re2c:flags:computed-gotos, re2c:flags:g --computed-gotos . re2c:computed-gotos:relative, re2c:cgoto:relative --computed-gotos-relative . re2c:computed-gotos:threshold, re2c:cgoto:threshold goto if (bitmaps) . 9 . re2c:cond:abort (default case) (abort) . re2c:cond:goto :=> . goto @@; . @@ ( re2c:api:sigil re2c:cond:goto@cond ). re2c:cond:goto@cond re2c:cond:goto . @@ . re2c:api:sigil . re2c:cond:divider . /* *********************************** */ . (re2c:api;sigil re2c:cond:divider@cond ). re2c:cond:divider@cond re2c:cond:divider . @@ . re2c:api:sigil . re2c:cond:prefix, re2c:condprefix . yyc_ . re2c:cond:enumprefix, re2c:condenumprefix . yyc . re2c:debug-output, re2c:flags:debug-output, re2c:flags:d --debug-output . re2c:empty-class, re2c:flags:empty-class --empty-class . re2c:encoding:ebcdic, re2c:flags:ecb, re2c:flags:e --ebcdic . re2c:encoding:ucs2, re2c:flags:wide-chars, re2c:flags:w --ucs2 . re2c:encoding:utf8, re2c:flags:utf-8, re2c:flags:8 --utf8 . re2c:encoding:utf16, re2c:flags:utf-16, re2c:flags:x --utf16 . re2c:encoding:utf32, re2c:flags:unicode, re2c:flags:u --utf32 . re2c:encoding-policy, re2c:flags:encoding-policy --encoding-policy . re2c:eof (sentinel symbol) $ . -1 ( $ ). . . re2c:header, re2c:flags:type-header, re2c:flags:t . --header . re2c:indent:string . "\t" . (whitespace) . . re2c:indent:top . . . re2c:invert-captures --invert-captures . re2c:label:prefix, re2c:labelprefix DFA . yy . re2c:label:start, re2c:startlabel . . . . ( ). re2c:label:yyFillLabel YYFILL re2c:eof (storable state) . re2c:label:yyloop --loop-switch . yyloop . re2c:label:yyNext YYGETSTATE ( re2c:state:nextlabel). yyNext . re2c:lookahead, re2c:flags:lookahead ( --no-lookahead ). re2c:monadic ( Haskell )..TP re2c:nested-ifs, re2c:flags:nested-ifs, re2c:flags:s --nested-ifs . re2c:posix-captures, re2c:flags:posix-captures, re2c:flags:P --posix-captures . re2c:posix-captvars --posix-captvars . re2c:tags, re2c:flags:tags, re2c:flags:T --tags . re2c:tags:expression . re2hs yyt . struct . @@{tag} @@ . re2c:tags:expression = "s.@@"; s.yyt . re2c:api:sigil . re2c:tags:negative ( -1 null ). re2c:tags:prefix . yyt . re2c:sentinel (sentinel) ( re2c:yyfill:enable = 0; re2c:eof ) . : . -1` ( re2hs ). . re2c:state:abort (default case) (abort) -1 . re2c:state:nextlabel YYGETSTATE yyNext ( ). re2c:label:start getstate YYGETSTATE . re2c:unsafe, re2c:flags:unsafe --no-unsafe . unsafe YYPEEK . ( ). Rust . re2c:YYBACKUP, re2c:define:YYBACKUP API YYBACKUP . re2c:YYBACKUPCTX, re2c:define:YYBACKUPCTX API YYBACKUPCTX . re2c:YYCONDTYPE, re2c:define:YYCONDTYPE API YYCONDTYPE . re2c:YYCTYPE, re2c:define:YYCTYPE API YYCTYPE . re2c:YYCTXMARKER, re2c:define:YYCTXMARKER API YYCTXMARKER . re2c:YYCURSOR, re2c:define:YYCURSOR API YYCURSOR . re2c:YYDEBUG, re2c:define:YYDEBUG API YYDEBUG . re2c:YYFILL, re2c:define:YYFILL API YYFILL . re2c:YYFILL@len, re2c:define:YYFILL@len (sigil) YYFILL . @@ . re2c:api:sigil . re2c:YYFILL:naked, re2c:define:YYFILL:naked re2c:api:style YYFILL . API (free-form) . re2c:YYFN API YYFN . re2c:YYINPUT API YYINPUT . re2c:YYGETCOND, re2c:define:YYGETCONDITION API YYGETCOND . re2c:YYGETCOND:naked, re2c:define:YYGETCONDITION:naked re2c:api:style YYGETCOND . API . re2c:YYGETSTATE, re2c:define:YYGETSTATE API YYGETSTATE . re2c:YYGETSTATE:naked, re2c:define:YYGETSTATE:naked re2c:api:style YYGETSTATE . API . re2c:YYGETACCEPT, re2c:define:YYGETACCEPT API YYGETACCEPT . re2c:YYLESSTHAN, re2c:define:YYLESSTHAN API YYLESSTHAN . re2c:YYLIMIT, re2c:define:YYLIMIT API YYLIMIT . re2c:YYMARKER, re2c:define:YYMARKER API YYMARKER . re2c:YYMTAGN, re2c:define:YYMTAGN API YYMTAGN . re2c:YYMTAGP, re2c:define:YYMTAGP API YYMTAGP . re2c:YYPEEK, re2c:define:YYPEEK API YYPEEK . re2c:YYRESTORE, re2c:define:YYRESTORE API YYRESTORE . re2c:YYRESTORECTX, re2c:define:YYRESTORECTX API YYRESTORECTX . re2c:YYRESTORETAG, re2c:define:YYRESTORETAG API YYRESTORETAG . re2c:YYSETCOND, re2c:define:YYSETCONDITION API YYSETCOND . re2c:YYSETCOND@cond, re2c:define:YYSETCONDITION@cond (sigil) YYSETCOND . @@ . re2c:api:sigil . re2c:YYSETCOND:naked, re2c:define:YYSETCONDITION:naked re2c:api:style YYSETCOND . API . re2c:YYSETSTATE, re2c:define:YYSETSTATE API YYSETSTATE . re2c:YYSETSTATE@state, re2c:define:YYSETSTATE@state (sigil) YYSETSTATE . @@ . re2c:api:sigil . re2c:YYSETSTATE:naked, re2c:define:YYSETSTATE:naked re2c:api:style YYSETSTATE . API . re2c:YYSETACCEPT, re2c:define:YYSETACCEPT API YYSETACCEPT . re2c:YYSKIP, re2c:define:YYSKIP API YYSKIP . re2c:YYSHIFT, re2c:define:YYSHIFT API YYSHIFT . re2c:YYCOPYMTAG, re2c:define:YYCOPYMTAG API YYCOPYMTAG . re2c:YYCOPYSTAG, re2c:define:YYCOPYSTAG API YYCOPYSTAG ..TP re2c:YYSHIFTMTAG, re2c:define:YYSHIFTMTAG API YYSHIFTMTAG . re2c:YYSHIFTSTAG, re2c:define:YYSHIFTSTAG API YYSHIFTSTAG . re2c:YYSTAGN, re2c:define:YYSTAGN API YYSTAGN . re2c:YYSTAGP, re2c:define:YYSTAGP API YYSTAGP . re2c:yyaccept, re2c:variable:yyaccept API yyaccept . re2c:yybm, re2c:variable:yybm API yybm . re2c:yybm:hex, re2c:variable:yybm:hex --bit-vectors . ( ). re2c:yych, re2c:variable:yych API yych . re2c:yych:emit, re2c:variable:yych:emit yych . . re2c:yych:conversion, re2c:variable:yych:conversion re2hs yych YYCTYPE . ( ). re2c:yych:literals, re2c:variable:yych:literals yych . : char ( ) hex ( ) char_or_hex ( ). re2c:yyctable, re2c:variable:yyctable API yyctable . re2c:yynmatch, re2c:variable:yynmatch API yynmatch . re2c:yypmatch, re2c:variable:yypmatch API yypmatch . re2c:yytarget, re2c:variable:yytarget API yytarget . re2c:yystable, re2c:variable:yystable . re2c:yystate, re2c:variable:yystate API yystate . re2c:yyfill, re2c:variable:yyfill API yyfill . re2c:yyfill:check YYFILL ( YYLESSTHAN API YYLIMIT API C) . ( ). re2c:yyfill:enable YYFILL ( ) . ( ) YYFILL ( ). ( -W) re2c:sentinel . (YYFILL ). re2c:yyfill:parameter YYFILL . . ( ). re2c:YYFILL:naked re2c:api:style . re2c:yyfn:sep YYFN ( ). re2c:yyfn:throw YYFN ( ). re2hs : "foo" . 'foo' . [a-xyz], [^a-xyz] ( ). . (newline). R \ S R S. R* R. R+ R. R? R. R{n} R n . R{n,} R n . R{n,m} R n m . (R) R . --invert-captures (R) . (!R) --invert-captures (!R) . R S : R S. R | S : R S. R / S (Lookahead): R S S . name name ( "name" Flex). {name} name Flex. @stag s-tag: @stag stag . #mtag m-tag: #mtag mtag . $ . : \a \b \f \n \r \t \v \\ (octal) \ooo (hexadecimal) \xhh \uhhhh \Uhhhhhhhh. (Actions) re2hs : !entry code (Entry action) code . (start conditions) . (lexeme). !pre_rule code (Pre-rule action) code ( (start conditions)) . ( ). !post_rule code (Post-rule action) code ( (start conditions)) . (trap) . re2hs : !use:name ; use name . . : ( ). * $ . use use . !include file ; include : file . (Program interface) primitives ( ) API . . . re2hs ( ) . API . re2hs (API flavor) : Record API Record API backend Haskell . API (binding) yyrecord ( re2c:yyrecord ) _yyinput _yycursor _yymarker _yyctxmarker _yylimit . . Generic API API . --api generic re2c:api = generic . : YYPEEK YYSKIP YYBACKUP YYBACKUPCTX YYSTAGP YYSTAGN YYMTAGP YYMTAGN YYRESTORE YYRESTORECTX YYRESTORETAG YYCOPYSTAG YYCOPYMTAG YYSHIFT YYSHIFTSTAG YYSHIFTMTAG YYLESSTHAN YYEND. API . YYCTYPE ( code units). ASCII EBCDIC UTF-8 . UTF-16 UCS-2 . UTF-32 . YYCURSOR (l-value) ( YYINPUT). YYCURSOR . YYCURSOR . YYLIMIT (r-value) ( YYINPUT). YYLIMIT . . YYCURSOR YYLIMIT . YYMARKER (l-value) ( YYINPUT). YYCURSOR (rollback) . . YYCTXMARKER (l-value) (trailing context) ( YYINPUT). . YYCTXMARKER / . YYFILL API len. YYFILL len . re2c:eof len 1 YYFILL . re2c:eof YYFILL . len YYMAXFILL . YYFN --recursive-functions . re2c:fn:sep ( ) . ( void ). : ( ) ( - ). YYINPUT (r-value) ( ) . YYMAXFILL YYFILL. max . YYLESSTHAN API len. (r-value) len true . YYEND API . (r-value) ( ) true . YYEND $ . YYLESSTHAN . YYPEEK API . (r-value) YYCTYPE . YYSKIP API . YYBACKUP API ( YYRESTORE ). YYRESTORE API YYBACKUP . YYBACKUPCTX API ( YYRESTORECTX ). YYRESTORECTX API YYBACKUPCTX . YYRESTORETAG API tag tag . YYSTAGP API tag tag YYINPUT ( ). YYSTAGP tag ..TP YYSTAGN API tag tag YYINPUT ( ). YYSTAGN tag . YYMTAGP API tag. YYMTAGP tag ( ). YYMTAGN API tag. YYMTAGN tag ( ). YYSHIFT API shift shift ( ). YYCOPYSTAG API lhs rhs s-tag rhs s-tag lhs . lhs rhs . YYCOPYMTAG API lhs rhs m-tag rhs m-tag lhs . lhs rhs . YYSHIFTSTAG API tag shift tag shift (code unit) ( ). YYSHIFTMTAG API tag shift tag shift ( ). YYMAXNMATCH (capturing groups) POSIX . maxnmatch . YYCONDTYPE (enum) . conditions --header . YYGETACCEPT var . var . YYSETACCEPT : var ( l-value ) val ( ). val var . YYGETCOND r-value YYCONDTYPE . YYSETCOND cond cond . YYGETSTATE r-value (lexer) . -1 . YYSETSTATE state state . YYDEBUG -d --debug-output . () ( YYDEBUG ). YYDEBUG : state ( DFA -1) symbol ( ). yyaccept l-value . --storable-state . yybm ( --bitmaps ). . . k- n- (true) n- k- . if switch . yych l-value YYCTYPE . -f --storable-state . yyctable ( --conditions --computed-gotos). yyfill l-value YYFILL ( YYFILL ). yynmatch l-value POSIX . -P --posix-captures . yypmatch l-value . yynmatch * 2 ( YYMAXNMATCH * 2 ). -P --posix-captures . yystable . yystate l-value --loop-switch DFA . yytarget ( ) . . yytarget --computed-gotos . re2c . re2c (debug) . re2c . -? --help -h . --api API . simple . record (struct class ) . generic API . --bit-vectors -b . --nested-ifs . --captures, --leftmost-captures (leftmost greedy). yybmatch 2 * YYMAXNMATCH yynmatch . --captvars, --leftmost-captvars . yytl yytr k- . --case-insensitive . --case-inverted : . --case-ranges case switch low ... high. C/C++ GCC Clang Tcc . case Tcc . --computed-gotos -g "computed goto" ( ). re2hs . cgoto:threshold . cgoto:relative . --bit-vectors . --computed-gotos-relative --computed-gotos ( ). --computed-gotos . --conditions --start-conditions -c "conditions" Flex: . re2hs goto . --depfile FILE FILE Makefile : [include-file ...]. / include include . --output . --ebcdic --ecb -e EBCDIC . re2hs 0 -- 0xFF . --empty-class re2hs . match-empty () ( ). match-none . error . --encoding-policy re2hs (Unicode surrogates). fail re2hs . substitute re2hs 0xFFFD . ignore () re2hs . . --flex-syntax -F Flex: . . --goto-label "goto/label": DFA goto . goto . --header --type-header -t HEADER HEADER. header:on header:off . enum ( conditions ). -I PATH PATH include . include . re2hs FILE include -I . --input --api. default simple ( ). custom generic . --input-encoding re2hs. ascii () re2hs ASCII : . utf8 re2hs UTF8 . --invert-captures . (...) (! ...) . (! ...) (...) . --lang . C D Go Haskell Java JS OCaml Python Rust Swift V Zig ( --syntax ). none . --location-format . gnu 'filename:line:column: ...' . msvc 'filename(line,column) ...' . gnu . --loop-switch "loop/switch": DFA switch case switch . yystate . yystate case . --nested-ifs -s if switch . . --no-debug-info -i (line directives). ( (diff) ). --no-generation-date . --no-version . --no-unsafe unsafe YYPEEK ( Rust ). YYPEEK (bounds-checking) . YYPEEK (release) unsafe --no-unsafe unsafe . --output -o OUTPUT OUTPUT. --posix-captures, -P POSIX. yybmatch 2 * YYMAXNMATCH yynmatch ..TP --posix-captvars POSIX. k- yytl yytr . --recursive-functions (co-recursive) DFA - . --reusable -r 2.2 ( ). --skeleton -S <<>> ("skeleton"). <<>> . . C . --storable-state -f (lexer) . (push-model) . YYGETSTATE YYSETSTATE yych yyaccept state . --syntax FILE FILE (syntax) . FILE ( ) ( --lang none ). --tags -T (tags). --ucs2 --wide-chars -w UCS2 . re2hs 0 -- 0xFFFF 2 . --nested-ifs . --utf8 --utf-8 -8 UTF-8 . re2hs 0 -- 0x10FFFF 1 . --utf16 --utf-16 -x UTF16 . re2hs 0 -- 0x10FFFF 2 . --nested-ifs . --utf32 --unicode -u UTF32 . re2hs 0 -- 0x10FFFF 4 . --nested-ifs . --verbose . --vernum -V MMmmpp ( ). --version -v . --single-pass -1 . ( ). --debug-output -d YYDEBUG . . --dump-adfa : DFA (tunneling) ( .dot). --dump-cfg : (CFG) ( .dot). --dump-closure-stats : (closure). --dump-dfa-det : DFA (determinization) ( .dot). --dump-dfa-min : DFA (minimization) ( .dot). --dump-dfa-tagopt : DFA ( .dot). --dump-dfa-tree : DFA ( .dot). --dump-dfa-raw : DFA ( .dot). --dump-interf : (liveness analysis) . --dump-nfa : NFA ( .dot). --emit-dot -D .dot . Graphviz ( dot -Tpng -odfa.png dfa.dot). --dfa-minimization : DFA re2hs. moore (Moore) ( ). table << >> ("table filling") . DFA . --eager-skip : (eagerly) -- . (lazily) -- -- . --no-lookahead . TDFA(0) . TDFA(1) TDFA(0) . . TDFA(0) TDFA(1) . --no-optimize-tags : ( ). --posix-closure : POSIX: gor1 () - (Goldberg-Radzik) gtop << >> ("global topological order"). --posix-prectable : POSIX. complex TNFA . naive TNFA complex . --stadfa . staDFA TDFA . staDFA TDFA . --fixed-tags : (all) (none) (toplevel). all . <<>> ( <<>> "base") . . ( ). . . -W . -Werror . . -W warning. -Wno- warning. -Werror- warning ( -W ). -Wno-error- warning . . -Wcondition-order . --header conditions . -Wempty-character-class . : . re2hs . --empty-class . -Wmatch-empty-string ( ). . -Wswapped-range . . -Wundefined-control-flow ( ). . * (code unit) . -Wunreachable-rules (shadowed) . -Wdeprecated-eof_rule $ . $ . -Wuseless-escape (escaped) . re2hs (escape sequence) . -Wnondeterministic-tags n- n . -Wsentinel-in-midrule (sentinel) --- (crash) . . re2c:sentinel . -Wundefined-syntax-config --syntax . : . . re2c (syntax files) . -- . re2c re2c: : . ( include/syntax ) re2hs . ( --lang none ). : . (Feature lists) re2hs : supported_apis API simple record generic. supported_api_styles API functions free-form. supported_code_models goto-label loop-switch recursive-functions. supported_targets (codegen) code dot skeleton. supported_features nested-ifs bitmaps computed-gotos case-ranges monadic unsafe tags captures captvars. (Language configurations) re2hs : semicolons . backtick_quoted_strings . single_quoted_strings . indentation_sensitive . wrap_blocks_in_braces . (Inplace configurations) re2c: . . (Code templates) . : code-template :: name '=' code-exprs ';' | CODE_TEMPLATE ';' | '' ';' code-exprs :: | code-exprs code-expr code-expr :: STRING | VARIABLE | optional | list optional :: '(' CONDITIONAL '?' code-exprs ')' | '(' CONDITIONAL '?' code-exprs ':' code-exprs ')' list :: '[' VARIABLE ':' code-exprs ']' | '[' VARIABLE '{' NUMBER '}' ':' code-exprs ']' | '[' VARIABLE '{' NUMBER ',' NUMBER '}' ':' code-exprs ']' . . : re2hs . ( -1 ). . ( ) ( ) . re2hs . if-then-else C : code:if_then_else = [branch{0}: topindent "if " cond " {" nl indent [stmt: stmt] dedent] [branch{1:-1}: topindent "} else" (.cond ? " if " cond) " {" nl indent [stmt: stmt] dedent] topindent "}" nl; branch : branch{0} ( else ) branch{1:-1} ( ) . stmt : [stmt: stmt] . topindent indent dedent nl .cond ( ). : if x { // do something } else if y { // do something else } else { // don't do anything } re2hs . . . code:var_local . : type ( ) name ( ) init ( ). : .init ( ). code:var_global code:var_local (top-level) . code:const_local . : type ( ) name ( ) init ( ). code:const_global code:const_local (top-level) . code:array_local (). : type ( ) name ( ) size ( ) row ( ) elem ( -- row ). : .const ( ). code:array_global code:array_local (top-level) . code:array_elem (). : array ( ) index ( ). code:enum enum ( ). type ( ) elem ( ) init ( ). : .init ( ). code:enum_elem ( code:enum). name ( ) type ( ). code:assign . lhs ( ) rhs ( ). code:type_int (signed). code:type_uint (unsigned). code:type_yybm yybm. code:type_yytarget yytarget. code:type_yyctable yyctable. code:cmp_eq "". code:cmp_ne "". code:cmp_lt " ". code:cmp_gt " ". code:cmp_le " ". code:cmp_ge " ". code:if_then_else If-then-else . : branch ( ) cond ( ) stmt ( ). : .cond ( ) .many ( ). code:if_then_else_oneline code:if_then_else . code:if_then_else . code:switch switch (case). : expr ( switch ) case ( case ). code:switch_cases switch . case ( case ) stmt ( ). code:switch_cases_oneline code:switch_cases . code:switch_cases . code:switch_case_range switch ( ). : val ( ). : .many ( ) .char_literals ( switch -- ). code:switch_case_default (default) switch. code:loop ( ). : label ( ) stmt ( ). code:continue Continue. : label ( ). code:goto Goto. : label ( ). code:cgoto goto (computed). : array ( goto ) index ( ) base ( .cgoto.relative ). code:cgoto:data goto . : label ( ) type ( ) base ( - .cgoto.relative ). code:fndecl . : name ( ) type ( ) throw ( re2c:yyfn:throw ) arg ( ) argname ( ) argtype ( ). : .type ( void ). code:fndef code:fndecl stmt . code:fncall . : name ( ) retval ( l-value ) arg ( ). : .args ( ) .retval ( ). code:tailcall (tail call). : name ( ) arg ( ). : .args ( ) .retval ( void ). code:recursive_functions --recursive-functions. : fn ( ) fndecl ( ) fndef ( )..TP code:fingerprint (fingerprint) . : ver ( re2hs ) date ( ). code:line_info ( ). : line ( ) file ( ). code:abort (abort) . code:yydebug YYDEBUG API . : YYDEBUG yyrecord yych ( re2c:) state ( DFA). code:yypeek YYPEEK API . : YYPEEK YYCTYPE YYINPUT YYCURSOR yyrecord yych ( re2c:). : .cast ( re2c:yych:conversion ). code:yyskip YYSKIP API . : YYSKIP YYCURSOR yyrecord ( re2c:). code:yybackup YYBACKUP API . : YYBACKUP YYCURSOR YYMARKER yyrecord ( re2c:). code:yybackupctx YYBACKUPCTX API . : YYBACKUPCTX YYCURSOR YYCTXMARKER yyrecord ( re2c:). code:yyskip_yypeek code:yyskip code:yypeek ( code:yyskip code:yypeek ). code:yypeek_yyskip code:yypeek code:yyskip ( code:yypeek code:yyskip ). code:yyskip_yybackup code:yyskip code:yybackup ( code:yyskip code:yybackup ). code:yybackup_yyskip code:yybackup code:yyskip ( code:yybackup code:yyskip ). code:yybackup_yypeek code:yybackup code:yypeek ( code:yybackup code:yypeek ). code:yyskip_yybackup_yypeek code:yyskip code:yybackup code:yypeek ( ``code:yyskip`` code:yybackup code:yypeek ). code:yybackup_yypeek_yyskip code:yybackup code:yypeek code:yyskip ( ``code:yybackup`` code:yypeek code:yyskip ). code:yyrestore YYRESTORE API . : YYRESTORE YYCURSOR YYMARKER yyrecord ( re2c:). code:yyrestorectx YYRESTORECTX API . : YYRESTORECTX YYCURSOR YYCTXMARKER yyrecord ( re2c:). code:yyrestoretag YYRESTORETAG API . : YYRESTORETAG YYCURSOR yyrecord ( re2c:) tag ( ). code:yyshift YYSHIFT API . : YYSHIFT YYCURSOR yyrecord ( re2c:) offset ( ). code:yyshiftstag YYSHIFTSTAG API . : YYSHIFTSTAG yyrecord negative ( re2c:) tag ( ) offset ( ). : .nested ( -- re2c:tags:negative ). code:yyshiftmtag YYSHIFTMTAG API . : YYSHIFTMTAG ( re2c:) tag ( ) offset ( ). code:yystagp YYSTAGP API . : YYSTAGP YYCURSOR yyrecord ( re2c:) tag ( ). code:yymtagp YYMTAGP API . : YYMTAGP ( re2c:) tag ( ). code:yystagn YYSTAGN API . : YYSTAGN negative yyrecord ( re2c:) tag ( ). code:yymtagn YYMTAGN API . : YYMTAGN ( re2c:) tag ( ). code:yycopystag YYCOPYSTAG API . : YYCOPYSTAG yyrecord ( re2c:) lhs rhs ( ). code:yycopymtag YYCOPYMTAG API . : YYCOPYMTAG yyrecord ( re2c:) lhs rhs ( ). code:yygetaccept YYGETACCEPT API . : YYGETACCEPT yyrecord ( re2c:) var ( re2c:yyaccept). code:yysetaccept YYSETACCEPT API . : YYSETACCEPT yyrecord ( re2c:) var ( re2c:yyaccept) val ( ). code:yygetcond YYGETCOND API . : YYGETCOND yyrecord ( re2c:) var ( re2c:yycond). code:yysetcond YYSETCOND API . : YYSETCOND yyrecord ( re2c:) var ( re2c:yycond) val ( ). code:yygetstate YYGETSTATE API . : YYGETSTATE yyrecord ( re2c:) var ( re2c:yystate). code:yysetstate YYSETSTATE API . : YYSETSTATE yyrecord ( re2c:) var ( re2c:yystate) val ( ). code:yylessthan YYLESSTHAN API . : YYLESSTHAN YYCURSOR YYLIMIT yyrecord ( re2c:) need ( ). : .many ( need )..TP code:yyend YYEND API . : YYEND YYCURSOR YYLIMIT. code:yybm_filter yych yybm ( --bitmaps ). : yych ( re2c:yych ). code:yybm_match yybm ( --bitmaps). : yybm yych ( re2c: ) offset ( yybm yych ) mask ( ). code:yytarget_filter yych yytarget ( --computed-gotos ). : yych ( re2c:yych ). (syntax) : nl (newline). indent . dedent . topindent . . : .api.simple (True) API (--api simple re2c:api = simple). .api.generic API (generic) (--api generic re2c:api = generic). .api.record API (--api record re2c:api = record). .api_style.functions API (re2c:api-style = functions). .api_style.freeform API (re2c:api-style = free-form). .case_ranges case (--case-ranges re2c:case-ranges = 1). .cgoto.relative goto (--computed-gotos-relative re2c:cgoto:relative = 1). .code_model.goto_label goto/label (--goto-label). .code_model.loop_switch loop/switch (--loop-switch). .code_model.recursive_functions (--recursive-function). .date (fingerprint) . .loop_label re2hs (re2c:label:yyloop ). .monadic (monadic) (re2c:monadic = 1). . .start_conditions (--start-conditions). .storable_state (--storable-state). .unsafe re2hs "unsafe" (--unsafe re2c:unsafe = 1). "unsafe" . .version re2hs . .yyfill.enable YYFILL (re2c:yyfill:enable = 1). .yyfn.throw re2c:yyfn:throw . (lexer) . : o ( *) (final state) o (default state) o <<>> : . : . . re2hs . . : o (Sentinel). . <<>> . . : . . YYFILL re2c:yyfill:enable = 0; re2c:eof -1 . o (Sentinel with bounds checks). : . . <<>> <<>> . ( ). (bounds check) . . YYFILL ( ). . . re2c:eof ( ). YYFILL . o (Bounds checks with padding). << >> . DFA (SCC) SCC SCC . . (padding) ( ) . YYMAXFILL max . YYFILL . YYFILL re2c:eof -1 ( ). o (Custom checks). (Generic API) . . API --api custom re2c:api = custom; re2c:yyfill:enable = 0; re2c:yyfill:check = 0; . ..SS (Sentinel) . null . null : (lexeme) ( ). null (lexer) * ( -Wsentinel-in-midrule re2c:sentinel ). re2c:yyfill:enable = 0; YYFILL . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import Data.ByteString (ByteString, index) data State = State { _yyinput :: ByteString, _yycursor :: Int, _count :: Int } -- expect a null-terminated string %{ re2c:YYFN = ["lexer;Int", "State{..};State"]; re2c:yyfill:enable = 0; * { (-1) } [\x00] { _count } [a-z]+ { lexer State{_count = _count + 1, ..} } [ ]+ { lexer State{..} } %} main :: IO () main = do let test s n = when (lexer st /= n) $ error "failed" where st = State{_yyinput = s, _yycursor = 0, _count = 0} test "\0" 0 test "one two three\0" 3 test "f0ur\0" (-1) (Sentinel with bounds checks) ( 1.2 ). . null re2c:eof = 0; . sentinel null ( 'aaa\0aa'\0 'aaa\0 ). null . API YYLIMIT <= YYCURSOR YYLESSTHAN(1) . (YYFILL re2c:yyfill:enable = 0; YYFILL YYFILL with sentinel ). : $ ( *) -Wundefined-control-flow . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import qualified Data.ByteString as BS import Data.Word data State = State { _yyinput :: BS.ByteString, _yycursor :: Int, _yymarker :: Int, _yylimit :: Int, _count :: Int } -- expect a null-terminated string %{ re2c:YYFN = ["lexer;Int", "State{..};State"]; re2c:YYCTYPE = "Word8"; re2c:YYPEEK = "BS.index"; re2c:eof = 0; re2c:yyfill:enable = 0; str = ['] ([^'\\] | [\\][^])* [']; * { (-1) } $ { _count } str { lexer State{_count = _count + 1, ..} } [ ]+ { lexer State{..} } %} main :: IO () main = do let test s n = do let st = State { _yyinput = s, _yycursor = 0, _yymarker = 0, _yylimit = BS.length s - 1, -- terminating null not included _count = 0} when (lexer st /= n) $ error "failed" test "\0" 0 test "'qu\0tes' 'are' 'fine: \\'' \0" 3 test "'unterminated\\'\0" (-1) (padding) ( ). . YYMAXFILL null YYMAXFILL max . null --- (lexeme) ( ). "stop" (null) ( null ). (strongly connected components) . (YYLIMIT - YYCURSOR) < n API YYLESSTHAN(n) n ( n ). YYFILL(n) n . YYFILL ( ). YYFILL YYFILL with padding . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Exception import Control.Monad (when) import qualified Data.ByteString as BS data State = State { _yyinput :: BS.ByteString, _yycursor :: Int, _yylimit :: Int, _count :: Int } data FillException = UnexpectedFill deriving (Show) instance Exception FillException yymaxfill :: Int %{max %} %{ re2c:YYFN = ["lexer;IO Int", "State{..};State"]; re2c:YYPEEK = "BS.index"; re2c:YYFILL = "throw UnexpectedFill"; re2c:monadic = 1; // YYFILL requires monadic do-notation for `when` conditions str = ['] ([^'\\] | [\\][^])* [']; [\x00] { -- check that it is the sentinel, not some unexpected null return $ if _yycursor == BS.length _yyinput - yymaxfill + 1 then _count else (-1) } str { lexer State{_count = _count + 1, ..} } [ ]+ { lexer State{..} } * { return (-1) } %} main :: IO () main = do let test s n = do let buf = BS.concat [s, BS.replicate yymaxfill 0] let st = State { _yyinput = buf, _yycursor = 0, _yylimit = BS.length buf, _count = 0} m <- catch (lexer st) (\(_ :: FillException) -> return (-2)) when (m /= n) $ error "failed" test "" 0 test "'unterminated\\'" (-2) test "'qu\0tes' 'are' 'fine: \\'' " 3 test "'unexpected' \0 'null'" (-1) generic API . . sentinel null . YYPEEK . . YYPEEK . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import qualified Data.ByteString as BS data State = State { _str :: BS.ByteString, _cur :: Int, _lim :: Int, _cnt :: Int } -- Expect a string without terminating null. %{ re2c:api = generic; re2c:YYFN = ["lexer;Int", "State{..};State"]; re2c:YYPEEK = "if _cur < _lim then BS.index _str _cur else 0"; re2c:YYSKIP = "let cur = _cur + 1 in let _cur = cur in"; re2c:yyfill:enable = 0; * { (-1) } [\x00] { _cnt } [a-z]+ { lexer State{_cnt = _cnt + 1, ..} } [ ]+ { lexer State{..} } %} main :: IO () main = do let test s n = let st = State {_str = s, _cur = 0, _lim = BS.length s, _cnt = 0} in when (lexer st /= n) $ error "failed" test "" 0 test "one two three " 3 test "f0ur" (-1) (BUFFER REFILLING) : ( ) . . . : o (cursor): (YYCURSOR C pointer API YYSKIP/YYPEEK generic API) o (limit): (YYLIMIT C pointer API YYLESSTHAN generic API ) o (marker): (YYMARKER default API YYBACKUP/YYRESTORE generic API) o (token): (lexeme) ( re2hs API ) o (context marker): (YYCTXMARKER C pointer API YYBACKUPCTX/YYRESTORECTX generic API) o (tag variables): ( stags mtags generic API YYSTAGP/YYSTAGN/YYMTAGP/YYMTAGN) YYFILL . token cursor token token limit . YYFILL ( ). YYFILL EOF : EOF (padding) . -f --storable-state YYFILL ( )..SS "YYFILL (sentinel)" EOF YYFILL . YYFILL YYLIMIT <= YYCURSOR API C YYLESSTHAN() API . YYFILL . YYFILL . Limit limit re2c:eof . YYFILL ( # ). <-- shift --> >-A------------B---------C-------------D#-----------E-> buffer token marker limit, cursor >-A------------B---------C-------------D------------E#-> buffer, marker cursor limit token <-- shift --> >-A------------B---------C-------------D#--E (EOF) buffer token marker limit, cursor >-A------------B---------C-------------D---E#........ buffer, marker cursor limit token input.txt EOF . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad import Data.ByteString as BS import Data.Word import GHC.IO.Handle import System.Directory import System.IO chunk_size :: Int chunk_size = 4096 data State = State { _file :: !Handle, _yyinput :: !BS.ByteString, _yycursor :: !Int, _yymarker :: !Int, _yylimit :: !Int, _token :: !Int, _eof :: !Bool, _count :: !Int } %{ re2c:YYFN = ["lexer;IO Int", "State{..};State;!State{..}"]; re2c:YYCTYPE = "Word8"; re2c:YYPEEK = "BS.index"; re2c:YYFILL = "(State{..}, yyfill) <- fill State{..}"; re2c:eof = 0; re2c:monadic = 1; str = ['] ([^'\\] | [\\][^])* [']; * { return (-1) } $ { return _count } str { lexer State{_token = _yycursor, _count = _count + 1, ..} } [ ]+ { lexer State{_token = _yycursor, ..} } %} fill :: State -> IO (State, Bool) fill State{..} = do case _eof of True -> return (State{..}, False) False -> do -- Discard everything up to the current token, cut off terminating null, -- read new chunk from file and reappend terminating null at the end. chunk <- BS.hGet _file chunk_size return (State { _yyinput = BS.concat [(BS.init . BS.drop _token) _yyinput, chunk, "\0"], _yycursor = _yycursor - _token, _yymarker = _yymarker - _token, _yylimit = _yylimit - _token + BS.length chunk, -- exclude terminating null _token = 0, _eof = BS.null chunk, -- end of file? ..}, True) main :: IO () main = do let fname = "input" -- Prepare input file. BS.writeFile fname $ BS.concat ["'qu\0tes' 'are' 'fine: \\'' " | _ <- [1..chunk_size]] let expect = 3 * chunk_size -- the total number of strings in file -- Run lexer on the prepared file. fh <- openFile fname ReadMode let st = State { _file = fh, _yyinput = BS.singleton 0, _yycursor = 0, _yymarker = 0, _token = 0, _yylimit = 0, _eof = False, _count = 0 } result <- lexer st hClose fh -- Cleanup. removeFile fname -- Check result. when (result /= expect) $ error $ "expected " ++ show expect ++ ", got " ++ show result return () ( EOF ) YYFILL . YYFILL (YYLIMIT - YYCURSOR) < n API C YYLESSTHAN(n) API . YYFILL . YYFILL lexer ( ). YYFILL (limit) YYMAXFILL ( YYFILL n ). YYFILL ( YYMAXFILL # ). <-- shift --> <-- need --> >-A------------B---------C-----D-------E---F--------G-> buffer token marker cursor limit >-A------------B---------C-----D-------E---F--------G-> buffer, marker cursor limit token <-- shift --> <-- need --> >-A------------B---------C-----D-------E-F (EOF) buffer token marker cursor limit >-A------------B---------C-----D-------E-F############### buffer, marker cursor limit token <- YYMAXFILL -> input.txt (bounds-checking) . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad import qualified Data.ByteString as BS import GHC.IO.Handle import System.Directory import System.IO chunk_size :: Int chunk_size = 4096 data State = State { _file :: !Handle, _yyinput :: !BS.ByteString, _yycursor :: !Int, _yylimit :: !Int, _token :: !Int, _eof :: !Bool, _count :: !Int } %{ re2c:YYFN = ["lexer;IO Int", "State{..};State;!State{..}"]; re2c:YYPEEK = "BS.index"; // We have to turn off autogenerated YFILL check and write it manually as part of YYFILL // implementation, so that we can propagate the updated state out of it. re2c:yyfill:check = 0; re2c:YYFILL = "State{..} <- fill State{..} @@"; re2c:monadic = 1; str = ['] ([^'\\] | [\\][^])* [']; * { return (-1) } [\x00] { return $ if _yycursor == _yylimit - yymaxfill + 1 then _count else (-1) } str { lexer State{_token = _yycursor, _count = _count + 1, ..} } [ ]+ { lexer State{_token = _yycursor, ..} } %} yymaxfill :: Int %{max %} fill :: State -> Int -> IO State fill !st@State{..} !need = if _yylimit - _yycursor >= need then return st else case _eof of True -> error "fill failed" False -> do -- Discard everything up to the current token, cut off terminating null, -- read new chunk from file and reappend terminating null at the end. chunk <- BS.hGet _file chunk_size let !eof = BS.length chunk < need -- end of file ? let !buf = BS.concat [ BS.drop _token _yyinput, chunk, if eof then (BS.replicate yymaxfill 0) else BS.empty] return State { _yyinput = buf, _yycursor = _yycursor - _token, _yylimit = BS.length buf, _token = 0, _eof = eof, ..} main :: IO () main = do let fname = "input" -- Prepare input file. BS.writeFile fname $ BS.concat ["'qu\0tes' 'are' 'fine: \\'' " | _ <- [1..chunk_size]] let expect = 3 * chunk_size -- the total number of strings in file -- Run lexer on the prepared file. fh <- openFile fname ReadMode let st = State { _file = fh, _yyinput = BS.empty, _yycursor = 0, _token = 0, _yylimit = 0, _eof = False, _count = 0 } result <- lexer st hClose fh -- Cleanup. removeFile fname -- Check result. when (result /= expect) $ error $ "expected " ++ show expect ++ ", got " ++ show result return () (Multiple blocks) (lexer) ( ). re2hs . start conditions . . goto ( C/C++ Go) (label) . goto . goto ( Rust) switch /switch yystate re2hs . (binary) (octal) (decimal) (hexadecimal) . . . . -- re2hs $INPUT -o $OUTPUT -i {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import Data.ByteString (ByteString, index) data State = State { _yyinput :: !ByteString, _yycursor :: !Int, _yymarker :: !Int } peek_digit :: ByteString -> Int -> Int -> Int peek_digit str idx offs = fromIntegral (index str (idx - 1)) - offs %{ re2c:yyfill:enable = 0; %} %{local re2c:YYFN = ["parse_bin;Int", "State{..};State", "num;Int"]; [01] { parse_bin State{..} $ num * 2 + (peek_digit _yyinput _yycursor 48) } * { num } %} %{local re2c:YYFN = ["parse_oct;Int", "State{..};State", "num;Int"]; [0-7] { parse_oct State{..} $ num * 8 + (peek_digit _yyinput _yycursor 48) } * { num } %} %{local re2c:YYFN = ["parse_dec;Int", "State{..};State", "num;Int"]; [0-9] { parse_dec State{..} $ num * 10 + (peek_digit _yyinput _yycursor 48) } * { num } %} %{local re2c:YYFN = ["parse_hex;Int", "State{..};State", "num;Int"]; [0-9] { parse_hex State{..} $ num * 16 + (peek_digit _yyinput _yycursor 48) } [a-f] { parse_hex State{..} $ num * 16 + (peek_digit _yyinput _yycursor 87) } [A-F] { parse_hex State{..} $ num * 16 + (peek_digit _yyinput _yycursor 55) } * { num } %} %{local re2c:YYFN = ["parse;Maybe Int", "State{..};State"]; '0b' / [01] { Just $ parse_bin State{..} 0 } "0" { Just $ parse_oct State{..} 0 } "" / [1-9] { Just $ parse_dec State{..} 0 } '0x' / [0-9a-fA-F] { Just $ parse_hex State{..} 0 } * { Nothing } %} test :: ByteString -> Maybe Int -> IO () test str expect = do let s = State {_yyinput = str, _yycursor = 0, _yymarker = 0} when (parse s /= expect) $ error "failed!" main :: IO () main = do test "\0" Nothing test "1234567890\0" (Just 1234567890) test "0b1101\0" (Just 13) test "0x7Fe\0" (Just 2046) test "0644\0" (Just 420) test "9223372036854775807\0" (Just 9223372036854775807) --start-conditions . re2hs . re2hs . : YYGETCOND YYSETCOND . conditions --header re2c:header . re2c:cond:enumprefix . <*> . : < condition-list > regular-expression code condition-list . regular-expression code . < condition-list > regular-expression => condition code condition-list . regular-expression condition code . < condition-list > regular-expression :=> condition condition-list . regular-expression condition ( ). < condition-list > !action code code action condition-list ( actions ). code code condition-list . !pre_rule ( ). < > code "0" code . !entry ( ). < > => condition code condition . < > :=> condition condition . re2hs re2hs goto/label loop/switch . goto ( C/C++ Go) yyc_ cond ( re2c:cond:prefix ). goto . re2hs switch YYGETSTATE . :=> switch ( re2c:cond:goto ). . goto ( Rust) re2hs yystate . switch switch DFA . switch . ( YYGETCOND/YYSETCOND ). . . cond conditions . -- re2hs $INPUT -o $OUTPUT -ci {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import Data.ByteString (ByteString, index) %{conditions %} data State = State { _yyinput :: !ByteString, _yycursor :: !Int, _yymarker :: !Int, _yycond :: !YYCONDTYPE } peek_digit :: ByteString -> Int -> Int -> Int peek_digit str idx offs = fromIntegral (index str (idx - 1)) - offs %{ re2c:YYFN = ["parse;Maybe Int", "State{..};State", "_num;Int"]; re2c:yyfill:enable = 0; '0b' / [01] :=> bin "0" :=> oct "" / [1-9] :=> dec '0x' / [0-9a-fA-F] :=> hex * { Nothing } [01] { yyfnbin State{..} $ _num * 2 + (peek_digit _yyinput _yycursor 48) } [0-7] { yyfnoct State{..} $ _num * 8 + (peek_digit _yyinput _yycursor 48) } [0-9] { yyfndec State{..} $ _num * 10 + (peek_digit _yyinput _yycursor 48) } [0-9] { yyfnhex State{..} $ _num * 16 + (peek_digit _yyinput _yycursor 48) } [a-f] { yyfnhex State{..} $ _num * 16 + (peek_digit _yyinput _yycursor 87) } [A-F] { yyfnhex State{..} $ _num * 16 + (peek_digit _yyinput _yycursor 55) } * { Just _num } %} test :: ByteString -> Maybe Int -> IO () test str expect = do let s = State { _yyinput = str, _yycursor = 0, _yymarker = 0, _yycond = YYC_init} when (parse s 0 /= expect) $ error "failed!" main :: IO () main = do test "\0" Nothing test "1234567890\0" (Just 1234567890) test "0b1101\0" (Just 13) test "0x7Fe\0" (Just 2046) test "0644\0" (Just 420) test "9223372036854775807\0" (Just 9223372036854775807) --storable-state re2hs . re2hs <<>> ("pull") <<>>. ( ). : <<>> ("push") . . . <<>> : o YYSETSTATE() YYGETSTATE(state). o yych yyaccept ( ) state . state -1 . o YYFILL . . o . o getstate YYGETSTATE . goto ( --loop-switch). o re2c:eof sentinel with bounds checks . (padding) : . ( ). <<>> ("push") . . <<>> . . -- re2hs $INPUT -o $OUTPUT -fi {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Concurrent.Chan import Control.Monad import Data.ByteString as BS import Text.Printf debug :: IO () -> IO () debug = when False data State = State { _pipe :: !(Chan BS.ByteString), _yyinput :: !BS.ByteString, _yycursor :: !Int, _yymarker :: !Int, _yylimit :: !Int, _token :: !Int, _eof :: !Bool, _yystate :: !Int, _recv :: !Int } data Status = End | Ready | Waiting | BadPacket deriving (Eq) %{ re2c:YYFN = ["lexer;IO (State, Status)", "State{..};State;!State{..}"]; re2c:YYPEEK = "BS.index"; re2c:YYFILL = "return (State{..}, Waiting)"; re2c:eof = 0; re2c:monadic = 1; packet = [a-z]+[;]; * { return (State{..}, BadPacket) } $ { return (State{..}, End) } packet { lexer State{_token = _yycursor, _recv = _recv + 1, ..} } %} fill :: State -> IO (State, Status) fill st@State{..} = do case _eof of True -> return (st, End) False -> do -- Discard everything up to the current token, cut off terminating null, -- read new chunk from file and reappend terminating null at the end. chunk <- readChan _pipe return (State { _yyinput = BS.concat [(BS.init . BS.drop _token) _yyinput, chunk, "\0"], _yycursor = _yycursor - _token, _yymarker = _yymarker - _token, _yylimit = _yylimit - _token + BS.length chunk, -- exclude terminating null _token = 0, _eof = BS.null chunk, -- end of file? ..}, Ready) loop :: State -> [BS.ByteString] -> IO Status loop State{..} packets = do (State{..}, status) <- lexer State{..} case status of End -> do debug $ printf "done: got %d packets\n" _recv return End Waiting -> do debug $ printf "waiting...\n" packets' <- case packets of [] -> do writeChan _pipe BS.empty return [] p:ps -> do debug $ printf "sent packet '%s'\n" (show p) writeChan _pipe p return ps (State{..}, status') <- fill State{..} case status' of Ready -> loop State{..} packets' _ -> error "unexpected status after fill" BadPacket -> do debug $ printf "error: ill-formed packet\n" return BadPacket _ -> error "unexpected status" test :: [BS.ByteString] -> Status -> IO () test packets expect = do pipe <- newChan -- emulate pipe using a chan of bytestrings let st = State { _pipe = pipe, _yyinput = BS.singleton 0, -- null sentinel triggers YYFILL _yycursor = 0, _yymarker = 0, _token = 0, _yylimit = 0, _eof = False, _yystate = -1, _recv = 0 } status <- loop st packets when (status /= expect) $ error "failed" return () main :: IO () main = do test [] End test ["ze", "ro;o", "ne", ";t", "wo;thr", "e", "e", ";", "four;"] End test ["zer0;"] BadPacket /*!rules:re2c[:] ... */ %{rules[:] ... %} re2hs . . rules use . rules . use /*!use:re2c[:] ... */ %{use[:] ... %} . : rules () ( ). use . use (lexer) : rules ( ). !use:; re2hs . . : . * $ ( ) . use re2hs ( ). use 2.2 re2hs . ( ). 2.2 -r --reusable . 1.2 . !use -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} -- This example shows how to combine reusable re2c blocks: two blocks -- ('colors' and 'fish') are merged into one. The 'salmon' rule occurs -- in both blocks; the 'fish' block takes priority because it is used -- earlier. Default rule * occurs in all three blocks; the local (not -- inherited) definition takes priority. import Control.Monad (when) import Data.ByteString (ByteString, index) data Answer = Color | Fish | Dunno deriving (Eq) data State = State { _yyinput :: ByteString, _yycursor :: Int, _yymarker :: Int } %{rules:colors * { error "ah" } "red" | "salmon" | "magenta" { Color } %} %{rules:fish * { error "oh" } "haddock" | "salmon" | "eel" { Fish } %} %{ re2c:YYFN = ["lexer;Answer", "State{..};State"]; re2c:yyfill:enable = 0; !use:fish; !use:colors; * { Dunno } // overrides inherited '*' rules %} main :: IO () main = do let test str ans = do let st = State {_yyinput = str, _yycursor = 0, _yymarker = 0} when (lexer st /= ans) $ error "failed" test "salmon" Fish test "what?" Dunno -- re2hs $INPUT -o $OUTPUT --input-encoding utf8 {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} -- This example supports multiple input encodings: UTF-8 and UTF-32. -- Both lexers are generated from the same rules block, and the use -- blocks add only encoding-specific configurations. import Control.Monad (when) import Data.Array import Data.Word data State a = State { _yyinput :: a, _yycursor :: Int, _yymarker :: Int } %{rules re2c:yyfill:enable = 0; re2c:YYPEEK = "(!)"; "x y" { Just _yycursor } * { Nothing } %} %{use re2c:YYFN = ["lex8;Maybe Int", "State{..};State (Array Int Word8)"]; re2c:encoding:utf8 = 1; re2c:YYCTYPE = Word8; %} %{use re2c:YYFN = ["lex32;Maybe Int", "State{..};State (Array Int Word)"]; re2c:encoding:utf32 = 1; re2c:YYCTYPE = Word; %} main :: IO () main = do let make_st l = State { _yyinput = listArray (0, length l - 1) l, _yycursor = 0, _yymarker = 0} let s8 = [0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79] when (lex8 (make_st s8) /= Just (length s8)) $ error "lex8 failed" let s32 = [0x2200, 0x78, 0x20, 0x2203, 0x79] when (lex32 (make_st s32) /= Just (length s32)) $ error "lex32 failed" (Submatch extraction) re2hs . Tags (tags) @stag #mtag stag mtag . -T --tags re2c:tags = 1 . : ( ) . S-tag m-tag ( ). svars mvars. (leftmost greedy) . Captures (capturing groups) . --captures re2c:captures = 1 . --leftmost-captures () --posix-captures POSIX . : (! ... ). --invert-captures re2c:invert-captures = 1 . yynmatch ( ) yypmatch . yynmatch yypmatch yypmatch [yynmatch * 2] . maxnmatch YYMAXNMATCH yynmatch . Captvars --captvars re2c:captvars = 1 . --captures : yynmatch yypmatch re2hs yytl yytr k- ( svars ). : --leftmost-captvars re2c:leftmost-captvars = 1 () --posix-captvars re2c:posix-captvars POSIX. (Tagged Deterministic Finite Automata with Lookahead) . TDFA (overhead) . TDFA DFA . . TDFA (tag variables) : . . stags mtags . . YYFILL . S-tag : o s-tag: t = YYCURSOR API C YYSTAGP(t) API (generic) o s-tag: t = NULL API C YYSTAGN(t) API o s-tag : t1 = t2 M-tag : o m-tag: YYMTAGP(t) API o m-tag: YYMTAGN(t) API o m-tag : t1 = t2 S-tag ( ) . M-tag . m-tag ( ) m-tag (prefix-tree) (v, p) v p . s-tag : major minor patch ( ). YYFILL . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import Data.ByteString (ByteString, index) data State = State { -- Final tag bindings available in semantic action. %{svars format = "\n@@ :: !Int,"; %} -- Intermediate tag bindings used by the lexer (must be autogenerated). %{stags format = "\n@@ :: !Int,"; %} _yyinput :: !ByteString, _yycursor :: !Int, _yymarker :: !Int } data SemVer = SemVer { major :: !Int, minor :: !Int, patch :: !Int } deriving (Eq) s2n :: ByteString -> Int -> Int -> Int s2n s i j = f i 0 where f k n = if k >= j then n else f (k + 1) (n * 10 + (fromIntegral (index s k) - 48)) %{ re2c:YYFN = ["parse;Maybe SemVer", "State{..};State"]; re2c:tags = 1; re2c:yyfill:enable = 0; num = [0-9]+; @_1 num @_2 "." @_3 num @_4 ("." @_5 num)? [\x00] { Just SemVer { major = s2n _yyinput _1 _2, minor = s2n _yyinput _3 _4, patch = if _5 == (-1) then 0 else s2n _yyinput _5 (_yycursor - 1) } } * { Nothing } %} test :: ByteString -> Maybe SemVer -> IO () test str expect = do let s = State { %{svars format = "\n@@ = (-1),"; %} %{stags format = "\n@@ = (-1),"; %} _yyinput = str, _yycursor = 0, _yymarker = 0 } when (parse s /= expect) $ error "failed!" main :: IO () main = do test "23.34\0" (Just SemVer {major = 23, minor = 34, patch = 0}) test "1.2.99999\0" (Just SemVer {major = 1, minor = 2, patch = 99999}) test "1.a\0" Nothing s-tag YYFILL . (lexer) YYFILL . s-tag . ( m-tag ) . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad import Data.ByteString as BS import GHC.IO.Handle import System.Directory import System.IO chunk_size :: Int chunk_size = 4096 data State = State { _file :: !Handle, _yyinput :: !BS.ByteString, _yycursor :: !Int, _yymarker :: !Int, _yylimit :: !Int, _token :: !Int, -- Final tag bindings available in semantic action. %{svars format = "\n@@ :: !Int,"; %} -- Intermediate tag bindings used by the lexer (must be autogenerated). %{stags format = "\n@@ :: !Int,"; %} _eof :: !Bool } data SemVer = SemVer { major :: !Int, minor :: !Int, patch :: !Int } deriving (Eq, Show) s2n :: BS.ByteString -> Int -> Int -> Int s2n s i j = f i 0 where f k n = if k >= j then n else f (k + 1) (n * 10 + (fromIntegral (BS.index s k) - 48)) %{ re2c:YYFN = ["lexer;IO [SemVer]", "State{..};State", "_vers;[SemVer]"]; re2c:YYPEEK = "BS.index"; re2c:YYFILL = "(State{..}, yyfill) <- fill State{..}"; re2c:eof = 0; re2c:monadic = 1; re2c:tags = 1; num = [0-9]+; @_1 num @_2 "." @_3 num @_4 ("." @_5 num)? [\n] { let ver = SemVer { major = s2n _yyinput _1 _2, minor = s2n _yyinput _3 _4, patch = if _5 == (-1) then 0 else s2n _yyinput _5 (_yycursor - 1) } lexer State{..} (ver: _vers) } $ { return _vers } * { error "lexer failed" } %} fill :: State -> IO (State, Bool) fill State{..} = do case _eof of True -> return (State{..}, False) False -> do -- Discard everything up to the current token, cut off terminating null, -- read new chunk from file and reappend terminating null at the end. chunk <- BS.hGet _file chunk_size return (State{ _yyinput = BS.concat [(BS.init . BS.drop _token) _yyinput, chunk, "\0"], _yycursor = _yycursor - _token, _yymarker = _yymarker - _token, _yylimit = _yylimit - _token + BS.length chunk, -- exclude terminating null _token = 0, _eof = BS.null chunk, -- end of file? ..}, True) main :: IO () main = do let fname = "input" -- Prepare input file. BS.writeFile fname $ BS.concat ["1.22.333\n" | _ <- [1..chunk_size]] let expect = [SemVer {major = 1, minor = 22, patch = 333} | _ <- [1..chunk_size]] -- Run lexer on the prepared file. fh <- openFile fname ReadMode let st = State { _file = fh, _yyinput = BS.singleton 0, _yycursor = 0, _yymarker = 0, _yylimit = 0, _token = 0, %{svars format = "\n@@ = (-1),"; %} %{stags format = "\n@@ = (-1),"; %} _eof = False } result <- lexer st [] hClose fh -- Cleanup. removeFile fname -- Check result. when (result /= expect) $ error $ "expected " ++ show expect ++ ", got " ++ show result return () (capturing groups) . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import Data.ByteString (ByteString, index) import Data.Word (Word8) none :: Int none = -1 data State = State { -- Final tag bindings available in semantic action. %{svars format = "\n@@ :: !Int,"; %} -- Intermediate tag bindings used by the lexer (must be autogenerated). %{stags format = "\n@@ :: !Int,"; %} _yyinput :: !ByteString, _yycursor :: !Int, _yymarker :: !Int } data SemVer = SemVer { major :: !Int, minor :: !Int, patch :: !Int } deriving (Eq) s2n :: ByteString -> Int -> Int -> Int s2n s i j = f i 0 where f k n = if k >= j then n else f (k + 1) (n * 10 + (fromIntegral (index s k) - 48)) %{ re2c:YYFN = ["parse;Maybe SemVer", "State{..};State"]; re2c:YYCTYPE = "Word8"; re2c:captvars = 1; re2c:yypmatch = _; re2c:yyfill:enable = 0; num = [0-9]+; (num) "." (num) ("." num)? [\x00] { Just SemVer { major = s2n _yyinput _yytl1 _yytr1, minor = s2n _yyinput _yytl2 _yytr2, patch = if _yytl3 == none then 0 else s2n _yyinput (_yytl3 + 1) _yytr3 } } * { Nothing } %} test :: ByteString -> Maybe SemVer -> IO () test str expect = do let s = State { %{svars format = "\n@@ = none,"; %} %{stags format = "\n@@ = none,"; %} _yyinput = str, _yycursor = 0, _yymarker = 0 } when (parse s /= expect) $ error "failed!" main :: IO () main = do test "23.34\0" (Just SemVer {major = 23, minor = 34, patch = 0}) test "1.2.99999\0" (Just SemVer {major = 1, minor = 2, patch = 99999}) test "1.a\0" Nothing m-tag . (trie) . -- re2hs $INPUT -o $OUTPUT {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import Data.ByteString (ByteString, index) data State = State { -- Final tag bindings available in semantic action. %{svars format = "\n@@ :: !Int,"; %} %{mvars format = "\n@@ :: ![Int],"; %} -- Intermediate tag bindings used by the lexer (must be autogenerated). %{stags format = "\n@@ :: !Int,"; %} %{mtags format = "\n@@ :: ![Int],"; %} _yyinput :: !ByteString, _yycursor :: !Int, _yymarker :: !Int } s2n :: ByteString -> Int -> Int -> Int s2n s i j = f i 0 where f k n = if k >= j then n else f (k + 1) (n * 10 + (fromIntegral (index s k) - 48)) %{ re2c:YYFN = ["parse;Maybe [Int]", "State{..};State"]; re2c:YYMTAGP = "let tag = _yycursor : @@{tag} in let @@{tag} = tag in"; re2c:YYMTAGN = ""; // alternatively could add -1 to the list re2c:tags = 1; re2c:yyfill:enable = 0; num = [0-9]+; @_1 num @_2 ("." #_3 num #_4)* [\x00] { Just $ (s2n _yyinput _1 _2) : (reverse $ zipWith (\i j -> s2n _yyinput i j) _3 _4) } * { Nothing } %} test :: ByteString -> Maybe [Int] -> IO () test str expect = do let st = State { %{svars format = "\n@@ = (-1),"; %} %{stags format = "\n@@ = (-1),"; %} %{mvars format = "\n@@ = [],"; %} %{mtags format = "\n@@ = [],"; %} _yyinput = str, _yycursor = 0, _yymarker = 0 } when (parse st /= expect) $ error "failed!" main :: IO () main = do test "1\0" (Just [1]) test "1.2.3.4.5.6.7\0" (Just [1, 2, 3, 4, 5, 6, 7]) test "1.2.\0" Nothing (Encoding support) code points () code units ( ) . . . . . . "any" [^] ( * ). : yych YYCTYPE . . re2hs . : . : o ASCII ( ). [0-255] . o EBCDIC ( --ebcdic re2c:encoding:ebcdic). [0-255] . o UCS2 ( --ucs2 re2c:encoding:ucs2). [0-0xFFFF] . o UTF8 ( --utf8 re2c:encoding:utf8). . . -- . o UTF16 ( --utf16 re2c:encoding:utf16). . . -- . o UTF32 ( --utf32 re2c:encoding:utf32). [0-0x10FFFF] . include/unicode_blocks.re include/unicode_categories.re include/unicode_properties.re re2hs . --input-encoding . --input-encoding utf8 re2hs UTF8 ( --utf8 ). --encoding-policy re2hs (Unicode surrogates [0xD800-0xDFFF]) . UTF8 . -- re2hs $INPUT -o $OUTPUT --utf8 -i {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import Data.ByteString (ByteString, index) %{include "unicode_categories.re" %} data State = State { _yyinput :: ByteString, _yycursor :: Int, _yymarker :: Int, _yyaccept :: Int } %{ re2c:YYFN = ["lexer;Bool", "State{..};State"]; re2c:yyfill:enable = 0; // Simplified "Unicode Identifier and Pattern Syntax" // (see https://unicode.org/reports/tr31) id_start = L | Nl | [$_]; id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3]; identifier = id_start id_continue*; identifier { True } * { False } %} main :: IO () main = do let st = State { _yyinput = "_\x00", _yycursor = 0, _yymarker = 0, _yyaccept = 0} when (not $ lexer st) $ error "failed" re2hs /*!include:re2c FILE */ %{include FILE %} !include FILE ; FILE . re2hs include include -I . / include re2hs #include C/C++ : FILE / . include include . --depfile include . re2hs include include/ . ( Unicode) re2hs . include . (definitions.hs) data Number = INum | FNum | NNaN deriving (Eq) %{ number = [1-9][0-9]*; %} (extra_rules.re.inc) // floating-point numbers frac = [0-9]* "." [0-9]+ | [0-9]+ "."; exp = 'e' [+-]? [0-9]+; float = frac exp? | [0-9]+ exp; float { FNum } (Input file) -- re2hs $INPUT -o $OUTPUT -i {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import Data.ByteString (ByteString, index) %{include "definitions.hs" %} data State = State { _yyinput :: ByteString, _yycursor :: Int, _yymarker :: Int, _yyaccept :: Int } %{ re2c:YYFN = ["lexer;Number", "State{..};State"]; re2c:yyfill:enable = 0; * { NNaN } number { INum } !include "extra_rules.re.inc"; %} main :: IO () main = do let test s n = do let st = State { _yyinput = s, _yycursor = 0, _yymarker = 0, _yyaccept = 0} when (lexer st /= n) $ error "failed" test "123\0" INum test "123.4567\0" FNum re2hs .re --header re2c:header /*!header:re2c:on*/ /*!header:re2c:off*/ %{header:on%} %{header:off%} . . re2hs --header re2c:header ( stdout ) . re2hs (translation units) . (lexer state) ( ). (Input file) -- re2hs $INPUT -o $OUTPUT --header lexer/state.hs -i {-# OPTIONS_GHC -Wno-unused-record-wildcards #-} {-# LANGUAGE OverloadedStrings #-} import Control.Monad (when) import Data.ByteString (index) import State %{header:on %} module State where import Data.ByteString (ByteString) data State = State { _yyinput :: !ByteString, _yycursor :: !Int, %{stags format = "\n@@{tag} :: !Int,"; %} _tag :: !Int } %{header:off %} %{ re2c:YYFN = ["lexer;Int", "State{..};State"]; re2c:tags = 1; re2c:yyfill:enable = 0; re2c:header = "lexer/state.hs"; [a]* @_tag [b]* { _tag } %} main :: IO () main = do let s = State { _yyinput = "ab\0", _yycursor = 0, %{stags format = "\n@@{tag} = -1,"; %} _tag = 0} when (lexer s /= 1) $ error "failed!" (Header file) -- Generated by re2c {-# LANGUAGE RecordWildCards #-} module State where import Data.ByteString (ByteString) data State = State { _yyinput :: !ByteString, _yycursor :: !Int, _yyt1 :: !Int, _tag :: !Int } -S, --skeleton re2hs re2hs C . . DFA ( ) re2hs : .input DFA .keys . .input . : o re2hs ( DFA ). o (benchmarking) . o ( ). ( ). re2hs DFA . : DFA . ( ASCII UTF-8 EBCDIC) DFA . DFA : re2hs DFA . . re2hs ( ) . ~1Gb (re2hs ). (Visualization and debug) -D, --emit-dot re2hs . DFA DOT . Graphviz DFA . DFA . --dump-nfa --dump-dfa-raw ( ). (SEE ALSO) re2c : http://re2c.org. flex(1) lex(1) quex(http://quex.sourceforge.net). (AUTHORS) re2hs Peter Bumbulis (peter@csg.uwaterloo.ca) . Marcus Boerger Dan Nuffer . : Brian Young (bayoung@acm.org) Marcus Boerger Dan Nuffer (nuffer@users.sourceforge.net) Ulya Trofimovich (skvadrik@gmail.com) Serghei Iakovlev Sergei Trofimovich Petr Skocik ligfx raekye PolarGoose. RE2HS(1)