// re2c $INPUT -o $OUTPUT 
#include <assert.h>
// expect a null-terminated string
static int lex(const char *YYCURSOR)
{
    int count = 0;
loop:
    /*!re2c
    re2c:define:YYCTYPE = char;
    re2c:yyfill:enable = 0;
    *      { return -1; }
    [\x00] { return count; }
    [a-z]+ { ++count; goto loop; }
    [ ]+   { goto loop; }
    */
}
int main()
{
    assert(lex("") == 0);
    assert(lex("one two three") == 3);
    assert(lex("f0ur") == -1);
    return 0;
}

// re2c $INPUT -o $OUTPUT 
#include <assert.h>
// expect a null-terminated string
static int lex(const char *str, unsigned int len)
{
    const char *YYCURSOR = str, *YYLIMIT = str + len, *YYMARKER;
    int count = 0;
loop:
    /*!re2c
    re2c:define:YYCTYPE = char;
    re2c:yyfill:enable = 0;
    re2c:eof = 0;
    *                           { return -1; }
    $                           { return count; }
    ['] ([^'\\] | [\\][^])* ['] { ++count; goto loop; }
    [ ]+                        { goto loop; }
    */
}
#define TEST(s, r) assert(lex(s, sizeof(s) - 1) == r)
int main()
{
    TEST("", 0);
    TEST("'qu\0tes' 'are' 'fine: \\'' ", 3);
    TEST("'unterminated\\'", -1);
    return 0;
}

// re2c $INPUT -o $OUTPUT 
#include <assert.h>
#include <stdlib.h>
#include <string.h>
/*!max:re2c*/
// expect YYMAXFILL-padded string
static int lex(const char *str, unsigned int len)
{
    const char *YYCURSOR = str, *YYLIMIT = str + len + YYMAXFILL;
    int count = 0;
loop:
    /*!re2c
    re2c:api:style = free-form;
    re2c:define:YYCTYPE = char;
    re2c:define:YYFILL = "return -1;";
    *                           { return -1; }
    [\x00]                      { return YYCURSOR + YYMAXFILL - 1 == YYLIMIT ? count : -1; }
    ['] ([^'\\] | [\\][^])* ['] { ++count; goto loop; }
    [ ]+                        { goto loop; }
    */
}
// make a copy of the string with YYMAXFILL zeroes at the end
static void test(const char *str, unsigned int len, int res)
{
    char *s = (char*) malloc(len + YYMAXFILL);
    memcpy(s, str, len);
    memset(s + len, 0, YYMAXFILL);
    int r = lex(s, len);
    free(s);
    assert(r == res);
}
#define TEST(s, r) test(s, sizeof(s) - 1, r)
int main()
{
    TEST("", 0);
    TEST("'qu\0tes' 'are' 'fine: \\'' ", 3);
    TEST("'unterminated\\'", -1);
    return 0;
}

// re2c $INPUT -o $OUTPUT 
#include <assert.h>
#include <stdlib.h>
#include <string.h>
// expect a string without terminating null
static int lex(const char *str, unsigned int len)
{
    const char *cur = str, *lim = str + len, *mar;
    int count = 0;
loop:
    /*!re2c
    re2c:yyfill:enable = 0;
    re2c:eof = 0;
    re2c:flags:input = custom;
    re2c:api:style = free-form;
    re2c:define:YYCTYPE    = char;
    re2c:define:YYLESSTHAN = "cur >= lim";
    re2c:define:YYPEEK     = "cur < lim ? *cur : 0";  // fake null
    re2c:define:YYSKIP     = "++cur;";
    re2c:define:YYBACKUP   = "mar = cur;";
    re2c:define:YYRESTORE  = "cur = mar;";
    *                           { return -1; }
    $                           { return count; }
    ['] ([^'\\] | [\\][^])* ['] { ++count; goto loop; }
    [ ]+                        { goto loop; }
    */
}
// make a copy of the string without terminating null
static void test(const char *str, unsigned int len, int res)
{
    char *s = (char*) malloc(len);
    memcpy(s, str, len);
    int r = lex(s, len);
    free(s);
    assert(r == res);
}
#define TEST(s, r) test(s, sizeof(s) - 1, r)
int main()
{
    TEST("", 0);
    TEST("'qu\0tes' 'are' 'fine: \\'' ", 3);
    TEST("'unterminated\\'", -1);
    return 0;
}

               <-- shift -->
             >-A------------B---------C-------------D#-----------E->
             buffer       token    marker         limit,
                                                  cursor
>-A------------B---------C-------------D------------E#->
             buffer,  marker        cursor        limit
             token
               <-- shift -->
             >-A------------B---------C-------------D#--E (EOF)
             buffer       token    marker         limit,
                                                  cursor
>-A------------B---------C-------------D---E#........
             buffer,  marker       cursor limit
             token

// re2c $INPUT -o $OUTPUT 
#include <assert.h>
#include <stdio.h>
#include <string.h>
#define SIZE 4096
typedef struct {
    FILE *file;
    char buf[SIZE + 1], *lim, *cur, *mar, *tok;
    int eof;
} Input;
static int fill(Input *in)
{
    if (in->eof) {
        return 1;
    }
    const size_t free = in->tok - in->buf;
    if (free < 1) {
        return 2;
    }
    memmove(in->buf, in->tok, in->lim - in->tok);
    in->lim -= free;
    in->cur -= free;
    in->mar -= free;
    in->tok -= free;
    in->lim += fread(in->lim, 1, free, in->file);
    in->lim[0] = 0;
    in->eof |= in->lim < in->buf + SIZE;
    return 0;
}
static void init(Input *in, FILE *file)
{
    in->file = file;
    in->cur = in->mar = in->tok = in->lim = in->buf + SIZE;
    in->eof = 0;
    fill(in);
}
static int lex(Input *in)
{
    int count = 0;
loop:
    in->tok = in->cur;
    /*!re2c
    re2c:eof = 0;
    re2c:api:style = free-form;
    re2c:define:YYCTYPE  = char;
    re2c:define:YYCURSOR = in->cur;
    re2c:define:YYMARKER = in->mar;
    re2c:define:YYLIMIT  = in->lim;
    re2c:define:YYFILL   = "fill(in) == 0";
    *                           { return -1; }
    $                           { return count; }
    ['] ([^'\\] | [\\][^])* ['] { ++count; goto loop; }
    [ ]+                        { goto loop; }
    */
}
int main()
{
    const char *fname = "input";
    const char str[] = "'qu\0tes' 'are' 'fine: \\'' ";
    FILE *f;
    Input in;
    // prepare input file: a few times the size of the buffer,
    // containing strings with zeroes and escaped quotes
    f = fopen(fname, "w");
    for (int i = 0; i < SIZE; ++i) {
        fwrite(str, 1, sizeof(str) - 1, f);
    }
    fclose(f);
    f = fopen(fname, "r");
    init(&in, f);
    assert(lex(&in) == SIZE * 3);
    fclose(f);
    remove(fname);
    return 0;
}

               <-- shift -->                 <-- need -->
             >-A------------B---------C-----D-------E---F--------G->
             buffer       token    marker cursor  limit
>-A------------B---------C-----D-------E---F--------G->
             buffer,  marker cursor               limit
             token
               <-- shift -->                 <-- need -->
             >-A------------B---------C-----D-------E-F        (EOF)
             buffer       token    marker cursor  limit
>-A------------B---------C-----D-------E-F###############
             buffer,  marker cursor                   limit
             token                        <- YYMAXFILL ->

// re2c $INPUT -o $OUTPUT 
#include <assert.h>
#include <stdio.h>
#include <string.h>
/*!max:re2c*/
#define SIZE 4096
typedef struct {
    FILE *file;
    char buf[SIZE + YYMAXFILL], *lim, *cur, *mar, *tok;
    int eof;
} Input;
static int fill(Input *in, size_t need)
{
    if (in->eof) {
        return 1;
    }
    const size_t free = in->tok - in->buf;
    if (free < need) {
        return 2;
    }
    memmove(in->buf, in->tok, in->lim - in->tok);
    in->lim -= free;
    in->cur -= free;
    in->mar -= free;
    in->tok -= free;
    in->lim += fread(in->lim, 1, free, in->file);
    if (in->lim < in->buf + SIZE) {
        in->eof = 1;
        memset(in->lim, 0, YYMAXFILL);
        in->lim += YYMAXFILL;
    }
    return 0;
}
static void init(Input *in, FILE *file)
{
    in->file = file;
    in->cur = in->mar = in->tok = in->lim = in->buf + SIZE;
    in->eof = 0;
    fill(in, 1);
}
static int lex(Input *in)
{
    int count = 0;
loop:
    in->tok = in->cur;
    /*!re2c
    re2c:api:style = free-form;
    re2c:define:YYCTYPE  = char;
    re2c:define:YYCURSOR = in->cur;
    re2c:define:YYMARKER = in->mar;
    re2c:define:YYLIMIT  = in->lim;
    re2c:define:YYFILL   = "if (fill(in, @@) != 0) return -1;";
    *                           { return -1; }
    [\x00]                      { return (in->lim - in->cur == YYMAXFILL - 1) ? count : -1; }
    ['] ([^'\\] | [\\][^])* ['] { ++count; goto loop; }
    [ ]+                        { goto loop; }
    */
}
int main()
{
    const char *fname = "input";
    const char str[] = "'qu\0tes' 'are' 'fine: \\'' ";
    FILE *f;
    Input in;
    // prepare input file: a few times the size of the buffer,
    // containing strings with zeroes and escaped quotes
    f = fopen(fname, "w");
    for (int i = 0; i < SIZE; ++i) {
        fwrite(str, 1, sizeof(str) - 1, f);
    }
    fclose(f);
    f = fopen(fname, "r");
    init(&in, f);
    assert(lex(&in) == SIZE * 3);
    fclose(f);
    remove(fname);
    return 0;
}

typedef enum { OK, FAIL } Result;
/*!re2c
    number = [1-9][0-9]*;
*/

// floating-point numbers
frac  = [0-9]* "." [0-9]+ | [0-9]+ ".";
exp   = 'e' [+-]? [0-9]+;
float = frac exp? | [0-9]+ exp;
float { return OK; }

// re2c $INPUT -o $OUTPUT -i
#include <assert.h>
/*!include:re2c "definitions.h" */
Result lex(const char *YYCURSOR)
{
    const char *YYMARKER;
    /*!re2c
    re2c:define:YYCTYPE = char;
    re2c:yyfill:enable = 0;
    number { return OK; }
    !include "extra_rules.re.inc";
    * { return FAIL; }
    */
}
int main()
{
    assert(lex("123") == OK);
    assert(lex("123.4567") == OK);
    return 0;
}

// re2c $INPUT -o $OUTPUT -i --type-header src/lexer/lexer.h
#include <assert.h>
#include "src/lexer/lexer.h" // generated by re2c
/*!header:re2c:on*/
typedef struct {
    const char *str, *cur, *mar;
    /*!stags:re2c format = "const char *@@{tag}; "; */
} LexerState;
/*!header:re2c:off*/
int lex(LexerState *st)
{
    /*!re2c
    re2c:flags:type-header = "src/lexer/lexer.h";
    re2c:yyfill:enable = 0;
    re2c:flags:tags = 1;
    re2c:define:YYCTYPE  = char;
    re2c:define:YYCURSOR = "st->cur";
    re2c:define:YYMARKER = "st->mar";
    re2c:tags:expression = "st->@@{tag}";
    [x]{1,4} / [x]{3,5} { return 0; } // ambiguous trailing context
    *                   { return 1; }
    */
}
int main()
{
    LexerState st;
    st.str = st.cur = "xxxxxxxx";
    assert(lex(&st) == 0 && st.cur - st.str == 4);
    return 0;
}

/* Generated by re2c */
typedef struct {
    const char *str, *cur, *mar;
    const char *yyt1; const char *yyt2; const char *yyt3; 
} LexerState;

// re2c $INPUT -o $OUTPUT -f
#include <assert.h>
#include <stdio.h>
#include <string.h>
#define DEBUG    0
#define LOG(...) if (DEBUG) fprintf(stderr, __VA_ARGS__);
#define BUFSIZE  10
typedef struct {
    FILE *file;
    char buf[BUFSIZE + 1], *lim, *cur, *mar, *tok;
    unsigned yyaccept;
    int state;
} Input;
static void init(Input *in, FILE *f)
{
    in->file = f;
    in->cur = in->mar = in->tok = in->lim = in->buf + BUFSIZE;
    in->lim[0] = 0; // append sentinel symbol
    in->yyaccept = 0;
    in->state = -1;
}
typedef enum {END, READY, WAITING, BAD_PACKET, BIG_PACKET} Status;
static Status fill(Input *in)
{
    const size_t shift = in->tok - in->buf;
    const size_t free = BUFSIZE - (in->lim - in->tok);
    if (free < 1) return BIG_PACKET;
    memmove(in->buf, in->tok, BUFSIZE - shift);
    in->lim -= shift;
    in->cur -= shift;
    in->mar -= shift;
    in->tok -= shift;
    const size_t read = fread(in->lim, 1, free, in->file);
    in->lim += read;
    in->lim[0] = 0; // append sentinel symbol
    return READY;
}
static Status lex(Input *in, unsigned int *recv)
{
    char yych;
    /*!getstate:re2c*/
loop:
    in->tok = in->cur;
    /*!re2c
        re2c:eof = 0;
        re2c:api:style = free-form;
        re2c:define:YYCTYPE    = "char";
        re2c:define:YYCURSOR   = "in->cur";
        re2c:define:YYMARKER   = "in->mar";
        re2c:define:YYLIMIT    = "in->lim";
        re2c:define:YYGETSTATE = "in->state";
        re2c:define:YYSETSTATE = "in->state = @@;";
        re2c:define:YYFILL     = "return WAITING;";
        packet = [a-z]+[;];
        *      { return BAD_PACKET; }
        $      { return END; }
        packet { *recv = *recv + 1; goto loop; }
    */
}
void test(const char **packets, Status status)
{
    const char *fname = "pipe";
    FILE *fw = fopen(fname, "w");
    FILE *fr = fopen(fname, "r");
    setvbuf(fw, NULL, _IONBF, 0);
    setvbuf(fr, NULL, _IONBF, 0);
    Input in;
    init(&in, fr);
    Status st;
    unsigned int send = 0, recv = 0;
    for (;;) {
        st = lex(&in, &recv);
        if (st == END) {
            LOG("done: got %u packets\n", recv);
            break;
        } else if (st == WAITING) {
            LOG("waiting...\n");
            if (*packets) {
                LOG("sent packet %u\n", send);
                fprintf(fw, "%s", *packets++);
                ++send;
            }
            st = fill(&in);
            LOG("queue: '%s'\n", in.buf);
            if (st == BIG_PACKET) {
                LOG("error: packet too big\n");
                break;
            }
            assert(st == READY);
        } else {
            assert(st == BAD_PACKET);
            LOG("error: ill-formed packet\n");
            break;
        }
    }
    LOG("\n");
    assert(st == status);
    if (st == END) assert(recv == send);
    fclose(fw);
    fclose(fr);
    remove(fname);
}
int main()
{
    const char *packets1[] = {0};
    const char *packets2[] = {"zero;", "one;", "two;", "three;", "four;", 0};
    const char *packets3[] = {"zer0;", 0};
    const char *packets4[] = {"goooooooooogle;", 0};
    test(packets1, END);
    test(packets2, END);
    test(packets3, BAD_PACKET);
    test(packets4, BIG_PACKET);
    return 0;
}

// re2c $INPUT -o $OUTPUT
// This example shows how to combine reusable re2c blocks: two blocks
// ('colors' and 'fish') are merged into one. The 'salmon' rule occurs
// in both blocks; the 'fish' block takes priority because it is used
// earlier. Default rule * occurs in all three blocks; the local (not
// inherited) definition takes priority.
#include <assert.h>
enum What { COLOR, FISH, DUNNO };
/*!rules:re2c:colors
    *                            { assert(false); }
    "red" | "salmon" | "magenta" { return COLOR; }
*/
/*!rules:re2c:fish
    *                            { assert(false); }
    "haddock" | "salmon" | "eel" { return FISH; }
*/
static What lex(const char *YYCURSOR)
{
    const char *YYMARKER;
    /*!re2c
    re2c:yyfill:enable = 0;
    re2c:define:YYCTYPE = char;
    !use:fish;
    !use:colors;
    * { return DUNNO; }
    */
}
int main()
{
    assert(lex("salmon") == FISH);
    assert(lex("what?") == DUNNO);
    return 0;
}

// re2c $INPUT -o $OUTPUT --input-encoding utf8
// This example supports multiple input encodings: UTF-8 and UTF-32.
// Both lexers are generated from the same rules block, and the use
// blocks add only encoding-specific configurations.
#include <assert.h>
#include <stdint.h>
/*!rules:re2c
    re2c:yyfill:enable = 0;
    "∀x ∃y" { return 0; }
    *       { return 1; }
*/
static int lex_utf8(const uint8_t *YYCURSOR)
{
    const uint8_t *YYMARKER;
    /*!use:re2c
    re2c:define:YYCTYPE = uint8_t;
    re2c:flags:8 = 1;
    */
}
static int lex_utf32(const uint32_t *YYCURSOR)
{
    const uint32_t *YYMARKER;
    /*!use:re2c
    re2c:define:YYCTYPE = uint32_t;
    re2c:flags:8 = 0;
    re2c:flags:u = 1;
    */
}
int main()
{
    static const uint8_t s8[] = // UTF-8
        { 0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79 };
    static const uint32_t s32[] = // UTF32
        { 0x00002200, 0x00000078, 0x00000020, 0x00002203, 0x00000079 };
    assert(lex_utf8(s8) == 0);
    assert(lex_utf32(s32) == 0);
    return 0;
}

// re2c $INPUT -o $OUTPUT -8 --case-ranges -i
//
// Simplified "Unicode Identifier and Pattern Syntax"
// (see https://unicode.org/reports/tr31)
#include <assert.h>
#include <stdint.h>
/*!include:re2c "unicode_categories.re" */
static int lex(const char *YYCURSOR)
{
    const char *YYMARKER;
    /*!re2c
    re2c:define:YYCTYPE = 'unsigned char';
    re2c:yyfill:enable  = 0;
    id_start    = L | Nl | [$_];
    id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3];
    identifier  = id_start id_continue*;
    identifier { return 0; }
    *          { return 1; }
    */
}
int main()
{
    assert(lex("_Ыдентификатор") == 0);
    return 0;
}

// re2c $INPUT -o $OUTPUT -ci
#include <stdint.h>
#include <limits.h>
#include <assert.h>
static const uint64_t ERROR = ~0lu;
/*!types:re2c*/
template<int BASE> static void adddgt(uint64_t &u, unsigned int d)
{
    u = u * BASE + d;
    if (u > UINT32_MAX) u = ERROR;
}
static uint64_t parse_u32(const char *s)
{
    const char *YYMARKER;
    int c = yycinit;
    uint64_t u = 0;
    /*!re2c
    re2c:yyfill:enable = 0;
    re2c:api:style = free-form;
    re2c:define:YYCTYPE = char;
    re2c:define:YYCURSOR = s;
    re2c:define:YYGETCONDITION = "c";
    re2c:define:YYSETCONDITION = "c = @@;";
    <*> * { return ERROR; }
    <init> '0b' / [01]        :=> bin
    <init> "0"                :=> oct
    <init> "" / [1-9]         :=> dec
    <init> '0x' / [0-9a-fA-F] :=> hex
    <bin, oct, dec, hex> "\x00" { return u; }
    <bin> [01]  { adddgt<2> (u, s[-1] - '0');      goto yyc_bin; }
    <oct> [0-7] { adddgt<8> (u, s[-1] - '0');      goto yyc_oct; }
    <dec> [0-9] { adddgt<10>(u, s[-1] - '0');      goto yyc_dec; }
    <hex> [0-9] { adddgt<16>(u, s[-1] - '0');      goto yyc_hex; }
    <hex> [a-f] { adddgt<16>(u, s[-1] - 'a' + 10); goto yyc_hex; }
    <hex> [A-F] { adddgt<16>(u, s[-1] - 'A' + 10); goto yyc_hex; }
    */
}
int main()
{
    assert(parse_u32("1234567890") == 1234567890);
    assert(parse_u32("0b1101") == 13);
    assert(parse_u32("0x7Fe") == 2046);
    assert(parse_u32("0644") == 420);
    assert(parse_u32("9999999999") == ERROR);
    assert(parse_u32("") == ERROR);
    return 0;
}