66 {
67 static const std::pair<TokenType,const char*> regexpTokenPairs[] =
68 {
69 {t_decimalInt,"[+\\-]?\\d+(_\\d+)*"},
70 {t_decimalFloat,"[+\\-]?\\d+(_\\d+)*\\.(\\d+(_\\d+)*)*([eE][+\\-]?\\d+(_\\d+)*)?"},
71 {t_decimalFloat,"[+\\-]?\\d+(_\\d+)*[eE][+\\-]?\\d+(_\\d+)*"},
72
73 {t_hexInt,"[+\\-]?0[xX][\\da-fA-F]+(_[\\da-fA-F]+)*"},
74 {t_hexFloat,"[+\\-]?0[xX][\\da-fA-F]+(_[\\da-fA-F]+)*\\.([\\da-fA-F]+(_[\\da-fA-F]+)*)*([pP][+\\-]?\\d+(_\\d+)*)?"},
75 {t_hexFloat,"[+\\-]?0[xX][\\da-fA-F]+(_[\\da-fA-F]+)*[pP][+\\-]?\\d+(_\\d+)*"},
76
77 {t_floatNaN,"[+\\-]?nan(:0[xX][\\da-fA-F]+(_[\\da-fA-F]+)*)?"},
78 {t_floatInf,"[+\\-]?inf"},
79
80 {t_string,"\"([^\"\n\\\\]*(\\\\([^0-9a-fA-Fu]|[0-9a-fA-F][0-9a-fA-F]|u\\{[0-9a-fA-F]+})))*\""},
81
82 {t_name,"\\$[a-zA-Z0-9\'_+*/~=<>!?@#$%&|:`.\\-\\^\\\\]+"},
83 };
84
85 static const std::tuple<TokenType,const char*,bool> literalTokenTuples[] =
86 {
87 std::make_tuple(t_leftParenthesis,"(",true),
88 std::make_tuple(t_rightParenthesis,")",true),
89 std::make_tuple(t_equals,"=",true),
90
91 #define VISIT_TOKEN(name,description) std::make_tuple(t_##name,#name,false),
93 #undef VISIT_TOKEN
94
95 #undef VISIT_OPERATOR_TOKEN
96 #define VISIT_OPERATOR_TOKEN(_,name,nameString,...) std::make_tuple(t_##name,nameString,false),
98 #undef VISIT_OPERATOR_TOKEN
99 };
100
102
104
105 for(auto regexpTokenPair : regexpTokenPairs)
106 {
108 finalState = createTokenSeparatorPeekState(nfaBuilder,finalState);
110 }
111
112 for(auto literalTokenTuple : literalTokenTuples)
113 {
114 const TokenType tokenType = std::get<0>(literalTokenTuple);
115 const char* literalString = std::get<1>(literalTokenTuple);
116 const bool isTokenSeparator = std::get<2>(literalTokenTuple);
117
119 if(!isTokenSeparator) { finalState = createTokenSeparatorPeekState(nfaBuilder,finalState); }
120
121 addLiteralToNFA(literalString,nfaBuilder,0,finalState);
122 }
123
124 #ifndef _DEBUG
125 if(false)
126 #endif
127 {
128 std::ofstream debugGraphStream("nfaGraph.dot");
130 debugGraphStream.close();
131 }
132
134
135 #ifndef _DEBUG
136 if(false)
137 #endif
138 {
139 std::ofstream debugGraphStream("dfaGraph.dot");
141 debugGraphStream.close();
142 }
143
145 }
#define VISIT_OPERATOR_TOKEN(_, name, nameString,...)
#define ENUM_LITERAL_TOKENS()
#define ENUM_OPERATORS(visitOp)
std::string dumpNFAGraphViz(const Builder *builder)
@ maximumTerminalStateIndex
Builder * createBuilder()
void addToNFA(const char *regexpString, NFA::Builder *nfaBuilder, NFA::StateIndex initialState, NFA::StateIndex finalState)
void logTimer(const char *context, Timer &timer)
std::string dumpDFAGraphViz() const