CIRCT  19.0.0git
FIRLexer.cpp
Go to the documentation of this file.
1 //===- FIRLexer.cpp - .fir file lexer implementation ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements a .fir file lexer.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "FIRLexer.h"
14 #include "mlir/IR/Diagnostics.h"
15 #include "llvm/ADT/StringExtras.h"
16 #include "llvm/ADT/StringSwitch.h"
17 #include "llvm/Support/SourceMgr.h"
18 #include "llvm/Support/raw_ostream.h"
19 
20 using namespace circt;
21 using namespace firrtl;
22 using llvm::SMLoc;
23 using llvm::SMRange;
24 using llvm::SourceMgr;
25 
26 #define isdigit(x) DO_NOT_USE_SLOW_CTYPE_FUNCTIONS
27 #define isalpha(x) DO_NOT_USE_SLOW_CTYPE_FUNCTIONS
28 
29 //===----------------------------------------------------------------------===//
30 // FIRToken
31 //===----------------------------------------------------------------------===//
32 
33 SMLoc FIRToken::getLoc() const {
34  return SMLoc::getFromPointer(spelling.data());
35 }
36 
37 SMLoc FIRToken::getEndLoc() const {
38  return SMLoc::getFromPointer(spelling.data() + spelling.size());
39 }
40 
41 SMRange FIRToken::getLocRange() const { return SMRange(getLoc(), getEndLoc()); }
42 
43 /// Return true if this is one of the keyword token kinds (e.g. kw_wire).
44 bool FIRToken::isKeyword() const {
45  switch (kind) {
46  default:
47  return false;
48 #define TOK_KEYWORD(SPELLING) \
49  case kw_##SPELLING: \
50  return true;
51 #include "FIRTokenKinds.def"
52  }
53 }
54 
55 /// Given a token containing a string literal, return its value, including
56 /// removing the quote characters and unescaping the contents of the string. The
57 /// lexer has already verified that this token is valid.
58 std::string FIRToken::getStringValue() const {
59  assert(getKind() == string);
60  return getStringValue(getSpelling());
61 }
62 
63 std::string FIRToken::getStringValue(StringRef spelling) {
64  // Start by dropping the quotes.
65  StringRef bytes = spelling.drop_front().drop_back();
66 
67  std::string result;
68  result.reserve(bytes.size());
69  for (size_t i = 0, e = bytes.size(); i != e;) {
70  auto c = bytes[i++];
71  if (c != '\\') {
72  result.push_back(c);
73  continue;
74  }
75 
76  assert(i + 1 <= e && "invalid string should be caught by lexer");
77  auto c1 = bytes[i++];
78  switch (c1) {
79  case '\\':
80  case '"':
81  case '\'':
82  result.push_back(c1);
83  continue;
84  case 'b':
85  result.push_back('\b');
86  continue;
87  case 'n':
88  result.push_back('\n');
89  continue;
90  case 't':
91  result.push_back('\t');
92  continue;
93  case 'f':
94  result.push_back('\f');
95  continue;
96  case 'r':
97  result.push_back('\r');
98  continue;
99  // TODO: Handle the rest of the escapes (octal and unicode).
100  default:
101  break;
102  }
103 
104  assert(i + 1 <= e && "invalid string should be caught by lexer");
105  auto c2 = bytes[i++];
106 
107  assert(llvm::isHexDigit(c1) && llvm::isHexDigit(c2) && "invalid escape");
108  result.push_back((llvm::hexDigitValue(c1) << 4) | llvm::hexDigitValue(c2));
109  }
110 
111  return result;
112 }
113 
114 /// Given a token containing a verbatim string, return its value, including
115 /// removing the quote characters and unescaping the quotes of the string. The
116 /// lexer has already verified that this token is valid.
117 std::string FIRToken::getVerbatimStringValue() const {
118  assert(getKind() == verbatim_string);
120 }
121 
122 std::string FIRToken::getVerbatimStringValue(StringRef spelling) {
123  // Start by dropping the quotes.
124  StringRef bytes = spelling.drop_front().drop_back();
125 
126  std::string result;
127  result.reserve(bytes.size());
128  for (size_t i = 0, e = bytes.size(); i != e;) {
129  auto c = bytes[i++];
130  if (c != '\\') {
131  result.push_back(c);
132  continue;
133  }
134 
135  assert(i + 1 <= e && "invalid string should be caught by lexer");
136  auto c1 = bytes[i++];
137  if (c1 != '\'') {
138  result.push_back(c);
139  }
140  result.push_back(c1);
141  }
142 
143  return result;
144 }
145 
146 //===----------------------------------------------------------------------===//
147 // FIRLexer
148 //===----------------------------------------------------------------------===//
149 
150 static StringAttr getMainBufferNameIdentifier(const llvm::SourceMgr &sourceMgr,
151  MLIRContext *context) {
152  auto mainBuffer = sourceMgr.getMemoryBuffer(sourceMgr.getMainFileID());
153  StringRef bufferName = mainBuffer->getBufferIdentifier();
154  if (bufferName.empty())
155  bufferName = "<unknown>";
156  return StringAttr::get(context, bufferName);
157 }
158 
159 FIRLexer::FIRLexer(const llvm::SourceMgr &sourceMgr, MLIRContext *context)
160  : sourceMgr(sourceMgr),
161  bufferNameIdentifier(getMainBufferNameIdentifier(sourceMgr, context)),
162  curBuffer(
163  sourceMgr.getMemoryBuffer(sourceMgr.getMainFileID())->getBuffer()),
164  curPtr(curBuffer.begin()),
165  // Prime the first token.
166  curToken(lexTokenImpl()) {}
167 
168 /// Encode the specified source location information into a Location object
169 /// for attachment to the IR or error reporting.
170 Location FIRLexer::translateLocation(llvm::SMLoc loc) {
171  assert(loc.isValid());
172  unsigned mainFileID = sourceMgr.getMainFileID();
173  auto lineAndColumn = sourceMgr.getLineAndColumn(loc, mainFileID);
174  return FileLineColLoc::get(bufferNameIdentifier, lineAndColumn.first,
175  lineAndColumn.second);
176 }
177 
178 /// Emit an error message and return a FIRToken::error token.
179 FIRToken FIRLexer::emitError(const char *loc, const Twine &message) {
180  mlir::emitError(translateLocation(SMLoc::getFromPointer(loc)), message);
181  return formToken(FIRToken::error, loc);
182 }
183 
184 /// Return the indentation level of the specified token.
185 std::optional<unsigned> FIRLexer::getIndentation(const FIRToken &tok) const {
186  // Count the number of horizontal whitespace characters before the token.
187  auto *bufStart = curBuffer.begin();
188 
189  auto isHorizontalWS = [](char c) -> bool {
190  return c == ' ' || c == '\t' || c == ',';
191  };
192  auto isVerticalWS = [](char c) -> bool {
193  return c == '\n' || c == '\r' || c == '\f' || c == '\v';
194  };
195 
196  unsigned indent = 0;
197  const auto *ptr = (const char *)tok.getSpelling().data();
198  while (ptr != bufStart && isHorizontalWS(ptr[-1]))
199  --ptr, ++indent;
200 
201  // If the character we stopped at isn't the start of line, then return none.
202  if (ptr != bufStart && !isVerticalWS(ptr[-1]))
203  return std::nullopt;
204 
205  return indent;
206 }
207 
208 //===----------------------------------------------------------------------===//
209 // Lexer Implementation Methods
210 //===----------------------------------------------------------------------===//
211 
213  while (true) {
214  const char *tokStart = curPtr;
215  switch (*curPtr++) {
216  default:
217  // Handle identifiers.
218  if (llvm::isAlpha(curPtr[-1]))
219  return lexIdentifierOrKeyword(tokStart);
220 
221  // Unknown character, emit an error.
222  return emitError(tokStart, "unexpected character");
223 
224  case 0:
225  // This may either be a nul character in the source file or may be the EOF
226  // marker that llvm::MemoryBuffer guarantees will be there.
227  if (curPtr - 1 == curBuffer.end())
228  return formToken(FIRToken::eof, tokStart);
229 
230  [[fallthrough]]; // Treat as whitespace.
231 
232  case ' ':
233  case '\t':
234  case '\n':
235  case '\r':
236  case ',':
237  // Handle whitespace.
238  continue;
239 
240  case '`':
241  case '_':
242  // Handle identifiers.
243  return lexIdentifierOrKeyword(tokStart);
244 
245  case '.':
246  return formToken(FIRToken::period, tokStart);
247  case ':':
248  return formToken(FIRToken::colon, tokStart);
249  case '(':
250  return formToken(FIRToken::l_paren, tokStart);
251  case ')':
252  return formToken(FIRToken::r_paren, tokStart);
253  case '{':
254  if (*curPtr == '|')
255  return ++curPtr, formToken(FIRToken::l_brace_bar, tokStart);
256  return formToken(FIRToken::l_brace, tokStart);
257  case '}':
258  return formToken(FIRToken::r_brace, tokStart);
259  case '[':
260  return formToken(FIRToken::l_square, tokStart);
261  case ']':
262  return formToken(FIRToken::r_square, tokStart);
263  case '<':
264  if (*curPtr == '-')
265  return ++curPtr, formToken(FIRToken::less_minus, tokStart);
266  if (*curPtr == '=')
267  return ++curPtr, formToken(FIRToken::less_equal, tokStart);
268  return formToken(FIRToken::less, tokStart);
269  case '>':
270  return formToken(FIRToken::greater, tokStart);
271  case '=':
272  if (*curPtr == '>')
273  return ++curPtr, formToken(FIRToken::equal_greater, tokStart);
274  return formToken(FIRToken::equal, tokStart);
275  case '?':
276  return formToken(FIRToken::question, tokStart);
277  case '@':
278  if (*curPtr == '[')
279  return lexFileInfo(tokStart);
280  // Unknown character, emit an error.
281  return emitError(tokStart, "unexpected character");
282  case '%':
283  if (*curPtr == '[')
284  return lexInlineAnnotation(tokStart);
285  return emitError(tokStart, "unexpected character following '%'");
286  case '|':
287  if (*curPtr == '}')
288  return ++curPtr, formToken(FIRToken::r_brace_bar, tokStart);
289  // Unknown character, emit an error.
290  return emitError(tokStart, "unexpected character");
291 
292  case ';':
293  skipComment();
294  continue;
295 
296  case '"':
297  return lexString(tokStart, /*isVerbatim=*/false);
298  case '\'':
299  return lexString(tokStart, /*isVerbatim=*/true);
300 
301  case '-':
302  case '+':
303  case '0':
304  case '1':
305  case '2':
306  case '3':
307  case '4':
308  case '5':
309  case '6':
310  case '7':
311  case '8':
312  case '9':
313  return lexNumber(tokStart);
314  }
315  }
316 }
317 
318 /// Lex a file info specifier.
319 ///
320 /// FileInfo ::= '@[' ('\]'|.)* ']'
321 ///
322 FIRToken FIRLexer::lexFileInfo(const char *tokStart) {
323  while (1) {
324  switch (*curPtr++) {
325  case ']': // This is the end of the fileinfo literal.
326  return formToken(FIRToken::fileinfo, tokStart);
327  case '\\':
328  // Ignore escaped ']'
329  if (*curPtr == ']')
330  ++curPtr;
331  break;
332  case 0:
333  // This could be the end of file in the middle of the fileinfo. If so
334  // emit an error.
335  if (curPtr - 1 != curBuffer.end())
336  break;
337  [[fallthrough]];
338  case '\n': // Vertical whitespace isn't allowed in a fileinfo.
339  case '\v':
340  case '\f':
341  return emitError(tokStart, "unterminated file info specifier");
342  default:
343  // Skip over other characters.
344  break;
345  }
346  }
347 }
348 
349 /// Lex a non-standard inline Annotation file.
350 ///
351 /// InlineAnnotation ::= '%[' (.)* ']'
352 ///
354  size_t depth = 0;
355  bool stringMode = false;
356  while (1) {
357  switch (*curPtr++) {
358  case '\\':
359  ++curPtr;
360  break;
361  case '"':
362  stringMode = !stringMode;
363  break;
364  case ']':
365  if (stringMode)
366  break;
367  if (depth == 1)
368  return formToken(FIRToken::inlineannotation, tokStart);
369  --depth;
370  break;
371  case '[':
372  if (stringMode)
373  break;
374  ++depth;
375  break;
376  case 0:
377  if (curPtr - 1 != curBuffer.end())
378  break;
379  return emitError(tokStart, "unterminated inline annotation");
380  default:
381  break;
382  }
383  }
384 }
385 
386 /// Lex an identifier or keyword that starts with a letter.
387 ///
388 /// LegalStartChar ::= [a-zA-Z_]
389 /// LegalIdChar ::= LegalStartChar | [0-9] | '$'
390 ///
391 /// Id ::= LegalStartChar (LegalIdChar)*
392 /// LiteralId ::= [a-zA-Z0-9$_]+
393 ///
395  // Remember that this is a literalID
396  bool isLiteralId = *tokStart == '`';
397 
398  // Match the rest of the identifier regex: [0-9a-zA-Z_$-]*
399  while (llvm::isAlpha(*curPtr) || llvm::isDigit(*curPtr) || *curPtr == '_' ||
400  *curPtr == '$' || *curPtr == '-')
401  ++curPtr;
402 
403  // Consume the trailing '`' in a literal identifier.
404  if (isLiteralId) {
405  if (*curPtr != '`')
406  return emitError(tokStart, "unterminated literal identifier");
407  ++curPtr;
408  }
409 
410  StringRef spelling(tokStart, curPtr - tokStart);
411 
412  // Check to see if this is a 'primop', which is an identifier juxtaposed with
413  // a '(' character.
414  if (*curPtr == '(') {
415  FIRToken::Kind kind = llvm::StringSwitch<FIRToken::Kind>(spelling)
416 #define TOK_LPKEYWORD(SPELLING) .Case(#SPELLING, FIRToken::lp_##SPELLING)
417 #include "FIRTokenKinds.def"
418  .Default(FIRToken::identifier);
419  if (kind != FIRToken::identifier) {
420  ++curPtr;
421  return formToken(kind, tokStart);
422  }
423  }
424 
425  // See if the identifier is a keyword. By default, it is an identifier.
426  FIRToken::Kind kind = llvm::StringSwitch<FIRToken::Kind>(spelling)
427 #define TOK_KEYWORD(SPELLING) .Case(#SPELLING, FIRToken::kw_##SPELLING)
428 #include "FIRTokenKinds.def"
429  .Default(FIRToken::identifier);
430 
431  // If this has the backticks of a literal identifier and it fell through the
432  // above switch, indicating that it was not found to e a keyword, then change
433  // its kind from identifier to literal identifier.
434  if (isLiteralId && kind == FIRToken::identifier)
435  kind = FIRToken::literal_identifier;
436 
437  return FIRToken(kind, spelling);
438 }
439 
440 /// Skip a comment line, starting with a ';' and going to end of line.
442  while (true) {
443  switch (*curPtr++) {
444  case '\n':
445  case '\r':
446  // Newline is end of comment.
447  return;
448  case 0:
449  // If this is the end of the buffer, end the comment.
450  if (curPtr - 1 == curBuffer.end()) {
451  --curPtr;
452  return;
453  }
454  [[fallthrough]];
455  default:
456  // Skip over other characters.
457  break;
458  }
459  }
460 }
461 
462 /// StringLit ::= '"' UnquotedString? '"'
463 /// VerbatimStringLit ::= '\'' UnquotedString? '\''
464 /// UnquotedString ::= ( '\\\'' | '\\"' | ~[\r\n] )+?
465 ///
466 FIRToken FIRLexer::lexString(const char *tokStart, bool isVerbatim) {
467  while (1) {
468  switch (*curPtr++) {
469  case '"': // This is the end of the string literal.
470  if (isVerbatim)
471  break;
472  return formToken(FIRToken::string, tokStart);
473  case '\'': // This is the end of the raw string.
474  if (!isVerbatim)
475  break;
476  return formToken(FIRToken::verbatim_string, tokStart);
477  case '\\':
478  // Ignore escaped '\'' or '"'
479  if (*curPtr == '\'' || *curPtr == '"')
480  ++curPtr;
481  else if (*curPtr == 'u' || *curPtr == 'U')
482  return emitError(tokStart, "unicode escape not supported in string");
483  break;
484  case 0:
485  // This could be the end of file in the middle of the string. If so
486  // emit an error.
487  if (curPtr - 1 != curBuffer.end())
488  break;
489  [[fallthrough]];
490  case '\n': // Vertical whitespace isn't allowed in a string.
491  case '\r':
492  case '\v':
493  case '\f':
494  return emitError(tokStart, "unterminated string");
495  default:
496  if (curPtr[-1] & ~0x7F)
497  return emitError(tokStart, "string characters must be 7-bit ASCII");
498  // Skip over other characters.
499  break;
500  }
501  }
502 }
503 
504 /// Lex a number literal.
505 ///
506 /// UnsignedInt ::= '0' | PosInt
507 /// PosInt ::= [1-9] ([0-9])*
508 /// DoubleLit ::=
509 /// ( '+' | '-' )? Digit+ '.' Digit+ ( 'E' ( '+' | '-' )? Digit+ )?
510 /// TripleLit ::=
511 /// Digit+ '.' Digit+ '.' Digit+
512 /// Radix-specified Integer ::=
513 /// ( '-' )? '0' ( 'b' | 'o' | 'd' | 'h' ) LegalDigit*
514 ///
515 FIRToken FIRLexer::lexNumber(const char *tokStart) {
516  assert(llvm::isDigit(curPtr[-1]) || curPtr[-1] == '+' || curPtr[-1] == '-');
517 
518  // There needs to be at least one digit.
519  if (!llvm::isDigit(*curPtr) && !llvm::isDigit(curPtr[-1]))
520  return emitError(tokStart, "unexpected character after sign");
521 
522  // If we encounter a "b", "o", "d", or "h", this is a radix-specified integer
523  // literal. This is only supported for FIRRTL 2.4.0 or later. This is always
524  // lexed, but rejected during parsing if the version is too old.
525  const char *oldPtr = curPtr;
526  if (curPtr[-1] == '-' && *curPtr == '0')
527  ++curPtr;
528  if (curPtr[-1] == '0') {
529  switch (*curPtr) {
530  case 'b':
531  ++curPtr;
532  while (*curPtr >= '0' && *curPtr <= '1')
533  ++curPtr;
534  return formToken(FIRToken::radix_specified_integer, tokStart);
535  case 'o':
536  ++curPtr;
537  while (*curPtr >= '0' && *curPtr <= '7')
538  ++curPtr;
539  return formToken(FIRToken::radix_specified_integer, tokStart);
540  case 'd':
541  ++curPtr;
542  while (llvm::isDigit(*curPtr))
543  ++curPtr;
544  return formToken(FIRToken::radix_specified_integer, tokStart);
545  case 'h':
546  ++curPtr;
547  while (llvm::isHexDigit(*curPtr))
548  ++curPtr;
549  return formToken(FIRToken::radix_specified_integer, tokStart);
550  default:
551  curPtr = oldPtr;
552  break;
553  }
554  }
555 
556  while (llvm::isDigit(*curPtr))
557  ++curPtr;
558 
559  // If we encounter a '.' followed by a digit, then this is a floating point
560  // literal, otherwise this is an integer or negative integer.
561  if (*curPtr != '.' || !llvm::isDigit(curPtr[1])) {
562  if (*tokStart == '-' || *tokStart == '+')
563  return formToken(FIRToken::signed_integer, tokStart);
564  return formToken(FIRToken::integer, tokStart);
565  }
566 
567  // Lex a floating point literal.
568  curPtr += 2;
569  while (llvm::isDigit(*curPtr))
570  ++curPtr;
571 
572  bool hasE = false;
573  if (*curPtr == 'E') {
574  hasE = true;
575  ++curPtr;
576  if (*curPtr == '+' || *curPtr == '-')
577  ++curPtr;
578  while (llvm::isDigit(*curPtr))
579  ++curPtr;
580  }
581 
582  // If we encounter a '.' followed by a digit, again, and there was no
583  // exponent, then this is a version literal. Otherwise it is a floating point
584  // literal.
585  if (*curPtr != '.' || !llvm::isDigit(curPtr[1]) || hasE)
586  return formToken(FIRToken::floatingpoint, tokStart);
587 
588  // Lex a version literal.
589  curPtr += 2;
590  while (llvm::isDigit(*curPtr))
591  ++curPtr;
592  return formToken(FIRToken::version, tokStart);
593 }
assert(baseType &&"element must be base type")
static StringAttr getMainBufferNameIdentifier(const llvm::SourceMgr &sourceMgr, MLIRContext *context)
Definition: FIRLexer.cpp:150
FIRToken lexFileInfo(const char *tokStart)
Lex a file info specifier.
Definition: FIRLexer.cpp:322
FIRToken lexIdentifierOrKeyword(const char *tokStart)
Lex an identifier or keyword that starts with a letter.
Definition: FIRLexer.cpp:394
const llvm::SourceMgr & sourceMgr
Definition: FIRLexer.h:138
FIRToken formToken(FIRToken::Kind kind, const char *tokStart)
Definition: FIRLexer.h:124
FIRToken lexNumber(const char *tokStart)
Lex a number literal.
Definition: FIRLexer.cpp:515
FIRToken lexString(const char *tokStart, bool isVerbatim)
StringLit ::= '"' UnquotedString? '"' VerbatimStringLit ::= '\'' UnquotedString? '\'' UnquotedString ...
Definition: FIRLexer.cpp:466
const char * curPtr
Definition: FIRLexer.h:142
std::optional< unsigned > getIndentation(const FIRToken &tok) const
Return the indentation level of the specified token or None if this token is preceded by another toke...
Definition: FIRLexer.cpp:185
void skipComment()
Skip a comment line, starting with a ';' and going to end of line.
Definition: FIRLexer.cpp:441
FIRToken emitError(const char *loc, const Twine &message)
Emit an error message and return a FIRToken::error token.
Definition: FIRLexer.cpp:179
FIRLexer(const llvm::SourceMgr &sourceMgr, mlir::MLIRContext *context)
mlir::Location translateLocation(llvm::SMLoc loc)
Encode the specified source location information into a Location object for attachment to the IR or e...
Definition: FIRLexer.cpp:170
FIRToken lexInlineAnnotation(const char *tokStart)
Lex a non-standard inline Annotation file.
Definition: FIRLexer.cpp:353
const mlir::StringAttr bufferNameIdentifier
Definition: FIRLexer.h:139
This represents a specific token for .fir files.
Definition: FIRLexer.h:29
std::string getVerbatimStringValue() const
Given a token containing a verbatim string, return its value, including removing the quote characters...
Definition: FIRLexer.cpp:117
StringRef getSpelling() const
Definition: FIRLexer.h:44
StringRef spelling
A reference to the entire token contents; this is always a pointer into a memory buffer owned by the ...
Definition: FIRLexer.h:94
llvm::SMRange getLocRange() const
Definition: FIRLexer.cpp:41
std::string getStringValue() const
Given a token containing a string literal, return its value, including removing the quote characters ...
Definition: FIRLexer.cpp:58
Kind kind
Discriminator that indicates the sort of token this is.
Definition: FIRLexer.h:90
llvm::SMLoc getEndLoc() const
Definition: FIRLexer.cpp:37
Kind getKind() const
Definition: FIRLexer.h:47
llvm::SMLoc getLoc() const
Definition: FIRLexer.cpp:33
bool isKeyword() const
Return true if this is one of the keyword token kinds (e.g. kw_wire).
Definition: FIRLexer.cpp:44
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
Definition: CalyxOps.cpp:53
This file defines an intermediate representation for circuits acting as an abstraction for constraint...
Definition: DebugAnalysis.h:21