CIRCT  19.0.0git
FIRLexer.cpp
Go to the documentation of this file.
1 //===- FIRLexer.cpp - .fir file lexer implementation ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements a .fir file lexer.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "FIRLexer.h"
14 #include "mlir/IR/Diagnostics.h"
15 #include "llvm/ADT/StringExtras.h"
16 #include "llvm/ADT/StringSwitch.h"
17 #include "llvm/Support/SourceMgr.h"
18 #include "llvm/Support/raw_ostream.h"
19 
20 using namespace circt;
21 using namespace firrtl;
22 using llvm::SMLoc;
23 using llvm::SMRange;
24 using llvm::SourceMgr;
25 
26 #define isdigit(x) DO_NOT_USE_SLOW_CTYPE_FUNCTIONS
27 #define isalpha(x) DO_NOT_USE_SLOW_CTYPE_FUNCTIONS
28 
29 //===----------------------------------------------------------------------===//
30 // FIRToken
31 //===----------------------------------------------------------------------===//
32 
33 SMLoc FIRToken::getLoc() const {
34  return SMLoc::getFromPointer(spelling.data());
35 }
36 
37 SMLoc FIRToken::getEndLoc() const {
38  return SMLoc::getFromPointer(spelling.data() + spelling.size());
39 }
40 
41 SMRange FIRToken::getLocRange() const { return SMRange(getLoc(), getEndLoc()); }
42 
43 /// Return true if this is one of the keyword token kinds (e.g. kw_wire).
44 bool FIRToken::isKeyword() const {
45  switch (kind) {
46  default:
47  return false;
48 #define TOK_KEYWORD(SPELLING) \
49  case kw_##SPELLING: \
50  return true;
51 #include "FIRTokenKinds.def"
52  }
53 }
54 
55 /// Given a token containing a string literal, return its value, including
56 /// removing the quote characters and unescaping the contents of the string. The
57 /// lexer has already verified that this token is valid.
58 std::string FIRToken::getStringValue() const {
59  assert(getKind() == string);
60  return getStringValue(getSpelling());
61 }
62 
63 std::string FIRToken::getStringValue(StringRef spelling) {
64  // Start by dropping the quotes.
65  StringRef bytes = spelling.drop_front().drop_back();
66 
67  std::string result;
68  result.reserve(bytes.size());
69  for (size_t i = 0, e = bytes.size(); i != e;) {
70  auto c = bytes[i++];
71  if (c != '\\') {
72  result.push_back(c);
73  continue;
74  }
75 
76  assert(i + 1 <= e && "invalid string should be caught by lexer");
77  auto c1 = bytes[i++];
78  switch (c1) {
79  case '\\':
80  case '"':
81  case '\'':
82  result.push_back(c1);
83  continue;
84  case 'b':
85  result.push_back('\b');
86  continue;
87  case 'n':
88  result.push_back('\n');
89  continue;
90  case 't':
91  result.push_back('\t');
92  continue;
93  case 'f':
94  result.push_back('\f');
95  continue;
96  case 'r':
97  result.push_back('\r');
98  continue;
99  // TODO: Handle the rest of the escapes (octal and unicode).
100  default:
101  break;
102  }
103 
104  assert(i + 1 <= e && "invalid string should be caught by lexer");
105  auto c2 = bytes[i++];
106 
107  assert(llvm::isHexDigit(c1) && llvm::isHexDigit(c2) && "invalid escape");
108  result.push_back((llvm::hexDigitValue(c1) << 4) | llvm::hexDigitValue(c2));
109  }
110 
111  return result;
112 }
113 
114 /// Given a token containing a verbatim string, return its value, including
115 /// removing the quote characters and unescaping the quotes of the string. The
116 /// lexer has already verified that this token is valid.
117 std::string FIRToken::getVerbatimStringValue() const {
118  assert(getKind() == verbatim_string);
120 }
121 
122 std::string FIRToken::getVerbatimStringValue(StringRef spelling) {
123  // Start by dropping the quotes.
124  StringRef bytes = spelling.drop_front().drop_back();
125 
126  std::string result;
127  result.reserve(bytes.size());
128  for (size_t i = 0, e = bytes.size(); i != e;) {
129  auto c = bytes[i++];
130  if (c != '\\') {
131  result.push_back(c);
132  continue;
133  }
134 
135  assert(i + 1 <= e && "invalid string should be caught by lexer");
136  auto c1 = bytes[i++];
137  if (c1 != '\'') {
138  result.push_back(c);
139  }
140  result.push_back(c1);
141  }
142 
143  return result;
144 }
145 
146 //===----------------------------------------------------------------------===//
147 // FIRLexer
148 //===----------------------------------------------------------------------===//
149 
150 static StringAttr getMainBufferNameIdentifier(const llvm::SourceMgr &sourceMgr,
151  MLIRContext *context) {
152  auto mainBuffer = sourceMgr.getMemoryBuffer(sourceMgr.getMainFileID());
153  StringRef bufferName = mainBuffer->getBufferIdentifier();
154  if (bufferName.empty())
155  bufferName = "<unknown>";
156  return StringAttr::get(context, bufferName);
157 }
158 
159 FIRLexer::FIRLexer(const llvm::SourceMgr &sourceMgr, MLIRContext *context)
160  : sourceMgr(sourceMgr),
161  bufferNameIdentifier(getMainBufferNameIdentifier(sourceMgr, context)),
162  curBuffer(
163  sourceMgr.getMemoryBuffer(sourceMgr.getMainFileID())->getBuffer()),
164  curPtr(curBuffer.begin()),
165  // Prime the first token.
166  curToken(lexTokenImpl()) {}
167 
168 /// Encode the specified source location information into a Location object
169 /// for attachment to the IR or error reporting.
170 Location FIRLexer::translateLocation(llvm::SMLoc loc) {
171  assert(loc.isValid());
172  unsigned mainFileID = sourceMgr.getMainFileID();
173  auto lineAndColumn = sourceMgr.getLineAndColumn(loc, mainFileID);
174  return FileLineColLoc::get(bufferNameIdentifier, lineAndColumn.first,
175  lineAndColumn.second);
176 }
177 
178 /// Emit an error message and return a FIRToken::error token.
179 FIRToken FIRLexer::emitError(const char *loc, const Twine &message) {
180  mlir::emitError(translateLocation(SMLoc::getFromPointer(loc)), message);
181  return formToken(FIRToken::error, loc);
182 }
183 
184 /// Return the indentation level of the specified token.
185 std::optional<unsigned> FIRLexer::getIndentation(const FIRToken &tok) const {
186  // Count the number of horizontal whitespace characters before the token.
187  auto *bufStart = curBuffer.begin();
188 
189  auto isHorizontalWS = [](char c) -> bool {
190  return c == ' ' || c == '\t' || c == ',';
191  };
192  auto isVerticalWS = [](char c) -> bool {
193  return c == '\n' || c == '\r' || c == '\f' || c == '\v';
194  };
195 
196  unsigned indent = 0;
197  const auto *ptr = (const char *)tok.getSpelling().data();
198  while (ptr != bufStart && isHorizontalWS(ptr[-1]))
199  --ptr, ++indent;
200 
201  // If the character we stopped at isn't the start of line, then return none.
202  if (ptr != bufStart && !isVerticalWS(ptr[-1]))
203  return std::nullopt;
204 
205  return indent;
206 }
207 
208 //===----------------------------------------------------------------------===//
209 // Lexer Implementation Methods
210 //===----------------------------------------------------------------------===//
211 
213  while (true) {
214  const char *tokStart = curPtr;
215  switch (*curPtr++) {
216  default:
217  // Handle identifiers.
218  if (llvm::isAlpha(curPtr[-1]))
219  return lexIdentifierOrKeyword(tokStart);
220 
221  // Unknown character, emit an error.
222  return emitError(tokStart, "unexpected character");
223 
224  case 0:
225  // This may either be a nul character in the source file or may be the EOF
226  // marker that llvm::MemoryBuffer guarantees will be there.
227  if (curPtr - 1 == curBuffer.end())
228  return formToken(FIRToken::eof, tokStart);
229 
230  [[fallthrough]]; // Treat as whitespace.
231 
232  case ' ':
233  case '\t':
234  case '\n':
235  case '\r':
236  case ',':
237  // Handle whitespace.
238  continue;
239 
240  case '`':
241  case '_':
242  // Handle identifiers.
243  return lexIdentifierOrKeyword(tokStart);
244 
245  case '.':
246  return formToken(FIRToken::period, tokStart);
247  case ':':
248  return formToken(FIRToken::colon, tokStart);
249  case '(':
250  return formToken(FIRToken::l_paren, tokStart);
251  case ')':
252  return formToken(FIRToken::r_paren, tokStart);
253  case '{':
254  if (*curPtr == '|')
255  return ++curPtr, formToken(FIRToken::l_brace_bar, tokStart);
256  return formToken(FIRToken::l_brace, tokStart);
257  case '}':
258  return formToken(FIRToken::r_brace, tokStart);
259  case '[':
260  return formToken(FIRToken::l_square, tokStart);
261  case ']':
262  return formToken(FIRToken::r_square, tokStart);
263  case '<':
264  if (*curPtr == '=')
265  return ++curPtr, formToken(FIRToken::less_equal, tokStart);
266  return formToken(FIRToken::less, tokStart);
267  case '>':
268  return formToken(FIRToken::greater, tokStart);
269  case '=':
270  if (*curPtr == '>')
271  return ++curPtr, formToken(FIRToken::equal_greater, tokStart);
272  return formToken(FIRToken::equal, tokStart);
273  case '?':
274  return formToken(FIRToken::question, tokStart);
275  case '@':
276  if (*curPtr == '[')
277  return lexFileInfo(tokStart);
278  // Unknown character, emit an error.
279  return emitError(tokStart, "unexpected character");
280  case '%':
281  if (*curPtr == '[')
282  return lexInlineAnnotation(tokStart);
283  return emitError(tokStart, "unexpected character following '%'");
284  case '|':
285  if (*curPtr == '}')
286  return ++curPtr, formToken(FIRToken::r_brace_bar, tokStart);
287  // Unknown character, emit an error.
288  return emitError(tokStart, "unexpected character");
289 
290  case ';':
291  skipComment();
292  continue;
293 
294  case '"':
295  return lexString(tokStart, /*isVerbatim=*/false);
296  case '\'':
297  return lexString(tokStart, /*isVerbatim=*/true);
298 
299  case '-':
300  case '+':
301  case '0':
302  case '1':
303  case '2':
304  case '3':
305  case '4':
306  case '5':
307  case '6':
308  case '7':
309  case '8':
310  case '9':
311  return lexNumber(tokStart);
312  }
313  }
314 }
315 
316 /// Lex a file info specifier.
317 ///
318 /// FileInfo ::= '@[' ('\]'|.)* ']'
319 ///
320 FIRToken FIRLexer::lexFileInfo(const char *tokStart) {
321  while (1) {
322  switch (*curPtr++) {
323  case ']': // This is the end of the fileinfo literal.
324  return formToken(FIRToken::fileinfo, tokStart);
325  case '\\':
326  // Ignore escaped ']'
327  if (*curPtr == ']')
328  ++curPtr;
329  break;
330  case 0:
331  // This could be the end of file in the middle of the fileinfo. If so
332  // emit an error.
333  if (curPtr - 1 != curBuffer.end())
334  break;
335  [[fallthrough]];
336  case '\n': // Vertical whitespace isn't allowed in a fileinfo.
337  case '\v':
338  case '\f':
339  return emitError(tokStart, "unterminated file info specifier");
340  default:
341  // Skip over other characters.
342  break;
343  }
344  }
345 }
346 
347 /// Lex a non-standard inline Annotation file.
348 ///
349 /// InlineAnnotation ::= '%[' (.)* ']'
350 ///
352  size_t depth = 0;
353  bool stringMode = false;
354  while (1) {
355  switch (*curPtr++) {
356  case '\\':
357  ++curPtr;
358  break;
359  case '"':
360  stringMode = !stringMode;
361  break;
362  case ']':
363  if (stringMode)
364  break;
365  if (depth == 1)
366  return formToken(FIRToken::inlineannotation, tokStart);
367  --depth;
368  break;
369  case '[':
370  if (stringMode)
371  break;
372  ++depth;
373  break;
374  case 0:
375  if (curPtr - 1 != curBuffer.end())
376  break;
377  return emitError(tokStart, "unterminated inline annotation");
378  default:
379  break;
380  }
381  }
382 }
383 
384 /// Lex an identifier or keyword that starts with a letter.
385 ///
386 /// LegalStartChar ::= [a-zA-Z_]
387 /// LegalIdChar ::= LegalStartChar | [0-9] | '$'
388 ///
389 /// Id ::= LegalStartChar (LegalIdChar)*
390 /// LiteralId ::= [a-zA-Z0-9$_]+
391 ///
393  // Remember that this is a literalID
394  bool isLiteralId = *tokStart == '`';
395 
396  // Match the rest of the identifier regex: [0-9a-zA-Z_$-]*
397  while (llvm::isAlpha(*curPtr) || llvm::isDigit(*curPtr) || *curPtr == '_' ||
398  *curPtr == '$' || *curPtr == '-')
399  ++curPtr;
400 
401  // Consume the trailing '`' in a literal identifier.
402  if (isLiteralId) {
403  if (*curPtr != '`')
404  return emitError(tokStart, "unterminated literal identifier");
405  ++curPtr;
406  }
407 
408  StringRef spelling(tokStart, curPtr - tokStart);
409 
410  // Check to see if this is a 'primop', which is an identifier juxtaposed with
411  // a '(' character.
412  if (*curPtr == '(') {
413  FIRToken::Kind kind = llvm::StringSwitch<FIRToken::Kind>(spelling)
414 #define TOK_LPKEYWORD(SPELLING) .Case(#SPELLING, FIRToken::lp_##SPELLING)
415 #include "FIRTokenKinds.def"
416  .Default(FIRToken::identifier);
417  if (kind != FIRToken::identifier) {
418  ++curPtr;
419  return formToken(kind, tokStart);
420  }
421  }
422 
423  // See if the identifier is a keyword. By default, it is an identifier.
424  FIRToken::Kind kind = llvm::StringSwitch<FIRToken::Kind>(spelling)
425 #define TOK_KEYWORD(SPELLING) .Case(#SPELLING, FIRToken::kw_##SPELLING)
426 #include "FIRTokenKinds.def"
427  .Default(FIRToken::identifier);
428 
429  // If this has the backticks of a literal identifier and it fell through the
430  // above switch, indicating that it was not found to e a keyword, then change
431  // its kind from identifier to literal identifier.
432  if (isLiteralId && kind == FIRToken::identifier)
433  kind = FIRToken::literal_identifier;
434 
435  return FIRToken(kind, spelling);
436 }
437 
438 /// Skip a comment line, starting with a ';' and going to end of line.
440  while (true) {
441  switch (*curPtr++) {
442  case '\n':
443  case '\r':
444  // Newline is end of comment.
445  return;
446  case 0:
447  // If this is the end of the buffer, end the comment.
448  if (curPtr - 1 == curBuffer.end()) {
449  --curPtr;
450  return;
451  }
452  [[fallthrough]];
453  default:
454  // Skip over other characters.
455  break;
456  }
457  }
458 }
459 
460 /// StringLit ::= '"' UnquotedString? '"'
461 /// VerbatimStringLit ::= '\'' UnquotedString? '\''
462 /// UnquotedString ::= ( '\\\'' | '\\"' | ~[\r\n] )+?
463 ///
464 FIRToken FIRLexer::lexString(const char *tokStart, bool isVerbatim) {
465  while (1) {
466  switch (*curPtr++) {
467  case '"': // This is the end of the string literal.
468  if (isVerbatim)
469  break;
470  return formToken(FIRToken::string, tokStart);
471  case '\'': // This is the end of the raw string.
472  if (!isVerbatim)
473  break;
474  return formToken(FIRToken::verbatim_string, tokStart);
475  case '\\':
476  // Ignore escaped '\'' or '"'
477  if (*curPtr == '\'' || *curPtr == '"' || *curPtr == '\\')
478  ++curPtr;
479  else if (*curPtr == 'u' || *curPtr == 'U')
480  return emitError(tokStart, "unicode escape not supported in string");
481  break;
482  case 0:
483  // This could be the end of file in the middle of the string. If so
484  // emit an error.
485  if (curPtr - 1 != curBuffer.end())
486  break;
487  [[fallthrough]];
488  case '\n': // Vertical whitespace isn't allowed in a string.
489  case '\r':
490  case '\v':
491  case '\f':
492  return emitError(tokStart, "unterminated string");
493  default:
494  if (curPtr[-1] & ~0x7F)
495  return emitError(tokStart, "string characters must be 7-bit ASCII");
496  // Skip over other characters.
497  break;
498  }
499  }
500 }
501 
502 /// Lex a number literal.
503 ///
504 /// UnsignedInt ::= '0' | PosInt
505 /// PosInt ::= [1-9] ([0-9])*
506 /// DoubleLit ::=
507 /// ( '+' | '-' )? Digit+ '.' Digit+ ( 'E' ( '+' | '-' )? Digit+ )?
508 /// TripleLit ::=
509 /// Digit+ '.' Digit+ '.' Digit+
510 /// Radix-specified Integer ::=
511 /// ( '-' )? '0' ( 'b' | 'o' | 'd' | 'h' ) LegalDigit*
512 ///
513 FIRToken FIRLexer::lexNumber(const char *tokStart) {
514  assert(llvm::isDigit(curPtr[-1]) || curPtr[-1] == '+' || curPtr[-1] == '-');
515 
516  // There needs to be at least one digit.
517  if (!llvm::isDigit(*curPtr) && !llvm::isDigit(curPtr[-1]))
518  return emitError(tokStart, "unexpected character after sign");
519 
520  // If we encounter a "b", "o", "d", or "h", this is a radix-specified integer
521  // literal. This is only supported for FIRRTL 2.4.0 or later. This is always
522  // lexed, but rejected during parsing if the version is too old.
523  const char *oldPtr = curPtr;
524  if (curPtr[-1] == '-' && *curPtr == '0')
525  ++curPtr;
526  if (curPtr[-1] == '0') {
527  switch (*curPtr) {
528  case 'b':
529  ++curPtr;
530  while (*curPtr >= '0' && *curPtr <= '1')
531  ++curPtr;
532  return formToken(FIRToken::radix_specified_integer, tokStart);
533  case 'o':
534  ++curPtr;
535  while (*curPtr >= '0' && *curPtr <= '7')
536  ++curPtr;
537  return formToken(FIRToken::radix_specified_integer, tokStart);
538  case 'd':
539  ++curPtr;
540  while (llvm::isDigit(*curPtr))
541  ++curPtr;
542  return formToken(FIRToken::radix_specified_integer, tokStart);
543  case 'h':
544  ++curPtr;
545  while (llvm::isHexDigit(*curPtr))
546  ++curPtr;
547  return formToken(FIRToken::radix_specified_integer, tokStart);
548  default:
549  curPtr = oldPtr;
550  break;
551  }
552  }
553 
554  while (llvm::isDigit(*curPtr))
555  ++curPtr;
556 
557  // If we encounter a '.' followed by a digit, then this is a floating point
558  // literal, otherwise this is an integer or negative integer.
559  if (*curPtr != '.' || !llvm::isDigit(curPtr[1])) {
560  if (*tokStart == '-' || *tokStart == '+')
561  return formToken(FIRToken::signed_integer, tokStart);
562  return formToken(FIRToken::integer, tokStart);
563  }
564 
565  // Lex a floating point literal.
566  curPtr += 2;
567  while (llvm::isDigit(*curPtr))
568  ++curPtr;
569 
570  bool hasE = false;
571  if (*curPtr == 'E') {
572  hasE = true;
573  ++curPtr;
574  if (*curPtr == '+' || *curPtr == '-')
575  ++curPtr;
576  while (llvm::isDigit(*curPtr))
577  ++curPtr;
578  }
579 
580  // If we encounter a '.' followed by a digit, again, and there was no
581  // exponent, then this is a version literal. Otherwise it is a floating point
582  // literal.
583  if (*curPtr != '.' || !llvm::isDigit(curPtr[1]) || hasE)
584  return formToken(FIRToken::floatingpoint, tokStart);
585 
586  // Lex a version literal.
587  curPtr += 2;
588  while (llvm::isDigit(*curPtr))
589  ++curPtr;
590  return formToken(FIRToken::version, tokStart);
591 }
assert(baseType &&"element must be base type")
static StringAttr getMainBufferNameIdentifier(const llvm::SourceMgr &sourceMgr, MLIRContext *context)
Definition: FIRLexer.cpp:150
FIRToken lexFileInfo(const char *tokStart)
Lex a file info specifier.
Definition: FIRLexer.cpp:320
FIRToken lexIdentifierOrKeyword(const char *tokStart)
Lex an identifier or keyword that starts with a letter.
Definition: FIRLexer.cpp:392
const llvm::SourceMgr & sourceMgr
Definition: FIRLexer.h:138
FIRToken formToken(FIRToken::Kind kind, const char *tokStart)
Definition: FIRLexer.h:124
FIRToken lexNumber(const char *tokStart)
Lex a number literal.
Definition: FIRLexer.cpp:513
FIRToken lexString(const char *tokStart, bool isVerbatim)
StringLit ::= '"' UnquotedString? '"' VerbatimStringLit ::= '\'' UnquotedString? '\'' UnquotedString ...
Definition: FIRLexer.cpp:464
const char * curPtr
Definition: FIRLexer.h:142
std::optional< unsigned > getIndentation(const FIRToken &tok) const
Return the indentation level of the specified token or None if this token is preceded by another toke...
Definition: FIRLexer.cpp:185
void skipComment()
Skip a comment line, starting with a ';' and going to end of line.
Definition: FIRLexer.cpp:439
FIRToken emitError(const char *loc, const Twine &message)
Emit an error message and return a FIRToken::error token.
Definition: FIRLexer.cpp:179
FIRLexer(const llvm::SourceMgr &sourceMgr, mlir::MLIRContext *context)
mlir::Location translateLocation(llvm::SMLoc loc)
Encode the specified source location information into a Location object for attachment to the IR or e...
Definition: FIRLexer.cpp:170
FIRToken lexInlineAnnotation(const char *tokStart)
Lex a non-standard inline Annotation file.
Definition: FIRLexer.cpp:351
const mlir::StringAttr bufferNameIdentifier
Definition: FIRLexer.h:139
This represents a specific token for .fir files.
Definition: FIRLexer.h:29
std::string getVerbatimStringValue() const
Given a token containing a verbatim string, return its value, including removing the quote characters...
Definition: FIRLexer.cpp:117
StringRef getSpelling() const
Definition: FIRLexer.h:44
StringRef spelling
A reference to the entire token contents; this is always a pointer into a memory buffer owned by the ...
Definition: FIRLexer.h:94
llvm::SMRange getLocRange() const
Definition: FIRLexer.cpp:41
std::string getStringValue() const
Given a token containing a string literal, return its value, including removing the quote characters ...
Definition: FIRLexer.cpp:58
Kind kind
Discriminator that indicates the sort of token this is.
Definition: FIRLexer.h:90
llvm::SMLoc getEndLoc() const
Definition: FIRLexer.cpp:37
Kind getKind() const
Definition: FIRLexer.h:47
llvm::SMLoc getLoc() const
Definition: FIRLexer.cpp:33
bool isKeyword() const
Return true if this is one of the keyword token kinds (e.g. kw_wire).
Definition: FIRLexer.cpp:44
Direction get(bool isOutput)
Returns an output direction if isOutput is true, otherwise returns an input direction.
Definition: CalyxOps.cpp:54
The InstanceGraph op interface, see InstanceGraphInterface.td for more details.
Definition: DebugAnalysis.h:21