verilog_lexer_unittest.cc

// Copyright 2017-2020 The Verible Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Unit tests for VerilogLexer (from verilog.lex)
#include "verilog/parser/verilog_lexer.h"

#include <initializer_list>
#include <utility>
#include <vector>

#include "absl/strings/string_view.h"
#include "common/lexer/lexer_test_util.h"
#include "common/text/token_info.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "verilog/parser/verilog_token_enum.h"

namespace verilog {
namespace {

using testing::ElementsAre;
using verible::TokenInfo;

// Removes non-essential tokens from token output stream, such as spaces.
class FilteredVerilogLexer : public VerilogLexer {
 public:
  explicit FilteredVerilogLexer(absl::string_view code) : VerilogLexer(code) {}

  const TokenInfo& DoNextToken() final {
    do {
      VerilogLexer::DoNextToken();
    } while (!VerilogLexer::KeepSyntaxTreeTokens(GetLastToken()));
    return GetLastToken();
  }
};

// Some tests work on filtered tokens, others check unfiltered tokens.
// TODO(b/139743437): Upgrade GenericTestDataSequence to LexerTestData.
using verible::GenericTestDataSequence;
using verible::SimpleTestData;
using LexerTestData = verible::SynthesizedLexerTestData;

// Forwarding function to the template test driver function.
template <typename... Args>
static void TestLexer(Args&&... args) {
  verible::TestLexer<VerilogLexer>(std::forward<Args>(args)...);
}

// Forwarding function to the template test driver function.
template <typename... Args>
static void TestFilteredLexer(Args&&... args) {
  verible::TestLexer<FilteredVerilogLexer>(std::forward<Args>(args)...);
}

static std::initializer_list<LexerTestData> kCommentTests = {
    {{TK_COMMENT_BLOCK, "/**/"}},
    {{TK_COMMENT_BLOCK, "/***/"}},
    {{TK_COMMENT_BLOCK, "/*  */"}},
    {{TK_COMMENT_BLOCK, "/* foo */"}},
    {{TK_COMMENT_BLOCK, "/* foo\nbar */"}},
    {{TK_EOL_COMMENT, "//"}, {TK_NEWLINE, "\n"}},
    {{TK_EOL_COMMENT, "//"}, {TK_NEWLINE, "\r"}},
    {{TK_EOL_COMMENT, "// foo"}, {TK_NEWLINE, "\n"}},
    {{TK_EOL_COMMENT, "// bar"}, {TK_NEWLINE, "\r"}},
    {{TK_EOL_COMMENT, "//"}},     // missing \n, but treat as if it were there
    {{TK_EOL_COMMENT, "//foo"}},  // missing \n, but treat as if it were there
};

static std::initializer_list<LexerTestData> kNewlineTests = {
    {{TK_NEWLINE, "\n"}, {TK_NEWLINE, "\n"}},
    {{TK_NEWLINE, "\r"}, {TK_NEWLINE, "\r"}},
    {{TK_NEWLINE, "\r\n"}, {TK_NEWLINE, "\r\n"}},
    {{TK_NEWLINE, "\r"}, {TK_NEWLINE, "\r\n"}, {TK_NEWLINE, "\n"}},  // mix
};

// treating attributes lists as C-style comments,
// except they are not returned as comment blocks.
static std::initializer_list<SimpleTestData> kAttributeTests = {
    {"(**)"},
    {"(*     *)"},
    {"(* **  *)"},
    {"(***)"},
    {"(** **)"},
    {"(*\n*)"},
    {"(* style=flat *)"},
    {"(*foo=bar*)"},
    {"(*foo=bar()*)"},
    {"(*foo=\"bar()\"*)"},
    {"(*foo=\"bar(abc)\"*)"},
    {"(*foo='{\"bar_.*\"} *)"},
    {"(* style=flat, fill=empty *)"},
};
static std::initializer_list<LexerTestData> kAttributeSequenceTests = {
    {{TK_ATTRIBUTE, "(**)"}, {TK_ATTRIBUTE, "(**)"}},
    {{TK_ATTRIBUTE, "(* style=flat,\nfill=empty *)"}, {TK_NEWLINE, "\n"}},
    {{TK_ATTRIBUTE, "(*garbage\nGARBAGE*)"}, {TK_NEWLINE, "\n"}},
    {{TK_ATTRIBUTE, "(*\ntoken\nsoup\n*)"}, {TK_NEWLINE, "\n"}},
};

static std::initializer_list<LexerTestData> kMacroCallTests = {
    {{MacroCallId, "`FOO"}, '(', ')'},
    {{MacroCallId, "`FOO1"}, '(', ')', ';'},
    {{MacroCallId, "`FOO2"}, '(', ')'},
    {{MacroCallId, "`FOO3"}, '(', {MacroCallCloseToEndLine, ")"}, "\n"},
    {{MacroCallId, "`FOO4"}, '(', {MacroCallCloseToEndLine, ")"}, "\n", ';'},
    {{MacroCallId, "`FOO5"}, '(', {MacroCallCloseToEndLine, ")"}, "  ", "\n"},
    {{MacroCallId, "`FOO5b"}, '(', {MacroCallCloseToEndLine, ")"}, "  \n"},
    {{MacroCallId, "`FOO6"}, '(', ')', ';'},
    {{MacroCallId, "`FOO7"},
     '(',
     {MacroArg, "BAR"},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO8"}, '(', {MacroArg, "BAR"}, ')'},
    {{MacroCallId, "`FOO9"},
     {TK_SPACE, "  "},
     '(',
     {MacroArg, "BAR"},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO10"}, "  ", '(', {MacroArg, "BAR"}, ')'},
    {{MacroCallId, "`FOO11"},
     "  ",
     '(',
     {MacroArg, "\"bar\""},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO11b"},
     "  ",
     '(',
     {MacroArg, "`\"bar`\""},  // preprocess-evaluated string literal
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO11c"},
     "  ",
     '(',
     {MacroArg, "`\"bar()`\""},  // preprocess-evaluated string literal
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO12"},
     '(',
     {MacroArg, "BAR"},
     ',',
     {MacroArg, "\"bar\""},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO13"},
     " ",
     '(',
     {MacroArg, "`BAR"},
     ',',
     {MacroArg, "\"bar\""},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO14"},
     " ",
     '(',
     {MacroArg, "`BAR"},
     ',',
     " ",
     {MacroArg, "\"bar\""},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO15"},
     '(',
     {MacroArg, "`BAR"},
     ',',
     " ",
     {MacroArg, "\"bar\""},
     ')'},
    // macro arg string literals containing balance characters:
    {{MacroCallId, "`FOO16"},
     '(',
     {MacroArg, "`BAR"},
     ',',
     " ",
     {MacroArg, "\"bar([{\""},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    // macro arg string literals containing balance characters:
    {{MacroCallId, "`FOO17"},
     '(',
     {MacroArg, "`BAR"},
     ',',
     " ",
     {MacroArg, "\"bar)]}\""},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO18"},
     '(',
     {MacroArg, "{1,2,3}"},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO19"},
     '(',
     {MacroArg, "BAR()"},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO20"},
     '(',
     {MacroArg, "`BAR()"},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO21"}, '(', {MacroArg, "`BAR()"}, ')'},
    {{MacroCallId, "`FOO22"},
     '(',
     {MacroArg, "{1,2,3}"},
     ',',
     {TK_SPACE, " "},
     {MacroArg, "`BAR()"},  // macro arg without trailing spaces
     {TK_SPACE, " "},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO22b"},
     '(',
     {MacroArg, "bar"},
     {TK_SPACE, " "},
     ',',
     {MacroArg, "none"},
     {TK_SPACE, " "},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO23"}, " ", '(', {MacroArg, "{{{}}((){})}"}, ')'},
    {{MacroCallId, "`FOO24"}, '(', {MacroArg, "{{{}}((){})}"}, ')'},
    {{MacroCallId, "`FOO25"},
     '(',
     {MacroArg, "{{{}}((){})}"},
     {MacroCallCloseToEndLine, ")"},
     "\n"},
    {{MacroCallId, "`FOO26"}, " ", '(', {TK_SPACE, "  "}, ')'},
    {{MacroCallId, "`FOO27"}, '(', {TK_NEWLINE, "\n"}, ')'},
    {{MacroCallId, "`FOO28"}, " ", '(', {TK_SPACE, "\t"}, ')'},
    {{MacroCallId, "`FOO29"}, '(', {TK_SPACE, " "}, ',', {TK_SPACE, " "}, ')'},
    {{MacroCallId, "`FOO30"}, '(', ',', ')'},
    {{MacroCallId, "`FOO30b"}, '(', ',', {TK_SPACE, "   "}, ')'},
    {{MacroCallId, "`FOO31"}, '(', ',', ',', ',', ')'},
    {{MacroCallId, "`FOO32"},
     '(',
     " ",
     ',',
     "\n",
     ',',
     "\n\n",
     ',',
     "   ",
     ')'},
    {{MacroCallId, "`FOO33"},
     '(',
     "  ",
     {MacroArg, "abc"},
     "  ",  // trailing spaces separated from preceding macro argument
     ',',
     " ",
     ')'},
    {{MacroCallId, "`FOO34"},
     '(',
     "  ",
     ',',
     "  ",
     {MacroArg, "def"},
     "  ",
     ')'},
    {{MacroCallId, "`FOO35"},
     '(',
     "  ",
     ',',
     "  ",
     {MacroArg, "def"},
     "  ",
     ',',
     " ",
     ')'},
    {{MacroCallId, "`FOO36"}, '(', ',', {MacroArg, "def"}, ',', ')'},
    {{MacroIdItem, "`uvm_func_new"}, {TK_NEWLINE, "\n"}},
    {{MacroIdentifier, "`uvm_func_new"}, ';', {TK_NEWLINE, "\n"}},
    {{MacroIdItem, "`uvm_func_new"}, {TK_SPACE, "      "}, "\n"},
    {{MacroIdItem, "`uvm_func_new"},
     "   ",
     {TK_EOL_COMMENT, "// comment"},
     "\n"},
    {{MacroIdItem, "`uvm_func_new"},
     "   ",
     {TK_COMMENT_BLOCK, "/* comment */"},
     "\n"},
    {{MacroIdItem, "`uvm_func_new"},
     "   ",
     {TK_COMMENT_BLOCK, "/* comment */"},
     "  \n"},
    {{MacroIdItem, "`uvm_func_new"},
     {TK_COMMENT_BLOCK, "/*C*/"},
     {TK_COMMENT_BLOCK, "/*D*/"},
     "  ",
     {TK_EOL_COMMENT, "//E"},
     "\n"},
    {{MacroIdItem, "`uvm_object_utils_end"}, {TK_NEWLINE, "\n"}},
    {{MacroIdItem, "`uvm_func_new"},
     {TK_NEWLINE, "\n"},
     {MacroIdItem, "`uvm_func_tew"},
     {TK_NEWLINE, "\n"}},
    {{MacroIdentifier, "`uvm_blah"},
     ';',
     " ",
     {MacroIdItem, "`uvm_blah"},
     "\n"},
    {{MacroIdentifier, "`macro"},
     " ",
     '+',
     " ",
     {MacroIdentifier, "`schmacro"},
     ';'},
    {{MacroIdentifier, "`macro"}, '+', {MacroIdentifier, "`schmacro"}, ';'},
    {{SymbolIdentifier, "lvalue"},
     " ",
     '=',
     " ",
     {MacroIdItem, "`schmacro"},
     "\n"},
    {{SymbolIdentifier, "lvalue"}, '=', {MacroIdItem, "`schmacro"}, "\n"},

    // with escaped identifiers in call args
    {{MacroCallId, "`FOO"}, '(', {MacroArg, "\\BAR"}, " ", ')'},
    {{MacroCallId, "`FOO"}, '(', {MacroArg, "\\BAR"}, "\t", ')'},
    {{MacroCallId, "`FOO"}, '(', {MacroArg, "\\BAR"}, "\n", ')'},

    // Comments are lexed together with macro arguments
    {{MacroCallId, "`FOO"},
     '(',
     "\n",
     {MacroArg, "/*comment*/ bar"},
     "\n",
     ')'},
    {{MacroCallId, "`FOO"},
     '(',
     "\n",
     {MacroArg, "bar /*comment*/"},
     "\n",
     ')'},
    {{MacroCallId, "`FOO"},
     '(',
     "\n",
     {MacroArg, "/*\nmulti\nline\n*/ bar"},
     "\n",
     ')'},
    {{MacroCallId, "`FOO"},
     '(',
     "\n",
     {MacroArg, "//comment\n bar"},
     "\n",
     ')'},
    {{MacroCallId, "`FOO"},
     '(',
     "\n",
     {MacroArg, "//multi\n//line\n//comment\n bar"},
     "\n",
     ')'},
    {
        // fuzzer test case
        {MacroCallId, "`g"},
        '(',
        {MacroArg, "(\\x\" `g(::\"\n)"},
        ','  // unterminated call, unexpected EOF here
    },
};

static std::initializer_list<LexerTestData> kUnfilteredMacroCallTests = {
    {
        {MacroIdItem, "`uvm_func_new"},
        {TK_SPACE, "   "},
        {TK_EOL_COMMENT, "// comment"},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroIdItem, "`uvm_func_new"},
        {TK_SPACE, "   "},
        {TK_COMMENT_BLOCK, "/* comment */"},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroIdItem, "`uvm_func_new"},
        {TK_SPACE, "   "},
        {TK_COMMENT_BLOCK, "/* comment */"},
        {TK_SPACE, "  "},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroIdItem, "`uvm_func_new"},
        {TK_COMMENT_BLOCK, "/*C*/"},
        {TK_COMMENT_BLOCK, "/*D*/"},
        {TK_SPACE, "  "},
        {TK_EOL_COMMENT, "//E"},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroCallId, "`FOO1"},
        '(',
        {TK_SPACE, "  "},
        {MacroCallCloseToEndLine, ")"},
        {TK_SPACE, "  "},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroCallId, "`FOO2"},
        '(',
        {TK_SPACE, "  "},
        {MacroCallCloseToEndLine, ")"},
        {TK_SPACE, "  "},
        {TK_COMMENT_BLOCK, "/* comment */"},
        {TK_SPACE, " "},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroCallId, "`FOO3"},
        '(',
        {MacroCallCloseToEndLine, ")"},
        {TK_SPACE, "  "},
        {TK_EOL_COMMENT, "// comment"},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroCallId, "`FOO4"},
        '(',
        {MacroCallCloseToEndLine, ")"},
        {TK_SPACE, " "},
        {TK_COMMENT_BLOCK, "/* comment */"},
        {TK_SPACE, " "},
        {TK_EOL_COMMENT, "// comment"},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroCallId, "`FOO5"},
        '(',
        {MacroCallCloseToEndLine, ")"},
        {TK_SPACE, " "},
        {TK_COMMENT_BLOCK, "/* comment */"},
        {TK_SPACE, " "},
        {TK_COMMENT_BLOCK, "/* comment */"},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroCallId, "`FOO6"},
        '(',
        {MacroArg, "a"},
        {MacroCallCloseToEndLine, ")"},
        {TK_SPACE, " "},
        {TK_EOL_COMMENT, "// comment"},
        {TK_NEWLINE, "\n"},
    },
    {
        {MacroCallId, "`FOO7"},
        '(',
        {MacroArg, "a"},
        ',',
        {MacroArg, "b"},
        {MacroCallCloseToEndLine, ")"},
        {TK_SPACE, " "},
        {TK_EOL_COMMENT, "// comment"},
        {TK_NEWLINE, "\n"},
    },
};

static std::initializer_list<LexerTestData> kMacroDefineTests = {
    {{PP_define, "`define"},
     {TK_SPACE, " "},
     {PP_Identifier, "FOO"},
     {PP_define_body, ""},
     {TK_NEWLINE, "\n"}},
    {{PP_define, "`define"},
     {TK_SPACE, "    "},
     {PP_Identifier, "FOO"},
     {PP_define_body, ""},
     {TK_NEWLINE, "\n"}},
    {{PP_define, "`define"},
     {TK_SPACE, "    "},
     {PP_Identifier, "FOO"},
     "   ",
     {PP_define_body, ""},
     {TK_NEWLINE, "\n"}},
    {{PP_define, "`define"},
     {TK_SPACE, "\t"},
     {PP_Identifier, "FOO"},
     {PP_define_body, ""},
     "\n"},
    {{PP_define, "`define"},
     {TK_SPACE, "\t\t"},
     {PP_Identifier, "FOO"},
     '(',
     ')',
     " ",
     {PP_define_body, "foo()"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     "  ",
     {PP_define_body, "12"},
     "\n"},
    {{PP_define, "`define"},
     "\t",
     {PP_Identifier, "FOO"},
     "\t",
     {PP_define_body, "13"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     " ",
     {PP_define_body, "\\\n14"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     {PP_define_body, "\\\n"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     {PP_define_body, "\\\n15"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     ')',
     "  ",
     {PP_define_body, "bar()"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     ')',
     "  ",
     {PP_define_body, "bar(x);"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     " ",
     {PP_define_body, "\"hello\\world\""},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     ')',
     " ",
     {PP_define_body, "\"hello\\world\""},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "y"},
     ')',
     " ",
     {PP_define_body, "\"hello\\world\"\\\n\"goodbye.\""},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     ',',
     " ",
     {PP_Identifier, "y"},
     ')',
     "  ",
     {PP_define_body, "$bar(x, y);"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     ',',
     " ",
     {PP_Identifier, "y"},
     ')',
     "   ",
     {PP_define_body, "\\\n$bar(x, y);"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     ',',
     " ",
     {PP_Identifier, "y"},
     ',',
     " ",
     {PP_Identifier, "z"},
     ')',
     "   ",
     {PP_define_body, "\\\n$bar(x, y);\\\nzoo();"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "y"},
     ',',
     " ",
     {PP_Identifier, "z"},
     ')',
     "   ",
     {PP_define_body, "\\\nalways look on\\\n\tthe bright side of life"},
     "\n"},
    // 2 definitions:
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     "  ",
     {PP_define_body, "12"},
     "\n",
     {PP_define, "`define"},
     " ",
     {PP_Identifier, "ZOO"},
     '(',
     {PP_Identifier, "a"},
     ',',
     " ",
     {PP_Identifier, "b"},
     ')',
     "  ",
     {PP_define_body, "b - a"},
     "\n"},
    // definition with condition definitions inside body:
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "F"},
     '(',
     {PP_Identifier, "z"},
     ')',
     {PP_define_body,
      "\\\n"
      "`ifdef Z \\\n"
      "`define X z\\\n"
      "`else \\\n"
      "`define X 2 \\\n"
      "`endif"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "GOOD"},
     " ",
     {PP_define_body, "EVIL"},
     "\n",
     {TK_static, "static"},
     " ",
     {TK_void, "void"},
     " ",
     {SymbolIdentifier, "tokens"},
     "\n"},
    // macro parameters with defaults:
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     '=',
     {PP_default_text, "88"},
     ',',
     " ",
     {PP_Identifier, "y"},
     '=',
     {PP_default_text, "\"abc\""},
     ')',
     " ",
     {PP_define_body, "\\\n$bar(y, x);"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     '=',
     {PP_default_text, "\"88\""},
     ',',
     " ",
     {PP_Identifier, "y"},
     '=',
     {PP_default_text, "abc"},
     ')',
     " ",
     {PP_define_body, "\\\n$bar(y, x);"},
     "\n"},
    // macro parameters with defaults, padded with spaces
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     " ",
     '=',
     {TK_SPACE, " "},  // leading space
     {PP_default_text, "88"},
     {TK_SPACE, " "},  // trailing space
     ',',
     " ",
     {PP_Identifier, "y"},
     " ",
     '=',
     {TK_SPACE, " "},  // leading space
     {PP_default_text, "\"abc\""},
     {TK_SPACE, " "},  // trailing space
     ')',
     " ",
     {PP_define_body, "\\\n$bar(y, x);"},
     "\n"},
    // testing parenthesis balancing in PP_default_text
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     '=',
     {PP_default_text, "(a,(b,c))"},
     ',',
     " ",
     {PP_Identifier, "y"},
     " ",
     '=',
     " ",  // leading space
     {PP_default_text, "\"non)},{(sense\""},
     " ",  // trailing space
     ')',
     " ",
     {PP_define_body, "\\\nbar(y, x);"},
     "\n"},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "FOO"},
     '(',
     {PP_Identifier, "x"},
     '=',
     {PP_default_text, "\"\""},
     ',',
     " ",
     {PP_Identifier, "y"},
     " ",
     '=',
     {PP_default_text, "(a,\"bc\")"},
     ')',
     " ",
     {PP_define_body, "\\\nbar(y, x);"},
     "\n"},
    // no line following continuation:
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "UNFINISHED"},
     " ",
     {PP_define_body, "blah + \\"},
     "\n"},
    // multiple blank newlines after empty definition body
    {{PP_define, "`define"},
     "    ",
     {PP_Identifier, "BAR"},
     {PP_define_body, ""},
     "\n\n\n"},
    // conditional define
    {{PP_ifndef, "`ifndef"},
     "  ",
     {PP_Identifier, "FOO"},
     "\n",
     {PP_define, "`define"},
     "    ",
     {PP_Identifier, "BAR"},
     {PP_define_body, ""},
     "\n\n",
     {PP_endif, "`endif"},
     "\n"},
    {
        {PP_define, "`define"}, " ", {PP_Identifier, "UNFINISHED"},
        // No \n before EOF
    },
    {
        {PP_define, "`define"},
        " ",
        {PP_Identifier, "UNFINISHED"},
        " ",
        {PP_define_body, "symphony"},
        // No \n before EOF
    },
    {
        {PP_define, "`define"},
        " ",
        {PP_Identifier, "UNFINISHED"},
        " ",
        {PP_define_body, "symphony \\"},  // end with line continuation
                                          // No \n before EOF
    },
    {
        {PP_define, "`define"},
        " ",
        {PP_Identifier, "UNFINISHED"},
        " ",
        {PP_define_body, "symphony \\\n  number"},
        // No \n before EOF
    },
};

static std::initializer_list<GenericTestDataSequence> kProtectedTests = {
    // TODO(b/134180314): lexer current drops tokens from protected sections,
    // but instead emit the tokens and filter them out.
    {"module foo;\n"
     "`protected\n"
     "`endprotected\n"
     "endmodule\n",
     {TK_module, TK_SPACE, SymbolIdentifier, ';', TK_NEWLINE, TK_NEWLINE,
      TK_endmodule, TK_NEWLINE}},
    {"abc\n"
     "`protected\n"
     "(*&^%$#@!!@#$%^&*(a8sayasd\n"
     "87y&*H*HYgyGYh@#BH)72361^&*\n"
     "`endprotected\n"
     "begin\n",
     {SymbolIdentifier, TK_NEWLINE, TK_NEWLINE, TK_begin, TK_NEWLINE}},
    {"task bar;\n"
     "// pragma protect begin_protected\n"
     "// pragma protect end_protected\n"
     "endtask\n",
     {TK_task, TK_SPACE, SymbolIdentifier, ';', TK_NEWLINE, TK_NEWLINE,
      TK_endtask, TK_NEWLINE}},
    {"end\n"
     "//pragma protect begin_protected\n"
     "(*&^%$#@!!@#$%^&*(a8sayasd\n"
     "87y&*H*HYgyGYh@#BH)72361^&*\n"
     "//pragma protect end_protected\n"
     "always\n",
     {TK_end, TK_NEWLINE, TK_NEWLINE, TK_always, TK_NEWLINE}},
    {"end\n"
     "`pragma protect begin_protected\n"
     "(*&^%$#@!!@#$%^&*(a8sayasd\n"
     "87y&*H*HYgyGYh@#BH)72361^&*\n"
     "`pragma protect end_protected\n"
     "begin\n",
     {TK_end, TK_NEWLINE, TK_NEWLINE, TK_begin, TK_NEWLINE}},
};

static std::initializer_list<SimpleTestData> kSymbolIdentifierTests = {
    {"a"},    {" a"},   {"\t\ta"},   {"\nb"},     {"_"},   {"_987"},
    {"_1u"},  {"_$"},   {"_$_"},     {"Foo"},     {"F00"}, {"Foo222"},
    {"_Foo"}, {"Foo$"}, {"Foo_Bar"}, {"Foo$Bar"},
};

static std::initializer_list<SimpleTestData> kEscapedIdentifierTests = {
    // escaped identifiers
    {"\\foo "},
    {"\\__ "},
    {"\\555 "},
    {"\\these.are[really]ugly "},
    {"\\5.6.7.8 "},
    {"\\)(*&^%$#@! "},
    {"\\]fdg[];'[as321=-+/.,>. "},
    {"\\..:;;{]{*&!6^$94* "},
};

static std::initializer_list<SimpleTestData> kMacroIdentifierTests = {
    // treat `identifier (macro-constants) like plain identifier
    {"`foo"},
    {"`foo789"},
    {"`___"},
    {"`_oo00OO"},
};

static std::initializer_list<LexerTestData> kKeywordTests = {
    {{TK_1step, "1step"}},
    {{TK_always, "always"}},
    {{TK_and, "and"}},
    {{TK_assign, "assign"}},
    {{TK_begin, "begin"}},
    {{TK_buf, "buf"}},
    {{TK_bufif0, "bufif0"}},
    {{TK_bufif1, "bufif1"}},
    {{TK_case, "case"}},
    {{TK_casex, "casex"}},
    {{TK_casez, "casez"}},
    {{TK_cmos, "cmos"}},
    {{TK_deassign, "deassign"}},
    {{TK_default, "default"}},
    {{TK_defparam, "defparam"}},
    {{TK_disable, "disable"}},
    {{TK_edge, "edge"}},
    {{TK_else, "else"}},
    {{TK_end, "end"}},
    {{TK_endcase, "endcase"}},
    {{TK_endfunction, "endfunction"}},
    {{TK_endmodule, "endmodule"}},
    {{TK_endprimitive, "endprimitive"}},
    {{TK_endspecify, "endspecify"}},
    {{SymbolIdentifier, "endtable"}},  // is keyword in table context
    {{TK_endtask, "endtask"}},
    {{TK_event, "event"}},
    {{SymbolIdentifier, "find"}},
    {{SymbolIdentifier, "find_index"}},
    {{SymbolIdentifier, "find_first"}},
    {{SymbolIdentifier, "find_first_index"}},
    {{SymbolIdentifier, "find_last"}},
    {{SymbolIdentifier, "find_last_index"}},
    {{TK_for, "for"}},
    {{TK_force, "force"}},
    {{TK_forever, "forever"}},
    {{TK_fork, "fork"}},
    {{TK_function, "function"}},
    {{TK_highz0, "highz0"}},
    {{TK_highz1, "highz1"}},
    {{TK_if, "if"}},
    {{TK_ifnone, "ifnone"}},
    {{TK_initial, "initial"}},
    {{TK_inout, "inout"}},
    {{TK_input, "input"}},
    {{TK_integer, "integer"}},
    {{TK_join, "join"}},
    {{TK_large, "large"}},
    {{TK_macromodule, "macromodule"}},
    {{TK_medium, "medium"}},
    {{TK_module, "module"}},
    {{TK_nand, "nand"}},
    {{TK_negedge, "negedge"}},
    {{TK_nmos, "nmos"}},
    {{TK_nor, "nor"}},
    {{TK_not, "not"}},
    {{TK_notif0, "notif0"}},
    {{TK_notif1, "notif1"}},
    {{TK_or, "or"}},
    {{TK_output, "output"}},
    {{TK_parameter, "parameter"}},
    {{TK_pmos, "pmos"}},
    {{TK_posedge, "posedge"}},
    {{TK_primitive, "primitive"}},
    {{TK_pull0, "pull0"}},
    {{TK_pull1, "pull1"}},
    {{TK_pulldown, "pulldown"}},
    {{TK_pullup, "pullup"}},
    {{TK_rcmos, "rcmos"}},
    {{TK_real, "real"}},
    {{TK_realtime, "realtime"}},
    {{TK_reg, "reg"}},
    {{TK_release, "release"}},
    {{TK_repeat, "repeat"}},
    {{TK_rnmos, "rnmos"}},
    {{TK_rpmos, "rpmos"}},
    {{TK_rtran, "rtran"}},
    {{TK_rtranif0, "rtranif0"}},
    {{TK_rtranif1, "rtranif1"}},
    {{TK_scalared, "scalared"}},
    {{TK_small, "small"}},
    {{TK_specify, "specify"}},
    {{TK_specparam, "specparam"}},
    {{TK_strong0, "strong0"}},
    {{TK_strong1, "strong1"}},
    {{TK_supply0, "supply0"}},
    {{TK_supply1, "supply1"}},
    {{SymbolIdentifier, "table"}},  // is keyword inside primitive context
    {{TK_task, "task"}},
    {{TK_time, "time"}},
    {{TK_tran, "tran"}},
    {{TK_tranif0, "tranif0"}},
    {{TK_tranif1, "tranif1"}},
    {{TK_tri, "tri"}},
    {{TK_tri0, "tri0"}},
    {{TK_tri1, "tri1"}},
    {{TK_triand, "triand"}},
    {{TK_trior, "trior"}},
    {{TK_trireg, "trireg"}},
    {{TK_vectored, "vectored"}},
    {{TK_wait, "wait"}},
    {{TK_wand, "wand"}},
    {{TK_weak0, "weak0"}},
    {{TK_weak1, "weak1"}},
    {{TK_while, "while"}},
    {{TK_wire, "wire"}},
    {{TK_wor, "wor"}},
    {{TK_xnor, "xnor"}},
    {{TK_xor, "xor"}},
    {{TK_Shold, "$hold"}},
    {{TK_Snochange, "$nochange"}},
    {{TK_Speriod, "$period"}},
    {{TK_Srecovery, "$recovery"}},
    {{TK_Ssetup, "$setup"}},
    {{TK_Ssetuphold, "$setuphold"}},
    {{TK_Sskew, "$skew"}},
    {{TK_Swidth, "$width"}},
    {{TKK_attribute, "$attribute"}},
    {{TK_automatic, "automatic"}},
    {{TK_endgenerate, "endgenerate"}},
    {{TK_generate, "generate"}},
    {{TK_genvar, "genvar"}},
    {{TK_localparam, "localparam"}},
    {{TK_noshowcancelled, "noshowcancelled"}},
    {{TK_pulsestyle_onevent, "pulsestyle_onevent"}},
    {{TK_pulsestyle_ondetect, "pulsestyle_ondetect"}},
    {{TK_showcancelled, "showcancelled"}},
    {{TK_signed, "signed"}},
    {{TK_unsigned, "unsigned"}},
    {{TK_Sfullskew, "$fullskew"}},
    {{TK_Srecrem, "$recrem"}},
    {{TK_Sremoval, "$removal"}},
    {{TK_Stimeskew, "$timeskew"}},
    {{TK_cell, "cell"}},
    {{TK_config, "config"}},
    {{TK_design, "design"}},
    {{TK_endconfig, "endconfig"}},
    {{TK_incdir, "incdir"}},
    {{TK_include, "include"}},
    {{TK_instance, "instance"}},
    {{TK_liblist, "liblist"}},
    {{TK_library, "library"}},
    {{TK_use, "use"}},
    {{TK_wone, "wone"}},
    {{TK_uwire, "uwire"}},
    {{TK_alias, "alias"}},
    {{TK_always_comb, "always_comb"}},
    {{TK_always_ff, "always_ff"}},
    {{TK_always_latch, "always_latch"}},
    {{TK_assert, "assert"}},
    {{TK_assume, "assume"}},
    {{TK_before, "before"}},
    {{TK_bind, "bind"}},
    {{TK_bins, "bins"}},
    {{TK_binsof, "binsof"}},
    {{TK_bit, "bit"}},
    {{TK_break, "break"}},
    {{TK_byte, "byte"}},
    {{TK_chandle, "chandle"}},
    {{TK_class, "class"}},
    {{TK_clocking, "clocking"}},
    {{TK_const, "const"}},
    {{TK_constraint, "constraint"}},
    {{TK_context, "context"}},
    {{TK_continue, "continue"}},
    {{TK_cover, "cover"}},
    {{TK_covergroup, "covergroup"}},
    {{TK_coverpoint, "coverpoint"}},
    {{TK_cross, "cross"}},
    {{TK_dist, "dist"}},
    {{TK_do, "do"}},
    {{TK_endclass, "endclass"}},
    {{TK_endclocking, "endclocking"}},
    {{TK_endgroup, "endgroup"}},
    {{TK_endinterface, "endinterface"}},
    {{TK_endpackage, "endpackage"}},
    {{TK_endprogram, "endprogram"}},
    {{TK_endproperty, "endproperty"}},
    {{TK_endsequence, "endsequence"}},
    {{TK_enum, "enum"}},
    {{TK_expect, "expect"}},
    {{TK_export, "export"}},
    {{TK_extends, "extends"}},
    {{TK_extern, "extern"}},
    {{TK_final, "final"}},
    {{TK_first_match, "first_match"}},
    {{TK_foreach, "foreach"}},
    {{TK_forkjoin, "forkjoin"}},
    {{TK_iff, "iff"}},
    {{TK_ignore_bins, "ignore_bins"}},
    {{TK_illegal_bins, "illegal_bins"}},
    {{TK_import, "import"}},
    {{TK_inside, "inside"}},
    {{TK_int, "int"}},
    {{TK_interface, "interface"}},
    {{TK_intersect, "intersect"}},
    {{TK_join_any, "join_any"}},
    {{TK_join_none, "join_none"}},
    {{TK_local_SCOPE, "local::"}},
    {{TK_local, "local"}},
    {{TK_logic, "logic"}},
    {{TK_longint, "longint"}},
    {{TK_matches, "matches"}},
    {{TK_modport, "modport"}},
    {{TK_new, "new"}},
    {{TK_null, "null"}},
    {{TK_package, "package"}},
    {{TK_packed, "packed"}},
    {{TK_priority, "priority"}},
    {{TK_program, "program"}},
    {{TK_property, "property"}},
    {{TK_protected, "protected"}},
    {{TK_pure, "pure"}},
    {{TK_rand, "rand"}},
    {{TK_randc, "randc"}},
    {{TK_randcase, "randcase"}},
    {{TK_randsequence, "randsequence"}},
    {{TK_randomize, "randomize"}},
    {{TK_randomize, "std::randomize"}},
    {{TK_ref, "ref"}},
    {{TK_return, "return"}},
    {{TK_Sroot, "$root"}},
    {{TK_sequence, "sequence"}},
    {{TK_shortint, "shortint"}},
    {{TK_shortreal, "shortreal"}},
    {{TK_solve, "solve"}},
    {{TK_static, "static"}},
    {{TK_string, "string"}},
    {{TK_struct, "struct"}},
    {{TK_super, "super"}},
    {{TK_tagged, "tagged"}},
    {{TK_this, "this"}},
    {{TK_throughout, "throughout"}},
    {{TK_timeprecision, "timeprecision"}},
    {{TK_timeunit, "timeunit"}},
    {{TK_type, "type"}},
    {{TK_typedef, "typedef"}},
    {{TK_union, "union"}},
    {{TK_unique, "unique"}},
    {{TK_Sunit, "$unit"}},
    {{TK_var, "var"}},
    {{TK_virtual, "virtual"}},
    {{TK_void, "void"}},
    {{TK_wait_order, "wait_order"}},
    {{TK_wildcard, "wildcard"}},
    {{TK_with, "with"}},
    {{TK_within, "within"}},
    {{TK_timeprecision_check, "timeprecision_check"}},
    {{TK_timeunit_check, "timeunit_check"}},
    {{TK_accept_on, "accept_on"}},
    {{TK_checker, "checker"}},
    {{TK_endchecker, "endchecker"}},
    {{TK_eventually, "eventually"}},
    {{TK_global, "global"}},
    {{TK_implies, "implies"}},
    {{TK_let, "let"}},
    {{TK_nexttime, "nexttime"}},
    {{TK_reject_on, "reject_on"}},
    {{TK_restrict, "restrict"}},
    {{TK_s_always, "s_always"}},
    {{TK_s_eventually, "s_eventually"}},
    {{TK_s_nexttime, "s_nexttime"}},
    {{TK_s_until, "s_until"}},
    {{TK_s_until_with, "s_until_with"}},
    {{TK_strong, "strong"}},
    {{TK_sync_accept_on, "sync_accept_on"}},
    {{TK_sync_reject_on, "sync_reject_on"}},
    {{TK_unique0, "unique0"}},
    {{TK_until, "until"}},
    {{TK_until_with, "until_with"}},
    {{TK_untyped, "untyped"}},
    {{TK_weak, "weak"}},
    {{TK_implements, "implements"}},
    {{TK_interconnect, "interconnect"}},
    {{TK_nettype, "nettype"}},
    {{TK_soft, "soft"}},
    {{TK_absdelay, "absdelay"}},
    {{TK_abstol, "abstol"}},
    {{TK_access, "access"}},
    {{TK_ac_stim, "ac_stim"}},
    {{TK_aliasparam, "aliasparam"}},
    {{TK_analog, "analog"}},
    {{TK_analysis, "analysis"}},
    {{TK_connectmodule, "connectmodule"}},
    {{TK_connectrules, "connectrules"}},
    {{TK_continuous, "continuous"}},
    {{TK_ddt_nature, "ddt_nature"}},
    {{TK_discipline, "discipline"}},
    {{TK_discrete, "discrete"}},
    {{SymbolIdentifier, "domain"}},  // is keyword in discipline context
    {{TK_driver_update, "driver_update"}},
    {{TK_endconnectrules, "endconnectrules"}},
    {{TK_enddiscipline, "enddiscipline"}},
    {{TK_endnature, "endnature"}},
    {{TK_endparamset, "endparamset"}},
    {{TK_exclude, "exclude"}},
    {{TK_flicker_noise, "flicker_noise"}},
    {{TK_flow, "flow"}},
    {{TK_from, "from"}},
    {{TK_idt_nature, "idt_nature"}},
    {{TK_inf, "inf"}},
    {{TK_infinite, "infinite"}},
    {{TK_laplace_nd, "laplace_nd"}},
    {{TK_laplace_np, "laplace_np"}},
    {{TK_laplace_zd, "laplace_zd"}},
    {{TK_laplace_zp, "laplace_zp"}},
    {{TK_last_crossing, "last_crossing"}},
    {{TK_limexp, "limexp"}},
    {{SymbolIdentifier, "max"}},
    {{SymbolIdentifier, "min"}},
    {{TK_nature, "nature"}},
    {{TK_net_resolution, "net_resolution"}},
    {{TK_noise_table, "noise_table"}},
    {{TK_paramset, "paramset"}},
    {{TK_potential, "potential"}},
    {{TK_resolveto, "resolveto"}},
    {{TK_transition, "transition"}},
    {{TK_units, "units"}},
    {{TK_white_noise, "white_noise"}},
    {{TK_wreal, "wreal"}},
    {{TK_zi_nd, "zi_nd"}},
    {{TK_zi_np, "zi_np"}},
    {{TK_zi_zd, "zi_zd"}},
    {{TK_zi_zp, "zi_zp"}},
    {{TK_PLUS_EQ, "+="}},
    {{TK_MINUS_EQ, "-="}},
    {{TK_MUL_EQ, "*="}},
    {{TK_DIV_EQ, "/="}},
    {{TK_MOD_EQ, "%="}},
    {{TK_AND_EQ, "&="}},
    {{TK_OR_EQ, "|="}},
    {{TK_XOR_EQ, "^="}},
    {{TK_INCR, "++"}},
    {{TK_DECR, "--"}},
    {{TK_LE, "<="}},
    {{TK_GE, ">="}},
    {{TK_EG, "=>"}},
    {{TK_WILDCARD_EQ, "==?"}},
    {{TK_EQ, "=="}},
    {{TK_WILDCARD_NE, "!=?"}},
    {{TK_NE, "!="}},
    {{TK_CEQ, "==="}},
    {{TK_CNE, "!=="}},
    {{TK_LP, "'{"}},
    {{TK_LS, "<<"}},
    {{TK_RS, ">>"}},
    {{TK_RSS, ">>>"}},
    {{TK_SG, "*>"}},
    {{TK_CONTRIBUTE, "<+"}},
    {{TK_PO_POS, "+:"}},
    {{TK_PO_NEG, "-:"}},
    {{TK_POW, "**"}},
    // {{TK_PSTAR , { "(*"}},  // ignored by lexer, treated as comment
    // {{TK_STARP , { "*)"}},  // ignored by lexer, treated as comment
    {{TK_DOTSTAR, ".*"}},
    {{TK_LOR, "||"}},
    {{TK_LAND, "&&"}},
    {{TK_TAND, "&&&"}},
    {{TK_NAND, "~&"}},
    {{TK_NOR, "~|"}},
    {{TK_NXOR, "~^"}},
    {{TK_NONBLOCKING_TRIGGER, "->>"}},
    {{_TK_RARROW, "->"}},  // This can disambiguate to different enums,
                           // depending on context.
    {{TK_LOGEQUIV, "<->"}},
    {{TK_SCOPE_RES, "::"}},
    {{TK_COLON_EQ, ":="}},
    {{TK_COLON_DIV, ":/"}},
    {{TK_POUNDPOUND, "##"}},
    {{TK_LBSTAR, "[*"}},
    {{TK_LBSTARRB, "[*]"}},
    {{TK_LBPLUSRB, "[+]"}},
    {{TK_LBEQ, "[="}},
    {{TK_LBRARROW, "[->"}},
};

static std::initializer_list<SimpleTestData> kSingleCharTokenTests = {
    {"}"}, {"{"}, {";"}, {":"}, {"["}, {"]"}, {","}, {"("}, {")"},
    {"#"}, {"="}, {"."}, {"@"}, {"&"}, {"!"}, {"?"}, {"<"}, {">"},
    {"%"}, {"|"}, {"^"}, {"~"}, {"+"}, {"*"}, {"/"}, {"-"}, {"\'"},
};

static std::initializer_list<SimpleTestData> kRealTimeTests = {
    // scientific units
    {"1e-3"},
    {"2e+3"},
    {"3e6"},
    {"4E6"},
    {"5.12e-6"},
    {"0e-0"},
    {"0e+0"},
    {"6e-12"},
    {"7e-100"},
    {"6.02e-23"},
    {"2.9e8"},
    // scaled units are for Verilog-AMS
    {"1u"},
    {"2a"},
    {"3f"},
    {"4m"},
    {"5p"},
    {"6k"},
    {"6K"},
    {"7M"},
    {"8G"},
    {"9T"},
    {"2.5n"},
    {"100p"},
    {"10.0p"},
    {"0.01m"},
    {"0m"},
};

static std::initializer_list<LexerTestData> kBasedNumberTests = {
    {{TK_UnBasedNumber, "'0"}},
    {{TK_UnBasedNumber, "'1"}},
    {{TK_UnBasedNumber, "'x"}},
    {{TK_UnBasedNumber, "'X"}},
    {{TK_UnBasedNumber, "'z"}},
    {{TK_UnBasedNumber, "'Z"}},
    {{TK_DecBase, "'d"}, {TK_DecDigits, "0"}},
    {{TK_DecBase, "'d"}, {TK_SPACE, " "}, {TK_DecDigits, "0"}},
    {{TK_DecBase, "'d"}, {TK_DecDigits, "20"}},
    {{TK_DecBase, "'d"}, {TK_DecDigits, "20__"}},
    {{TK_DecBase, "'sd"}, {TK_DecDigits, "9"}},
    {{TK_DecBase, "'SD"}, {TK_DecDigits, "12"}},
    {{TK_DecBase, "'d"}, {TK_XZDigits, "x"}},
    {{TK_DecBase, "'d"}, {TK_SPACE, " "}, {TK_XZDigits, "x"}},
    {{TK_DecBase, "'d"}, {TK_NEWLINE, "\n"}, {TK_XZDigits, "x"}},
    {{TK_DecBase, "'d"}, {TK_XZDigits, "x_"}},
    {{TK_DecBase, "'d"}, {TK_XZDigits, "Z"}},
    {{TK_DecBase, "'d"}, {TK_XZDigits, "?"}},
    {{TK_BinBase, "'B"}, {TK_BinDigits, "0"}},
    {{TK_BinBase, "'b"}, {TK_BinDigits, "1"}},
    {{TK_BinBase, "'b"}, {TK_SPACE, " "}, {TK_BinDigits, "1"}},
    {{TK_BinBase, "'b"}, {TK_NEWLINE, "\n"}, {TK_BinDigits, "1"}},
    {{TK_BinBase, "'b"}, {TK_BinDigits, "x"}},
    {{TK_BinBase, "'b"}, {TK_BinDigits, "z"}},
    {{TK_BinBase, "'b"}, {TK_BinDigits, "1100"}},
    {{TK_BinBase, "'b"}, {TK_BinDigits, "xX10Zz"}},
    {{TK_BinBase, "'b"}, {TK_BinDigits, "_x_X_10_Zz"}},
    {{TK_OctBase, "'o"}, {TK_OctDigits, "0"}},
    {{TK_OctBase, "'o"}, {TK_OctDigits, "000"}},
    {{TK_OctBase, "'o"}, {TK_OctDigits, "777"}},
    {{TK_OctBase, "'o"}, {TK_SPACE, " "}, {TK_OctDigits, "777"}},
    {{TK_OctBase, "'o"}, {TK_NEWLINE, "\n"}, {TK_OctDigits, "777"}},
    {{TK_OctBase, "'o"}, {TK_OctDigits, "_007_007_"}},
    {{TK_HexBase, "'h"}, {TK_HexDigits, "0"}},
    {{TK_HexBase, "'h"}, {TK_SPACE, " "}, {TK_HexDigits, "0"}},
    {{TK_HexBase, "'h"}, {TK_NEWLINE, "\n"}, {TK_HexDigits, "0"}},
    {{TK_HexBase, "'h"}, {TK_HexDigits, "a"}},
    {{TK_HexBase, "'h"}, {TK_HexDigits, "e110"}},
    {{TK_HexBase, "'h"}, {TK_HexDigits, "_feed_face"}},
    {{TK_HexBase, "'h"}, {TK_HexDigits, "0_cafebabe_"}},
    {{TK_HexBase, "'h"}, {TK_HexDigits, "00d_f00d"}},
    {{TK_HexBase, "'h"}, {TK_HexDigits, "xXXxcaca"}},
    {{TK_HexBase, "'h"}, {TK_HexDigits, "cacazzZZ"}},
    {{TK_HexBase, "'h"}, {TK_HexDigits, "e110???_000"}},
};

static std::initializer_list<SimpleTestData> kTimeLiteralTests = {
    {"1fs"},    {"2ps"},    {"3ns"},      {"4us"},    {"5ms"}, {"6.6fs"},
    {"7.77ps"}, {"88.0ns"}, {"999.99us"}, {"10.0ms"}, {"66s"},
    // {"1 ms"}, {"11.0 ms"},  // only allowed in `timescale context
};

static std::initializer_list<SimpleTestData> kStringLiteralTests = {
    {"\"\""},
    {"\".\""},
    {"\"...\""},
    {"\":)\""},
    {"\"^ ^\""},
    {"\"O_o\""},
    {"\"foobar\""},
    {"\"foo bar\""},
    {"\"the\\\\great\\\\escape\""},
    {"\"the\\\"great\\\"escape\""},
    {"\"the\\nGREAT\\nescape\""},  // with line continuations
    {"\"`abc\""},
    {"\"`abc()\""},
    {"\"`abc(d)\""},
    {"\"`abc(d,e)\""},
};

// `"...`" preprocessor-evaluated string literals
static std::initializer_list<SimpleTestData> kEvalStringLiteralTests = {
    {"`\"`\""},           //
    {"`\" `\""},          //
    {"`\"123`\""},        //
    {"`\"abc`\""},        //
    {"`\"`abc`\""},       //
    {"`\"`abc()`\""},     //
    {"`\"`abc(d)`\""},    //
    {"`\"`abc(d,e)`\""},  //
    {"`\"```\""},         // token concatenation operator in middle
};

// tokens with special handling in lexer
static std::initializer_list<LexerTestData> kTrickyTests = {
    {{TK_COLON_DIV, ":/"}, {TK_SPACE, " "}},
    {{TK_COLON_DIV, ":/"}, {TK_DecNumber, "8"}},
    {':', {TK_EOL_COMMENT, "//"}, {TK_NEWLINE, "\n"}},
    {':', {TK_COMMENT_BLOCK, "/*ignore*/"}, {TK_NEWLINE, "\n"}},
    {{TK_DecNumber, "0"},
     ':',
     {TK_EOL_COMMENT, "// case description"},
     {TK_NEWLINE, "\n"}},
    {{SymbolIdentifier, "FOO"},
     {TK_SPACE, " "},
     {TK_LINE_CONT, "\\"},
     {TK_NEWLINE, "\n"},
     {SymbolIdentifier, "BAR"},
     ';'},
    {
        {TK_EOL_COMMENT, "//FOO"},
        {TK_LINE_CONT, "\\"},
        {TK_NEWLINE, "\n"},
    },
    {
        {TK_EOL_COMMENT, "//FOO"},
        {TK_LINE_CONT, "\\"},
        {TK_NEWLINE, "\n"},
        {TK_EOL_COMMENT, "//BAR  "},
        {TK_LINE_CONT, "\\"},
        {TK_NEWLINE, "\n"},
    },
};

// make sure these are lexed as separate tokens
static std::initializer_list<LexerTestData> kSequenceTests = {
    {{'+', "+"}, {TK_EG, "=>"}},
    {{'-', "-"}, {TK_EG, "=>"}},
    {{TK_POUNDPOUND, "##"}, {'#', "#"}},
    {{TK_POUNDPOUND, "##"}, {TK_POUNDPOUND, "##"}},
    {{'[', "["}, {'-', "-"}, {TK_DecNumber, "12"}, {']', "]"}},
    {{TK_LBSTAR, "[*"}, {TK_DecNumber, "12"}, {']', "]"}},
    {{'[', "["}, " ", {'*', "*"}, {TK_DecNumber, "3"}, {']', "]"}},
    {{TK_LBEQ, "[="}, {TK_DecNumber, "1"}, {']', "]"}},
    {{'[', "["}, " ", {'=', "="}, {TK_DecNumber, "1"}, {']', "]"}},
    {{TK_LBRARROW, "[->"}, {']', "]"}},
    {{'[', "["}, " ", {_TK_RARROW, "->"}, {']', "]"}},
    {{MacroNumericWidth, "`WIDTH"}, {TK_DecBase, "'d"}, {TK_DecDigits, "777"}},
    {{MacroNumericWidth, "`WIDTH"},
     {TK_DecBase, "'d"},
     {MacroIdentifier, "`DIGITS"}},
    {{MacroNumericWidth, "`WIDTH"},
     " ",
     {TK_HexBase, "'h"},
     {TK_HexDigits, "888"}},
    {{MacroNumericWidth, "`WIDTH"},
     " ",
     {TK_HexBase, "'h"},
     {MacroIdentifier, "`DIGITS"}},
    {{MacroNumericWidth, "`WIDTH"},
     " \t ",
     {TK_OctBase, "'o"},
     {TK_OctDigits, "654"}},
    {{MacroNumericWidth, "`WIDTH"}, {TK_BinBase, "'b"}, {TK_BinDigits, "101"}},
    {{MacroNumericWidth, "`WIDTH"}, {TK_BinBase, "'b"}, {TK_BinDigits, "10zx"}},
    {{MacroNumericWidth, "`WIDTH"},
     {TK_BinBase, "'b"},
     {MacroIdentifier, "`DIGITS"}},
};

static std::initializer_list<LexerTestData> kContextKeywordTests = {
    {{SymbolIdentifier, "option"}},
    {{TK_covergroup, "covergroup"},
     " ",
     {SymbolIdentifier, "blah1"},
     " ",
     {TK_option, "option"},
     " ",
     {SymbolIdentifier, "blah2"},
     " ",
     {TK_endgroup, "endgroup"},
     " ",
     {SymbolIdentifier, "option"}},
    {{TK_with, "with"},
     " ",
     {TK_covergroup, "covergroup"},
     " ",
     {TK_with__covergroup, "with"},
     " ",
     {TK_endgroup, "endgroup"},
     " ",
     {TK_with, "with"}},
    {{SymbolIdentifier, "domain"},
     " ",
     {TK_discipline, "discipline"},
     " ",
     {TK_domain, "domain"},
     " ",
     {TK_enddiscipline, "enddiscipline"},
     " ",
     {SymbolIdentifier, "domain"}}};

static std::initializer_list<LexerTestData> kBuiltinMethodKeywordTests = {
    {{'.', "."}, " ", {TK_randomize, "randomize"}, {'(', "("}, {')', ")"}},
    {{'.', "."}, {TK_randomize, "randomize"}, " ", {TK_with, "with"}},
    {{SymbolIdentifier, "min"},
     " ",
     {'.', "."},
     " ",
     {TK_min, "min"},
     " ",
     {SymbolIdentifier, "min"}},
    {{SymbolIdentifier, "min"},
     {'.', "."},
     {SymbolIdentifier, "minime"},
     " ",
     {SymbolIdentifier, "min"}},
    {{SymbolIdentifier, "max"},
     {'.', "."},
     " ",
     {TK_max, "max"},
     " ",
     {SymbolIdentifier, "max"}},
    {{SymbolIdentifier, "max"},
     {'.', "."},
     {SymbolIdentifier, "maxime"},
     " ",
     {SymbolIdentifier, "max"}},
    {{SymbolIdentifier, "find"},
     " ",
     {'.', "."},
     " ",
     {TK_find, "find"},
     " ",
     {SymbolIdentifier, "find"}},
    {{'.', "."}, {SymbolIdentifier, "finder"}},
    {{SymbolIdentifier, "find_index"},
     " ",
     {'.', "."},
     " ",
     {TK_find_index, "find_index"},
     " ",
     {SymbolIdentifier, "find_index"}},
    {{SymbolIdentifier, "find_first_index"},
     {'.', "."},
     {TK_find_first_index, "find_first_index"},
     " ",
     {SymbolIdentifier, "find_first_index"}},
    {{SymbolIdentifier, "find_last"},
     " ",
     {'.', "."},
     " ",
     {TK_find_last, "find_last"},
     " ",
     {SymbolIdentifier, "find_last"}},
    {{SymbolIdentifier, "find_last_index"},
     {'.', "."},
     {TK_find_last_index, "find_last_index"},
     " ",
     {SymbolIdentifier, "find_last_index"}},
    {{TK_unique, "unique"},
     " ",
     {'.', "."},
     " ",
     {TK_unique, "unique"},
     " ",
     {TK_unique, "unique"}},
    {{SymbolIdentifier, "uniquely"},
     {'.', "."},
     {SymbolIdentifier, "uniquely"},
     " ",
     {SymbolIdentifier, "uniquely"}},
    {{SymbolIdentifier, "unique_index"},
     " ",
     {'.', "."},
     " ",
     {TK_unique_index, "unique_index"},
     " ",
     {SymbolIdentifier, "unique_index"}},
    {{SymbolIdentifier, "sort"},
     " ",
     {'.', "."},
     " ",
     {TK_sort, "sort"},
     " ",
     {SymbolIdentifier, "sort"}},
    {{SymbolIdentifier, "rsort"},
     " ",
     {'.', "."},
     " ",
     {TK_rsort, "rsort"},
     " ",
     {SymbolIdentifier, "rsort"}},
    {{SymbolIdentifier, "reverse"},
     " ",
     {'.', "."},
     " ",
     {TK_reverse, "reverse"},
     " ",
     {SymbolIdentifier, "reverse"}},
    {{SymbolIdentifier, "shuffle"},
     " ",
     {'.', "."},
     {TK_shuffle, "shuffle"},
     " ",
     {SymbolIdentifier, "shuffle"}},
    {{SymbolIdentifier, "sum"},
     " ",
     {'.', "."},
     " ",
     {TK_sum, "sum"},
     " ",
     {SymbolIdentifier, "sum"}},
    {{SymbolIdentifier, "product"},
     " ",
     {'.', "."},
     " ",
     {TK_product, "product"},
     " ",
     {SymbolIdentifier, "product"}},
    {{TK_and, "and"},
     " ",
     {'.', "."},
     " ",
     {TK_and, "and"},
     " ",
     {TK_and, "and"}},
    {{TK_or, "or"}, " ", {'.', "."}, " ", {TK_or, "or"}, " ", {TK_or, "or"}},
    {{TK_xor, "xor"},
     " ",
     {'.', "."},
     " ",
     {TK_xor, "xor"},
     " ",
     {TK_xor, "xor"}},
};

static std::initializer_list<LexerTestData> kEdgeTests = {
    {{TK_edge, "edge"}, " ", {'[', "["}, {']', "]"}, " ", {TK_DecNumber, "10"}},
    {{TK_edge, "edge"},
     "\n",
     {'[', "["},
     {']', "]"},
     " ",
     {SymbolIdentifier, "z0"}},
    {{TK_edge, "edge"},
     {'[', "["},
     {TK_edge_descriptor, "10"},
     {']', "]"},
     " "},
    {{TK_edge, "edge"},
     "\n",
     {'[', "["},
     {TK_edge_descriptor, "z0"},
     {']', "]"}},
    {{TK_edge, "edge"},
     "\n",
     {'[', "["},
     {TK_edge_descriptor, "01"},
     " ",
     {TK_edge_descriptor, "0x"},
     " ",
     {TK_edge_descriptor, "1x"},
     " ",
     {TK_edge_descriptor, "x0"},
     " ",
     {TK_edge_descriptor, "z1"},
     " ",
     {']', "]"},
     " ",
     {TK_DecNumber, "10"}},
    {{TK_edge, "edge"},
     "\n",
     {'[', "["},
     {TK_edge_descriptor, "01"},
     " ",
     {',', ","},
     {TK_edge_descriptor, "0x"},
     " ",
     {',', ","},
     {TK_edge_descriptor, "1x"},
     {',', ","},
     {TK_edge_descriptor, "x0"},
     {',', ","},
     {TK_edge_descriptor, "z1"},
     {',', ","},
     {']', "]"},
     " ",
     {SymbolIdentifier, "x0"}},
    {{TK_edge, "edge"}, " ", {SymbolIdentifier, "x0"}},
    {{TK_edge, "edge"},
     " ",
     {SymbolIdentifier, "x0"},
     " ",
     {'[', "["},
     " ",
     {TK_DecNumber, "10"},
     " ",
     {']', "]"}},
};

static std::initializer_list<LexerTestData> kUDPTests = {
    {{TK_primitive, "primitive"},
     " ",
     {TK_table, "table"},
     " ",
     {TK_endtable, "endtable"},
     " ",
     {TK_endprimitive, "endprimitive"}},
    {{TK_primitive, "primitive"},
     " ",
     {TK_table, "table"},
     " ",
     {'b', "b"},
     {':', ":"},
     {'l', "l"},
     {';', ";"},
     " ",
     {TK_endtable, "endtable"},
     " ",
     {TK_endprimitive, "endprimitive"}},
    {{TK_primitive, "primitive"},
     " ",
     {TK_table, "table"},
     " ",
     {'b', "B"},
     {':', ":"},
     {'l', "L"},
     {':', ":"},
     {'h', "H"},
     {';', ";"},
     " ",
     {TK_endtable, "endtable"},
     " ",
     {TK_endprimitive, "endprimitive"}},
    {{TK_primitive, "primitive"},
     " ",
     {TK_table, "table"},
     " ",
     {'b', "b"},
     " ",
     {'b', "B"},
     " ",
     {'l', "l"},
     " ",
     {'l', "L"},
     " ",
     {'h', "h"},
     " ",
     {'h', "H"},
     " ",
     {'f', "f"},
     " ",
     {'f', "F"},
     " ",
     {'r', "r"},
     " ",
     {'r', "R"},
     " ",
     {'x', "x"},
     " ",
     {'x', "X"},
     " ",
     {'n', "n"},
     " ",
     {'n', "N"},
     " ",
     {'p', "p"},
     " ",
     {'p', "P"},
     " ",
     {'0', "0"},
     " ",
     {'1', "1"},
     " ",
     {'?', "?"},
     " ",
     {'*', "*"},
     " ",
     {'-', "-"},
     " ",
     {':', ":"},
     " ",
     {';', ";"},
     " ",
     {TK_endtable, "endtable"},
     " ",
     {SymbolIdentifier, "B"},
     " ",
     {SymbolIdentifier, "L"},
     " ",
     {SymbolIdentifier, "F"},
     " ",
     {SymbolIdentifier, "H"},
     " ",
     {SymbolIdentifier, "R"},
     " ",
     {TK_DecNumber, "1"},
     " ",
     {TK_endprimitive, "endprimitive"}},
    {{TK_primitive, "primitive"},
     " ",
     {TK_table, "table"},
     " ",
     {'_', "(?0)"},
     " ",
     {'+', "(?1)"},
     " ",
     {'%', "(?x)"},
     " ",
     {'%', "(?X)"},
     " ",
     {'*', "(?\?)"},  // Break tri-graph
     " ",
     {'r', "(01)"},
     " ",
     {'Q', "(0x)"},
     " ",
     {'Q', "(0X)"},
     " ",
     {'q', "(bx)"},
     " ",
     {'q', "(bX)"},
     " ",
     {'f', "(b0)"},
     " ",
     {'r', "(b1)"},
     " ",
     {'P', "(0?)"},
     " ",
     {'f', "(10)"},
     " ",
     {'M', "(1x)"},
     " ",
     {'M', "(1X)"},
     " ",
     {'N', "(1?)"},
     " ",
     {'F', "(x0)"},
     " ",
     {'F', "(X0)"},
     " ",
     {'R', "(x1)"},
     " ",
     {'R', "(X1)"},
     " ",
     {'B', "(x?)"},
     " ",
     {'B', "(X?)"},
     " ",
     {TK_endtable, "endtable"},
     " ",
     {SymbolIdentifier, "b0"},
     " ",
     {SymbolIdentifier, "bx"},
     " ",
     {SymbolIdentifier, "x1"},
     " ",
     {SymbolIdentifier, "X0"},
     " ",
     {TK_DecNumber, "10"},
     " ",
     {TK_endprimitive, "endprimitive"}},
    // TK_DecNumber inside UDP should be rejected, and will be caught by parser
    {{TK_primitive, "primitive"},
     " ",
     {TK_table, "table"},
     " ",
     {TK_OTHER, "321"}},
    {{TK_primitive, "primitive"},
     " ",
     {TK_table, "table"},
     " ",
     {'1', "1"},
     {'0', "0"},
     {'1', "1"},
     {'0', "0"}},
    {{TK_primitive, "primitive"},
     " ",
     {TK_table, "table"},
     " ",
     {'1', "1"},
     {'0', "0"},
     "\t",
     {'1', "1"},
     {'0', "0"}},
    {{TK_primitive, "primitive"},
     " ",
     {TK_table, "table"},
     " ",
     {'1', "1"},
     {'0', "0"},
     "\n",
     {PP_ifdef, "`ifdef"},
     " ",
     {PP_Identifier, "BLAH"},
     "\n",
     {'1', "1"},
     {'0', "0"}},
    {// should not crash or stack-underflow
     {TK_endprimitive, "endprimitive"}},
    {// should not crash or stack-underflow
     {TK_endprimitive, "endprimitive"},
     " ",
     {TK_endprimitive, "endprimitive"}},
    {// should not crash or stack-underflow
     {TK_EOL_COMMENT, "//zzz"},
     "\n",
     {TK_endprimitive, "endprimitive"}},
};

static std::initializer_list<LexerTestData> kUnfilteredUDPTests = {
    {{TK_primitive, "primitive"},
     {TK_SPACE, " "},
     {TK_table, "table"},
     {TK_SPACE, " "},
     {'1', "1"},
     {'0', "0"},
     {TK_COMMENT_BLOCK, "/*comment*/"},
     {'1', "1"},
     {'0', "0"}},
};

static std::initializer_list<LexerTestData> kTimingCheckTests = {
    {{TK_Shold, "$hold"}},           {{TK_Snochange, "$nochange"}},
    {{TK_Speriod, "$period"}},       {{TK_Srecovery, "$recovery"}},
    {{TK_Ssetup, "$setup"}},         {{TK_Ssetuphold, "$setuphold"}},
    {{TK_Sskew, "$skew"}},           {{TK_Swidth, "$width"}},
    {{TK_Sfullskew, "$fullskew"}},   {{TK_Srecrem, "$recrem"}},
    {{TK_Sremoval, "$removal"}},     {{TK_Stimeskew, "$timeskew"}},
    {{TKK_attribute, "$attribute"}},
};

static std::initializer_list<LexerTestData> kSystemTaskFunctionTests = {
    {{SystemTFIdentifier, "$display"}},   {{SystemTFIdentifier, "$error"}},
    {{SystemTFIdentifier, "$info"}},      {{SystemTFIdentifier, "$warning"}},
    {{SystemTFIdentifier, "$fatal"}},     {{SystemTFIdentifier, "$finish"}},
    {{SystemTFIdentifier, "$stop"}},      {{SystemTFIdentifier, "$hammertime"}},
    {{SystemTFIdentifier, "$__MAGIC__"}}, {{SystemTFIdentifier, "$thechosen1"}},
};

static std::initializer_list<LexerTestData> kPreprocessorTests = {
    {{}, ""},
    {{PP_define, "`define"},
     " ",
     {PP_Identifier, "true"},
     " ",
     {PP_define_body, "false"},
     "\n"},
    {"    ",
     {PP_define, "`define"},
     " ",
     {PP_Identifier, "SIMULTION"},
     {PP_define_body, ""},
     "\n"},
    {
        {PP_elsif, "`elsif"},
        "\n",
    },
    {{PP_elsif, "`elsif"}, "\n", {SymbolIdentifier, "foo"}},
    {{PP_elsif, "`elsif"}, " ", {PP_Identifier, "foo"}, "\n"},
    {{PP_elsif, "`elsif"}, " ", {PP_Identifier, "foo"}, "  ", "\n"},
    {"    ", {PP_elsif, "`elsif"}, " ", {PP_Identifier, "foo"}, "  ", "\n"},
    {"    ", {PP_elsif, "`elsif"}, " ", {PP_Identifier, "foo"}, "\n"},
    {"    ",
     {PP_define, "`define"},
     " ",
     {PP_Identifier, "BAR"},
     {PP_define_body, ""},
     "\n"},
    {"\t\t",
     {PP_define, "`define"},
     " ",
     {PP_Identifier, "BEAR"},
     {PP_define_body, ""},
     "\n"},
    {{PP_else, "`else"}, "\n"},
    {{PP_else, "`else"}, " ", {SymbolIdentifier, "blah"}, "\n"},
    {{PP_endif, "`endif"}, "\n"},
    {{PP_endif, "`endif"}, " ", {SymbolIdentifier, "blah"}, "\n"},
    {{PP_ifdef, "`ifdef"}, " ", {PP_Identifier, "evil"}, "\n"},
    {"    ",
     {PP_ifdef, "`ifdef"},
     "    ",
     {PP_Identifier, "dont_be_evil"},
     "\n"},
    {"    ",
     {PP_undef, "`undef"},
     "\t",
     {PP_Identifier, "dont_be_evil"},
     "\t\n"},
    {{PP_ifndef, "`ifndef"}, " ", {PP_Identifier, "dont_be_evil"}, "\n"},
    {{PP_include, "`include"},
     " ",
     {TK_StringLiteral, "\"lib/standard_cells.v\""},
     "\n"},
    {{PP_include, "`include"},
     " ",
     {TK_AngleBracketInclude, "<lib/standard_cells.v>"},
     "\n"},
    {{PP_undef, "`undef"}, " ", {PP_Identifier, "behavior"}, "\n"},
    {{PP_ifdef, "`ifdef"},
     " ",
     {PP_Identifier, "god_mode"},
     "\n",
     {PP_define, "`define"},
     " ",
     {PP_Identifier, "private"},
     " ",
     {PP_define_body, "public"},
     "\n",
     {PP_elsif, "`elsif"},
     " ",
     {PP_Identifier, "hacker_mode"},
     "\n",
     "  ",
     {PP_elsif, "`elsif"},
     " ",
     {PP_Identifier, "sneaky_mode"},
     "   \n",
     "  ",
     {PP_elsif, "`elsif"},
     " ",
     {PP_Identifier, "squeaky_mode"},
     "\n",
     {PP_else, "`else"},
     "\n",
     {PP_endif, "`endif"},
     "\n"},
    // Token concatenations
    {
        {PP_TOKEN_CONCAT, "``"},
    },
    {
        {PP_TOKEN_CONCAT, "``"},
        "\n",
    },
    {
        " ",
        {PP_TOKEN_CONCAT, "``"},
        " ",
    },
    {
        {SymbolIdentifier, "ab"},
        {PP_TOKEN_CONCAT, "``"},
        {SymbolIdentifier, "cd"},
    },
};

static std::initializer_list<LexerTestData> kUnfilteredPreprocessorTests = {
    {{PP_else, "`else"},
     {TK_SPACE, " "},
     {TK_EOL_COMMENT, "// comment"},
     {TK_NEWLINE, "\n"}},
};

static std::initializer_list<LexerTestData> kUnfilteredDirectiveTests = {
    {{DR_timescale, "`timescale"},
     {TK_SPACE, " "},
     {TK_TimeLiteral, "1ps"},
     {'/', "/"},
     {TK_TimeLiteral, "1ps"},
     {TK_NEWLINE, "\n"}},
    // TODO(b/134180314): lexer current drops tokens from pragma sections,
    // but instead emit the tokens and filter them out.
    // {{DR_pragma, "`pragma"}, " fragma", {TK_NEWLINE, "\n"}},
};

static std::initializer_list<LexerTestData> kDirectiveTests = {
    {{DR_timescale, "`timescale"}},
    {{DR_timescale, "`timescale"},
     " ",
     {TK_TimeLiteral, "1ps"},
     {'/', "/"},
     {TK_TimeLiteral, "1ps"}},
    {"   ",
     {DR_timescale, "`timescale"},
     " ",
     {TK_TimeLiteral, "1ps"},
     {'/', "/"},
     {TK_TimeLiteral, "1ps"}},
    {{DR_timescale, "`timescale"},
     " ",
     {TK_TimeLiteral, "1s"},
     " ",
     {'/', "/"},
     " ",
     {TK_TimeLiteral, "1ms"}},
    {{DR_timescale, "`timescale"},
     " ",
     {TK_TimeLiteral, "100ps"},
     " ",
     {'/', "/"},
     " ",
     {TK_TimeLiteral, "100ps"}},
    {{DR_timescale, "`timescale"},
     " ",
     {TK_DecNumber, "100"},
     " ",
     {TK_timescale_unit, "ps"},
     " ",
     {'/', "/"},
     " ",
     {TK_DecNumber, "100"},
     " ",
     {TK_timescale_unit, "ps"}},
    {{DR_celldefine, "`celldefine"}},
    {{DR_celldefine, "`celldefine"}, "\n", {TK_module, "module"}},
    {{DR_endcelldefine, "`endcelldefine"}},
    {{DR_resetall, "`resetall"}},
    {{DR_unconnected_drive, "`unconnected_drive"},
     " ",
     {SymbolIdentifier, "blah"}},
    {{DR_nounconnected_drive, "`nounconnected_drive"},
     " ",
     {SymbolIdentifier, "blah"}},
    {{DR_default_decay_time, "`default_decay_time"},
     " ",
     {SymbolIdentifier, "blah"}},
    {{DR_default_decay_time, "`default_decay_time"},
     " ",
     {TK_infinite, "infinite"}},
    {{DR_default_trireg_strength, "`default_trireg_strength"},
     " ",
     {SymbolIdentifier, "blah"}},
    {{DR_delay_mode_path, "`delay_mode_path"}, "    "},
    {{DR_delay_mode_unit, "`delay_mode_unit"}, " ", {SymbolIdentifier, "blah"}},
    {{DR_delay_mode_zero, "`delay_mode_zero"}, " ", {SymbolIdentifier, "blah"}},
    {{DR_disable_portfaults, "`disable_portfaults"},
     " ",
     {SymbolIdentifier, "blah"}},
    {{DR_enable_portfaults, "`enable_portfaults"},
     " ",
     {SymbolIdentifier, "blah"}},
    {{DR_suppress_faults, "`suppress_faults"}, " ", {SymbolIdentifier, "blah"}},
    // TODO(b/134180314): lexer current drops tokens from pragma sections,
    // but instead emit the tokens and filter them out.
    // {{DR_uselib, "`uselib"},  " lib=schmoozelib\n"},
    {{DR_protect, "`protect"}},
    {{DR_endprotect, "`endprotect"}},
    {{DR_begin_keywords, "`begin_keywords"},
     " ",
     {SymbolIdentifier, "random"},
     " ",
     {TK_StringLiteral, "\"keywords\""}},
    {{DR_end_keywords, "`end_keywords"}},
    {{DR_default_nettype, "`default_nettype"}, " ", {SymbolIdentifier, "blah"}},
};

static std::initializer_list<LexerTestData> kLibraryTests = {
    {
        {TK_library, "library"},
        {TK_SPACE, " "},
        {SymbolIdentifier, "mylib"},
        {TK_SPACE, " "},
        {TK_FILEPATH, "1mylibpath/"},  // not an identifier
        ';',
        {SymbolIdentifier, "foo_bar_pkg"},  // not a file-path
        {TK_NEWLINE, "\n"},
    },
    {
        // with newlines instead of spaces
        {TK_library, "library"},
        {TK_NEWLINE, "\n"},
        {SymbolIdentifier, "mylib"},
        {TK_NEWLINE, "\n"},
        {TK_FILEPATH, "6666/"},  // not a number
        {TK_NEWLINE, "\n"},
        ';',
        {SymbolIdentifier, "foo_bar_pkg"},  // not a file-path
        {TK_NEWLINE, "\n"},
    },
    {
        // with comments
        {TK_library, "library"},
        {TK_COMMENT_BLOCK, "/*c1*/"},
        {SymbolIdentifier, "mylib"},
        {TK_SPACE, " "},
        {TK_FILEPATH, "2222"},  // not a number
        ';',
        {TK_COMMENT_BLOCK, "/*c3*/"},
        {SymbolIdentifier, "foo_bar_pkg"},  // not a file-path
        {TK_NEWLINE, "\n"},
    },
    {
        {TK_library, "library"},
        {TK_SPACE, " "},
        {SymbolIdentifier, "_my_lib"},
        {TK_SPACE, " "},
        {TK_FILEPATH, "_1my_lib_path/"},  // underscores
        ';',
        {SymbolIdentifier, "foo_bar_pkg"},  // not a file-path
        {TK_NEWLINE, "\n"},
    },
    {
        {TK_library, "library"},
        {TK_SPACE, " "},
        {SymbolIdentifier, "_my_lib"},
        {TK_SPACE, " "},
        {TK_FILEPATH, "1my-lib-path/"},  // dashes
        ';',
        {SymbolIdentifier, "foo_bar_pkg"},  // not a file-path
        {TK_NEWLINE, "\n"},
    },
    {{TK_library, "library"},
     {TK_SPACE, " "},
     {SymbolIdentifier, "mylib"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "/path1"},  // absolute path
     ',',
     {TK_SPACE, " "},
     {TK_FILEPATH, "/path2/path3"},
     ';',
     {TK_NEWLINE, "\n"}},
    {{TK_library, "library"},
     {TK_SPACE, " "},
     {SymbolIdentifier, "mylib"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "*.v"},  // * wildcard
     ';',
     {TK_NEWLINE, "\n"}},
    {{TK_library, "library"},
     {TK_SPACE, " "},
     {SymbolIdentifier, "mylib"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "foo?.v"},  // ? wildcard
     ';',
     {TK_NEWLINE, "\n"}},
    {{TK_library, "library"},
     {TK_SPACE, " "},
     {SymbolIdentifier, "mylib"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "foo/"},
     ';',
     {TK_NEWLINE, "\n"}},
    {{TK_library, "library"},
     {TK_SPACE, " "},
     {SymbolIdentifier, "mylib"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "Foo/*.v"},
     ';',
     {TK_NEWLINE, "\n"}},
    {{TK_library, "library"},
     {TK_SPACE, " "},
     {SymbolIdentifier, "mylib"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "*.v"},
     {TK_SPACE, " "},
     '-',
     {TK_incdir, "incdir"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "/path/to/files"},  // absolute path
     ',',
     {TK_SPACE, " "},
     {TK_FILEPATH, "/path/to/morefiles"},
     ';',
     {TK_NEWLINE, "\n"},
     {SymbolIdentifier, "not_a_path"}},
    {{TK_include, "include"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "foo/*.v"},
     ';',
     {TK_NEWLINE, "\n"}},
    {{TK_include, "include"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "../../foo/*.v"},  // up-dir, relative
     ';',
     {TK_NEWLINE, "\n"}},
    {{TK_include, "include"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "../../foo/..."},  // ... wildcard
     ';',
     {TK_NEWLINE, "\n"}},
    {{TK_include, "include"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "../foo/.../*.vg"},  // ... wildcard
     ';',
     {TK_NEWLINE, "\n"}},
    {{TK_include, "include"},
     {TK_SPACE, " "},
     {TK_FILEPATH, "foo/*.v"},
     ',',
     {TK_SPACE, " "},
     {TK_FILEPATH, "bar/*.vg"},
     ';',
     {TK_NEWLINE, "\n"}},
};

static std::initializer_list<LexerTestData> kLexicalErrorTests = {
    // TODO(fangism): Return different error enums for different errors.
    {{TK_OTHER, "111wire"}},
    {{TK_wire, "wire"}, " ", {TK_OTHER, "111wire"}, {';', ";"}},
    {{TK_OTHER, "`111macro"}, "\n"},
    {{TK_OTHER, "`111macrocall"}, {'(', "("}, {')', ")"}, "\n"},
    {{TK_OTHER, "`"}},
    {{TK_OTHER, "`"}, " "},
    {{TK_OTHER, "`"}, "\t"},
    {{TK_OTHER, "`"}, "\n"},
    {{TK_OTHER, "`"}, {TK_EOL_COMMENT, "//"}, "\n"},
    {{TK_OTHER, "`"}, {TK_COMMENT_BLOCK, "/* */"}},
    {{TK_OTHER, "`"}, {TK_DecNumber, "11"}},
    {{TK_OTHER, "`"}, " ", {SymbolIdentifier, "spacebad"}, "\n"},
    {{PP_TOKEN_CONCAT, "``"}, {TK_OTHER, "`"}},  // triple-tick breakdown
    {{TK_OTHER, "\""}, "\n"},
    {{TK_OTHER, "\"unterminated string literal"}},
    {{TK_OTHER, "\"unterminated string literal"}, "\n"},
    {{TK_OTHER, "\"unterminated \\\nstring \\\nliteral"}, "\n"},
    {{TK_OTHER, "`\"unterminated eval string literal"}},
    {{TK_OTHER, "`\"unterminated eval string literal\n"}},
    {{TK_OTHER, "`\"unterminated \"eval\" string literal"}},
    {{TK_OTHER, "/*"}},
    {{TK_OTHER, "/*\n"}},
    {{TK_OTHER, "/* interminated comment"}},
    // Unterminated string literal inside macro call args
    {{MacroCallId, "`b"}, '(', {TK_OTHER, "\");"}},
    {{MacroCallId, "`c"}, '(', {TK_OTHER, "/*"}},
    // Rejected ACSII characters
    {{TK_OTHER, "\x7F"}},
    {{TK_OTHER, "\x80"}},
    {{TK_OTHER, "\xFF"}},
};

TEST(VerilogLexerTest, LexicalErrors) { TestLexer(kLexicalErrorTests); }
TEST(VerilogLexerTest, NewlineTests) { TestLexer(kNewlineTests); }
TEST(VerilogLexerTest, Comments) { TestLexer(kCommentTests); }
TEST(VerilogLexerTest, Attributes) { TestLexer(kAttributeTests, TK_ATTRIBUTE); }
TEST(VerilogLexerTest, AttributesSequence) {
  TestLexer(kAttributeSequenceTests);
}
TEST(VerilogLexerTest, MacroCalls) { TestLexer(kMacroCallTests); }
TEST(VerilogLexerTest, MacroCallsUnfiltered) {
  TestLexer(kUnfilteredMacroCallTests);
}
TEST(VerilogLexerTest, MacroDefines) { TestLexer(kMacroDefineTests); }
TEST(VerilogLexerTest, SymbolIdentifiers) {
  TestFilteredLexer(kSymbolIdentifierTests, SymbolIdentifier);
}
TEST(VerilogLexerTest, EscapedIdentifiers) {
  TestFilteredLexer(kEscapedIdentifierTests, EscapedIdentifier);
}
TEST(VerilogLexerTest, MacroIdentifiers) {
  TestLexer(kMacroIdentifierTests, MacroIdentifier);
}
TEST(VerilogLexerTest, Protected) { TestLexer(kProtectedTests); }
TEST(VerilogLexerTest, Keywords) { TestFilteredLexer(kKeywordTests); }
TEST(VerilogLexerTest, SingleCharToken) {
  TestLexer(kSingleCharTokenTests, verible::SingleChar);
}
TEST(VerilogLexerTest, RealTime) { TestLexer(kRealTimeTests, TK_RealTime); }
TEST(VerilogLexerTest, BasedNumbers) { TestLexer(kBasedNumberTests); }
TEST(VerilogLexerTest, TimeLiteral) {
  TestLexer(kTimeLiteralTests, TK_TimeLiteral);
}
TEST(VerilogLexerTest, Tricky) { TestLexer(kTrickyTests); }
TEST(VerilogLexerTest, Sequence) { TestLexer(kSequenceTests); }
TEST(VerilogLexerTest, StringLiteral) {
  TestLexer(kStringLiteralTests, TK_StringLiteral);
}
TEST(VerilogLexerTest, EvalStringLiteral) {
  TestLexer(kEvalStringLiteralTests, TK_EvalStringLiteral);
}
TEST(VerilogLexerTest, Edges) { TestLexer(kEdgeTests); }
TEST(VerilogLexerTest, UDP) { TestLexer(kUDPTests); }
TEST(VerilogLexerTest, UDPUnfiltered) { TestLexer(kUnfilteredUDPTests); }
TEST(VerilogLexerTest, ContextKeywords) { TestLexer(kContextKeywordTests); }
TEST(VerilogLexerTest, BuiltinMethods) {
  TestLexer(kBuiltinMethodKeywordTests);
}
TEST(VerilogLexerTest, TimingChecks) { TestFilteredLexer(kTimingCheckTests); }
TEST(VerilogLexerTest, SystemTaskFunctionChecks) {
  TestFilteredLexer(kSystemTaskFunctionTests);
}
TEST(VerilogLexerTest, Preprocessor) { TestLexer(kPreprocessorTests); }
TEST(VerilogLexerTest, PreprocessorUnfiltered) {
  TestLexer(kUnfilteredPreprocessorTests);
}
TEST(VerilogLexerTest, Directives) { TestLexer(kDirectiveTests); }
TEST(VerilogLexerTest, DirectivesUnfiltered) {
  TestLexer(kUnfilteredDirectiveTests);
}
TEST(VerilogLexerTest, Library) { TestLexer(kLibraryTests); }

TEST(RecursiveLexTextTest, Basic) {
  constexpr absl::string_view text("hello;");
  std::vector<TokenInfo> tokens;
  RecursiveLexText(text,
                   [&tokens](const TokenInfo& t) { tokens.push_back(t); });
  EXPECT_THAT(tokens,
              ElementsAre(TokenInfo(SymbolIdentifier, text.substr(0, 5)),
                          TokenInfo(';', text.substr(5, 1))));
}

TEST(VerilogLexerTest, StrayNulCharacterHandledCorrectly) {
  {  // baseline
    FilteredVerilogLexer lexer(absl::string_view("foo bar baz", 11));
    for (absl::string_view expected : {"foo", "bar", "baz"}) {
      auto token = lexer.DoNextToken();
      EXPECT_EQ(token.text(), expected);
      EXPECT_EQ(token.token_enum(), SymbolIdentifier);
    }
    EXPECT_TRUE(lexer.DoNextToken().isEOF());
  }
  {  // Stray nul character in text should just be skipped as space
    FilteredVerilogLexer lexer(absl::string_view("foo bar\0baz", 11));
    for (absl::string_view expected : {"foo", "bar", "baz"}) {
      auto token = lexer.DoNextToken();
      EXPECT_EQ(token.text(), expected);
      EXPECT_EQ(token.token_enum(), SymbolIdentifier);
    }
    EXPECT_TRUE(lexer.DoNextToken().isEOF());
  }
}
}  // namespace
}  // namespace verilog