src/google/protobuf/io/tokenizer_unittest.cc

// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Author: kenton@google.com (Kenton Varda)
//  Based on original Protocol Buffers design by
//  Sanjay Ghemawat, Jeff Dean, and others.

#include <limits.h>
#include <math.h>

#include <vector>

#include <google/protobuf/io/tokenizer.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>

#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/substitute.h>
#include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h>

namespace google {
namespace protobuf {
namespace io {
namespace {

// ===================================================================
// Data-Driven Test Infrastructure

// TODO(kenton):  This is copied from coded_stream_unittest.  This is
//   temporary until these fetaures are integrated into gTest itself.

// TEST_1D and TEST_2D are macros I'd eventually like to see added to
// gTest.  These macros can be used to declare tests which should be
// run multiple times, once for each item in some input array.  TEST_1D
// tests all cases in a single input array.  TEST_2D tests all
// combinations of cases from two arrays.  The arrays must be statically
// defined such that the GOOGLE_ARRAYSIZE() macro works on them.  Example:
//
// int kCases[] = {1, 2, 3, 4}
// TEST_1D(MyFixture, MyTest, kCases) {
//   EXPECT_GT(kCases_case, 0);
// }
//
// This test iterates through the numbers 1, 2, 3, and 4 and tests that
// they are all grater than zero.  In case of failure, the exact case
// which failed will be printed.  The case type must be printable using
// ostream::operator<<.

#define TEST_1D(FIXTURE, NAME, CASES)                                      \
  class FIXTURE##_##NAME##_DD : public FIXTURE {                           \
   protected:                                                              \
    template <typename CaseType>                                           \
    void DoSingleCase(const CaseType& CASES##_case);                       \
  };                                                                       \
                                                                           \
  TEST_F(FIXTURE##_##NAME##_DD, NAME) {                                    \
    for (int i = 0; i < GOOGLE_ARRAYSIZE(CASES); i++) {                           \
      SCOPED_TRACE(testing::Message()                                      \
        << #CASES " case #" << i << ": " << CASES[i]);                     \
      DoSingleCase(CASES[i]);                                              \
    }                                                                      \
  }                                                                        \
                                                                           \
  template <typename CaseType>                                             \
  void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType& CASES##_case)

#define TEST_2D(FIXTURE, NAME, CASES1, CASES2)                             \
  class FIXTURE##_##NAME##_DD : public FIXTURE {                           \
   protected:                                                              \
    template <typename CaseType1, typename CaseType2>                      \
    void DoSingleCase(const CaseType1& CASES1##_case,                      \
                      const CaseType2& CASES2##_case);                     \
  };                                                                       \
                                                                           \
  TEST_F(FIXTURE##_##NAME##_DD, NAME) {                                    \
    for (int i = 0; i < GOOGLE_ARRAYSIZE(CASES1); i++) {                          \
      for (int j = 0; j < GOOGLE_ARRAYSIZE(CASES2); j++) {                        \
        SCOPED_TRACE(testing::Message()                                    \
          << #CASES1 " case #" << i << ": " << CASES1[i] << ", "           \
          << #CASES2 " case #" << j << ": " << CASES2[j]);                 \
        DoSingleCase(CASES1[i], CASES2[j]);                                \
      }                                                                    \
    }                                                                      \
  }                                                                        \
                                                                           \
  template <typename CaseType1, typename CaseType2>                        \
  void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType1& CASES1##_case, \
                                           const CaseType2& CASES2##_case)

// -------------------------------------------------------------------

// An input stream that is basically like an ArrayInputStream but sometimes
// returns empty buffers, just to throw us off.
class TestInputStream : public ZeroCopyInputStream {
 public:
  TestInputStream(const void* data, int size, int block_size)
    : array_stream_(data, size, block_size), counter_(0) {}
  ~TestInputStream() {}

  // implements ZeroCopyInputStream ----------------------------------
  bool Next(const void** data, int* size) {
    // We'll return empty buffers starting with the first buffer, and every
    // 3 and 5 buffers after that.
    if (counter_ % 3 == 0 || counter_ % 5 == 0) {
      *data = NULL;
      *size = 0;
      ++counter_;
      return true;
    } else {
      ++counter_;
      return array_stream_.Next(data, size);
    }
  }

  void BackUp(int count)  { return array_stream_.BackUp(count); }
  bool Skip(int count)    { return array_stream_.Skip(count);   }
  int64 ByteCount() const { return array_stream_.ByteCount();   }

 private:
  ArrayInputStream array_stream_;
  int counter_;
};

// -------------------------------------------------------------------

// An error collector which simply concatenates all its errors into a big
// block of text which can be checked.
class TestErrorCollector : public ErrorCollector {
 public:
  TestErrorCollector() {}
  ~TestErrorCollector() {}

  string text_;

  // implements ErrorCollector ---------------------------------------
  void AddError(int line, int column, const string& message) {
    strings::SubstituteAndAppend(&text_, "$0:$1: $2\n",
                                 line, column, message);
  }
};

// -------------------------------------------------------------------

// We test each operation over a variety of block sizes to insure that
// we test cases where reads cross buffer boundaries as well as cases
// where they don't.  This is sort of a brute-force approach to this,
// but it's easy to write and easy to understand.
const int kBlockSizes[] = {1, 2, 3, 5, 7, 13, 32, 1024};

class TokenizerTest : public testing::Test {
 protected:
  // For easy testing.
  uint64 ParseInteger(const string& text) {
    uint64 result;
    EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result));
    return result;
  }
};

// ===================================================================

// These tests causes gcc 3.3.5 (and earlier?) to give the cryptic error:
//   "sorry, unimplemented: `method_call_expr' not supported by dump_expr"
#if !defined(__GNUC__) || __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3)

// In each test case, the entire input text should parse as a single token
// of the given type.
struct SimpleTokenCase {
  string input;
  Tokenizer::TokenType type;
};

inline std::ostream& operator<<(std::ostream& out,
                                const SimpleTokenCase& test_case) {
  return out << CEscape(test_case.input);
}

SimpleTokenCase kSimpleTokenCases[] = {
  // Test identifiers.
  { "hello",       Tokenizer::TYPE_IDENTIFIER },

  // Test integers.
  { "123",         Tokenizer::TYPE_INTEGER },
  { "0xab6",       Tokenizer::TYPE_INTEGER },
  { "0XAB6",       Tokenizer::TYPE_INTEGER },
  { "0X1234567",   Tokenizer::TYPE_INTEGER },
  { "0x89abcdef",  Tokenizer::TYPE_INTEGER },
  { "0x89ABCDEF",  Tokenizer::TYPE_INTEGER },
  { "01234567",    Tokenizer::TYPE_INTEGER },

  // Test floats.
  { "123.45",      Tokenizer::TYPE_FLOAT },
  { "1.",          Tokenizer::TYPE_FLOAT },
  { "1e3",         Tokenizer::TYPE_FLOAT },
  { "1E3",         Tokenizer::TYPE_FLOAT },
  { "1e-3",        Tokenizer::TYPE_FLOAT },
  { "1e+3",        Tokenizer::TYPE_FLOAT },
  { "1.e3",        Tokenizer::TYPE_FLOAT },
  { "1.2e3",       Tokenizer::TYPE_FLOAT },
  { ".1",          Tokenizer::TYPE_FLOAT },
  { ".1e3",        Tokenizer::TYPE_FLOAT },
  { ".1e-3",       Tokenizer::TYPE_FLOAT },
  { ".1e+3",       Tokenizer::TYPE_FLOAT },

  // Test strings.
  { "'hello'",     Tokenizer::TYPE_STRING },
  { "\"foo\"",     Tokenizer::TYPE_STRING },
  { "'a\"b'",      Tokenizer::TYPE_STRING },
  { "\"a'b\"",     Tokenizer::TYPE_STRING },
  { "'a\\'b'",     Tokenizer::TYPE_STRING },
  { "\"a\\\"b\"",  Tokenizer::TYPE_STRING },
  { "'\\xf'",      Tokenizer::TYPE_STRING },
  { "'\\0'",       Tokenizer::TYPE_STRING },

  // Test symbols.
  { "+",           Tokenizer::TYPE_SYMBOL },
  { ".",           Tokenizer::TYPE_SYMBOL },
};

TEST_2D(TokenizerTest, SimpleTokens, kSimpleTokenCases, kBlockSizes) {
  // Set up the tokenizer.
  TestInputStream input(kSimpleTokenCases_case.input.data(),
                        kSimpleTokenCases_case.input.size(),
                        kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);

  // Before Next() is called, the initial token should always be TYPE_START.
  EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
  EXPECT_EQ("", tokenizer.current().text);
  EXPECT_EQ(0, tokenizer.current().line);
  EXPECT_EQ(0, tokenizer.current().column);
  EXPECT_EQ(0, tokenizer.current().end_column);

  // Parse the token.
  ASSERT_TRUE(tokenizer.Next());

  // Check that it has the right type.
  EXPECT_EQ(kSimpleTokenCases_case.type, tokenizer.current().type);
  // Check that it contains the complete input text.
  EXPECT_EQ(kSimpleTokenCases_case.input, tokenizer.current().text);
  // Check that it is located at the beginning of the input
  EXPECT_EQ(0, tokenizer.current().line);
  EXPECT_EQ(0, tokenizer.current().column);
  EXPECT_EQ(kSimpleTokenCases_case.input.size(),
            tokenizer.current().end_column);

  // There should be no more input.
  EXPECT_FALSE(tokenizer.Next());

  // After Next() returns false, the token should have type TYPE_END.
  EXPECT_EQ(Tokenizer::TYPE_END, tokenizer.current().type);
  EXPECT_EQ("", tokenizer.current().text);
  EXPECT_EQ(0, tokenizer.current().line);
  EXPECT_EQ(kSimpleTokenCases_case.input.size(), tokenizer.current().column);
  EXPECT_EQ(kSimpleTokenCases_case.input.size(),
            tokenizer.current().end_column);

  // There should be no errors.
  EXPECT_TRUE(error_collector.text_.empty());
}

TEST_1D(TokenizerTest, FloatSuffix, kBlockSizes) {
  // Test the "allow_f_after_float" option.

  // Set up the tokenizer.
  const char* text = "1f 2.5f 6e3f 7F";
  TestInputStream input(text, strlen(text), kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);
  tokenizer.set_allow_f_after_float(true);

  // Advance through tokens and check that they are parsed as expected.
  ASSERT_TRUE(tokenizer.Next());
  EXPECT_EQ(tokenizer.current().text, "1f");
  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
  ASSERT_TRUE(tokenizer.Next());
  EXPECT_EQ(tokenizer.current().text, "2.5f");
  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
  ASSERT_TRUE(tokenizer.Next());
  EXPECT_EQ(tokenizer.current().text, "6e3f");
  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);
  ASSERT_TRUE(tokenizer.Next());
  EXPECT_EQ(tokenizer.current().text, "7F");
  EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT);

  // There should be no more input.
  EXPECT_FALSE(tokenizer.Next());
  // There should be no errors.
  EXPECT_TRUE(error_collector.text_.empty());
}

#endif

// -------------------------------------------------------------------

// In each case, the input is parsed to produce a list of tokens.  The
// last token in "output" must have type TYPE_END.
struct MultiTokenCase {
  string input;
  Tokenizer::Token output[10];  // The compiler wants a constant array
                                // size for initialization to work.  There
                                // is no reason this can't be increased if
                                // needed.
};

inline std::ostream& operator<<(std::ostream& out,
                                const MultiTokenCase& test_case) {
  return out << CEscape(test_case.input);
}

MultiTokenCase kMultiTokenCases[] = {
  // Test empty input.
  { "", {
    { Tokenizer::TYPE_END       , ""     , 0,  0,  0 },
  }},

  // Test all token types at the same time.
  { "foo 1 1.2 + 'bar'", {
    { Tokenizer::TYPE_IDENTIFIER, "foo"  , 0,  0,  3 },
    { Tokenizer::TYPE_INTEGER   , "1"    , 0,  4,  5 },
    { Tokenizer::TYPE_FLOAT     , "1.2"  , 0,  6,  9 },
    { Tokenizer::TYPE_SYMBOL    , "+"    , 0, 10, 11 },
    { Tokenizer::TYPE_STRING    , "'bar'", 0, 12, 17 },
    { Tokenizer::TYPE_END       , ""     , 0, 17, 17 },
  }},

  // Test that consecutive symbols are parsed as separate tokens.
  { "!@+%", {
    { Tokenizer::TYPE_SYMBOL    , "!"    , 0, 0, 1 },
    { Tokenizer::TYPE_SYMBOL    , "@"    , 0, 1, 2 },
    { Tokenizer::TYPE_SYMBOL    , "+"    , 0, 2, 3 },
    { Tokenizer::TYPE_SYMBOL    , "%"    , 0, 3, 4 },
    { Tokenizer::TYPE_END       , ""     , 0, 4, 4 },
  }},

  // Test that newlines affect line numbers correctly.
  { "foo bar\nrab oof", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0, 3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 0,  4, 7 },
    { Tokenizer::TYPE_IDENTIFIER, "rab", 1,  0, 3 },
    { Tokenizer::TYPE_IDENTIFIER, "oof", 1,  4, 7 },
    { Tokenizer::TYPE_END       , ""   , 1,  7, 7 },
  }},

  // Test that tabs affect column numbers correctly.
  { "foo\tbar  \tbaz", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0,  3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 0,  8, 11 },
    { Tokenizer::TYPE_IDENTIFIER, "baz", 0, 16, 19 },
    { Tokenizer::TYPE_END       , ""   , 0, 19, 19 },
  }},

  // Test that tabs in string literals affect column numbers correctly.
  { "\"foo\tbar\" baz", {
    { Tokenizer::TYPE_STRING    , "\"foo\tbar\"", 0,  0, 12 },
    { Tokenizer::TYPE_IDENTIFIER, "baz"         , 0, 13, 16 },
    { Tokenizer::TYPE_END       , ""            , 0, 16, 16 },
  }},

  // Test that line comments are ignored.
  { "foo // This is a comment\n"
    "bar // This is another comment", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0,  3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 1,  0,  3 },
    { Tokenizer::TYPE_END       , ""   , 1, 30, 30 },
  }},

  // Test that block comments are ignored.
  { "foo /* This is a block comment */ bar", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0,  3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 34, 37 },
    { Tokenizer::TYPE_END       , ""   , 0, 37, 37 },
  }},

  // Test that sh-style comments are not ignored by default.
  { "foo # bar\n"
    "baz", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 },
    { Tokenizer::TYPE_SYMBOL    , "#"  , 0, 4, 5 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 6, 9 },
    { Tokenizer::TYPE_IDENTIFIER, "baz", 1, 0, 3 },
    { Tokenizer::TYPE_END       , ""   , 1, 3, 3 },
  }},

  // Test all whitespace chars
  { "foo\n\t\r\v\fbar", {
    { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0,  3 },
    { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11, 14 },
    { Tokenizer::TYPE_END       , ""   , 1, 14, 14 },
  }},
};

TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {
  // Set up the tokenizer.
  TestInputStream input(kMultiTokenCases_case.input.data(),
                        kMultiTokenCases_case.input.size(),
                        kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);

  // Before Next() is called, the initial token should always be TYPE_START.
  EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type);
  EXPECT_EQ("", tokenizer.current().text);
  EXPECT_EQ(0, tokenizer.current().line);
  EXPECT_EQ(0, tokenizer.current().column);
  EXPECT_EQ(0, tokenizer.current().end_column);

  // Loop through all expected tokens.
  int i = 0;
  Tokenizer::Token token;
  do {
    token = kMultiTokenCases_case.output[i++];

    SCOPED_TRACE(testing::Message() << "Token #" << i << ": " << token.text);

    Tokenizer::Token previous = tokenizer.current();

    // Next() should only return false when it hits the end token.
    if (token.type != Tokenizer::TYPE_END) {
      ASSERT_TRUE(tokenizer.Next());
    } else {
      ASSERT_FALSE(tokenizer.Next());
    }

    // Check that the previous token is set correctly.
    EXPECT_EQ(previous.type, tokenizer.previous().type);
    EXPECT_EQ(previous.text, tokenizer.previous().text);
    EXPECT_EQ(previous.line, tokenizer.previous().line);
    EXPECT_EQ(previous.column, tokenizer.previous().column);
    EXPECT_EQ(previous.end_column, tokenizer.previous().end_column);

    // Check that the token matches the expected one.
    EXPECT_EQ(token.type, tokenizer.current().type);
    EXPECT_EQ(token.text, tokenizer.current().text);
    EXPECT_EQ(token.line, tokenizer.current().line);
    EXPECT_EQ(token.column, tokenizer.current().column);
    EXPECT_EQ(token.end_column, tokenizer.current().end_column);

  } while (token.type != Tokenizer::TYPE_END);

  // There should be no errors.
  EXPECT_TRUE(error_collector.text_.empty());
}

// This test causes gcc 3.3.5 (and earlier?) to give the cryptic error:
//   "sorry, unimplemented: `method_call_expr' not supported by dump_expr"
#if !defined(__GNUC__) || __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3)

TEST_1D(TokenizerTest, ShCommentStyle, kBlockSizes) {
  // Test the "comment_style" option.

  const char* text = "foo # bar\n"
                     "baz // qux\n"
                     "corge /* grault */\n"
                     "garply";
  const char* const kTokens[] = {"foo",  // "# bar" is ignored
                                 "baz", "/", "/", "qux",
                                 "corge", "/", "*", "grault", "*", "/",
                                 "garply"};

  // Set up the tokenizer.
  TestInputStream input(text, strlen(text), kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);
  tokenizer.set_comment_style(Tokenizer::SH_COMMENT_STYLE);

  // Advance through tokens and check that they are parsed as expected.
  for (int i = 0; i < GOOGLE_ARRAYSIZE(kTokens); i++) {
    EXPECT_TRUE(tokenizer.Next());
    EXPECT_EQ(tokenizer.current().text, kTokens[i]);
  }

  // There should be no more input.
  EXPECT_FALSE(tokenizer.Next());
  // There should be no errors.
  EXPECT_TRUE(error_collector.text_.empty());
}

#endif

// -------------------------------------------------------------------

// In each case, the input is expected to have two tokens named "prev" and
// "next" with comments in between.
struct DocCommentCase {
  string input;

  const char* prev_trailing_comments;
  const char* detached_comments[10];
  const char* next_leading_comments;
};

inline std::ostream& operator<<(std::ostream& out,
                                const DocCommentCase& test_case) {
  return out << CEscape(test_case.input);
}

DocCommentCase kDocCommentCases[] = {
  {
    "prev next",

    "",
    {},
    ""
      },

        {
      "prev /* ignored */ next",

      "",
      {},
      ""
        },

          {
        "prev // trailing comment\n"
            "next",

            " trailing comment\n",
            {},
            ""
          },

            {
          "prev\n"
              "// leading comment\n"
              "// line 2\n"
              "next",

              "",
              {},
              " leading comment\n"
              " line 2\n"
            },

              {
            "prev\n"
                "// trailing comment\n"
                "// line 2\n"
                "\n"
                "next",

                " trailing comment\n"
                " line 2\n",
                {},
                ""
              },

                {
              "prev // trailing comment\n"
                  "// leading comment\n"
                  "// line 2\n"
                  "next",

                  " trailing comment\n",
                  {},
                  " leading comment\n"
                  " line 2\n"
                },

                  {
                "prev /* trailing block comment */\n"
                    "/* leading block comment\n"
                    " * line 2\n"
                    " * line 3 */"
                    "next",

                    " trailing block comment ",
                    {},
                    " leading block comment\n"
                    " line 2\n"
                    " line 3 "
                  },

                    {
                  "prev\n"
                      "/* trailing block comment\n"
                      " * line 2\n"
                      " * line 3\n"
                      " */\n"
                      "/* leading block comment\n"
                      " * line 2\n"
                      " * line 3 */"
                      "next",

                      " trailing block comment\n"
                      " line 2\n"
                      " line 3\n",
                      {},
                      " leading block comment\n"
                      " line 2\n"
                      " line 3 "
                    },

                      {
                    "prev\n"
                        "// trailing comment\n"
                        "\n"
                        "// detached comment\n"
                        "// line 2\n"
                        "\n"
                        "// second detached comment\n"
                        "/* third detached comment\n"
                        " * line 2 */\n"
                        "// leading comment\n"
                        "next",

                        " trailing comment\n",
                        {
                      " detached comment\n"
                          " line 2\n",
                          " second detached comment\n",
                          " third detached comment\n"
                          " line 2 "
                        },
                          " leading comment\n"
                        },

                          {
                        "prev /**/\n"
                            "\n"
                            "// detached comment\n"
                            "\n"
                            "// leading comment\n"
                            "next",

                            "",
                            {
                          " detached comment\n"
                            },
                              " leading comment\n"
                            },

                              {
                            "prev /**/\n"
                                "// leading comment\n"
                                "next",

                                "",
                                {},
                                " leading comment\n"
                              },
                              };

TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
  // Set up the tokenizer.
  TestInputStream input(kDocCommentCases_case.input.data(),
                        kDocCommentCases_case.input.size(),
                        kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);

  // Set up a second tokenizer where we'll pass all NULLs to NextWithComments().
  TestInputStream input2(kDocCommentCases_case.input.data(),
                        kDocCommentCases_case.input.size(),
                        kBlockSizes_case);
  Tokenizer tokenizer2(&input2, &error_collector);

  tokenizer.Next();
  tokenizer2.Next();

  EXPECT_EQ("prev", tokenizer.current().text);
  EXPECT_EQ("prev", tokenizer2.current().text);

  string prev_trailing_comments;
  std::vector<string> detached_comments;
  string next_leading_comments;
  tokenizer.NextWithComments(&prev_trailing_comments, &detached_comments,
                             &next_leading_comments);
  tokenizer2.NextWithComments(NULL, NULL, NULL);
  EXPECT_EQ("next", tokenizer.current().text);
  EXPECT_EQ("next", tokenizer2.current().text);

  EXPECT_EQ(kDocCommentCases_case.prev_trailing_comments,
            prev_trailing_comments);

  for (int i = 0; i < detached_comments.size(); i++) {
    ASSERT_LT(i, GOOGLE_ARRAYSIZE(kDocCommentCases));
    ASSERT_TRUE(kDocCommentCases_case.detached_comments[i] != NULL);
    EXPECT_EQ(kDocCommentCases_case.detached_comments[i],
              detached_comments[i]);
  }

  // Verify that we matched all the detached comments.
  EXPECT_EQ(NULL,
      kDocCommentCases_case.detached_comments[detached_comments.size()]);

  EXPECT_EQ(kDocCommentCases_case.next_leading_comments,
            next_leading_comments);
}

// -------------------------------------------------------------------

// Test parse helpers.  It's not really worth setting up a full data-driven
// test here.
TEST_F(TokenizerTest, ParseInteger) {
  EXPECT_EQ(0, ParseInteger("0"));
  EXPECT_EQ(123, ParseInteger("123"));
  EXPECT_EQ(0xabcdef12u, ParseInteger("0xabcdef12"));
  EXPECT_EQ(0xabcdef12u, ParseInteger("0xABCDEF12"));
  EXPECT_EQ(kuint64max, ParseInteger("0xFFFFFFFFFFFFFFFF"));
  EXPECT_EQ(01234567, ParseInteger("01234567"));
  EXPECT_EQ(0X123, ParseInteger("0X123"));

  // Test invalid integers that may still be tokenized as integers.
  EXPECT_EQ(0, ParseInteger("0x"));

  uint64 i;

  // Test invalid integers that will never be tokenized as integers.
  EXPECT_FALSE(Tokenizer::ParseInteger("zxy", kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("1.2", kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("08", kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("0xg", kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("-1", kuint64max, &i));

  // Test overflows.
  EXPECT_TRUE (Tokenizer::ParseInteger("0", 0, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("1", 0, &i));
  EXPECT_TRUE (Tokenizer::ParseInteger("1", 1, &i));
  EXPECT_TRUE (Tokenizer::ParseInteger("12345", 12345, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("12346", 12345, &i));
  EXPECT_TRUE (Tokenizer::ParseInteger("0xFFFFFFFFFFFFFFFF" , kuint64max, &i));
  EXPECT_FALSE(Tokenizer::ParseInteger("0x10000000000000000", kuint64max, &i));
}

TEST_F(TokenizerTest, ParseFloat) {
  EXPECT_DOUBLE_EQ(1    , Tokenizer::ParseFloat("1."));
  EXPECT_DOUBLE_EQ(1e3  , Tokenizer::ParseFloat("1e3"));
  EXPECT_DOUBLE_EQ(1e3  , Tokenizer::ParseFloat("1E3"));
  EXPECT_DOUBLE_EQ(1.5e3, Tokenizer::ParseFloat("1.5e3"));
  EXPECT_DOUBLE_EQ(.1   , Tokenizer::ParseFloat(".1"));
  EXPECT_DOUBLE_EQ(.25  , Tokenizer::ParseFloat(".25"));
  EXPECT_DOUBLE_EQ(.1e3 , Tokenizer::ParseFloat(".1e3"));
  EXPECT_DOUBLE_EQ(.25e3, Tokenizer::ParseFloat(".25e3"));
  EXPECT_DOUBLE_EQ(.1e+3, Tokenizer::ParseFloat(".1e+3"));
  EXPECT_DOUBLE_EQ(.1e-3, Tokenizer::ParseFloat(".1e-3"));
  EXPECT_DOUBLE_EQ(5    , Tokenizer::ParseFloat("5"));
  EXPECT_DOUBLE_EQ(6e-12, Tokenizer::ParseFloat("6e-12"));
  EXPECT_DOUBLE_EQ(1.2  , Tokenizer::ParseFloat("1.2"));
  EXPECT_DOUBLE_EQ(1.e2 , Tokenizer::ParseFloat("1.e2"));

  // Test invalid integers that may still be tokenized as integers.
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e"));
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e-"));
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.e"));

  // Test 'f' suffix.
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1f"));
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.0f"));
  EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1F"));

  // These should parse successfully even though they are out of range.
  // Overflows become infinity and underflows become zero.
  EXPECT_EQ(     0.0, Tokenizer::ParseFloat("1e-9999999999999999999999999999"));
  EXPECT_EQ(HUGE_VAL, Tokenizer::ParseFloat("1e+9999999999999999999999999999"));

#ifdef PROTOBUF_HAS_DEATH_TEST  // death tests do not work on Windows yet
  // Test invalid integers that will never be tokenized as integers.
  EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("zxy"),
    "passed text that could not have been tokenized as a float");
  EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("1-e0"),
    "passed text that could not have been tokenized as a float");
  EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("-1.0"),
    "passed text that could not have been tokenized as a float");
#endif  // PROTOBUF_HAS_DEATH_TEST
}

TEST_F(TokenizerTest, ParseString) {
  string output;
  Tokenizer::ParseString("'hello'", &output);
  EXPECT_EQ("hello", output);
  Tokenizer::ParseString("\"blah\\nblah2\"", &output);
  EXPECT_EQ("blah\nblah2", output);
  Tokenizer::ParseString("'\\1x\\1\\123\\739\\52\\334n\\3'", &output);
  EXPECT_EQ("\1x\1\123\739\52\334n\3", output);
  Tokenizer::ParseString("'\\x20\\x4'", &output);
  EXPECT_EQ("\x20\x4", output);

  // Test invalid strings that may still be tokenized as strings.
  Tokenizer::ParseString("\"\\a\\l\\v\\t", &output);  // \l is invalid
  EXPECT_EQ("\a?\v\t", output);
  Tokenizer::ParseString("'", &output);
  EXPECT_EQ("", output);
  Tokenizer::ParseString("'\\", &output);
  EXPECT_EQ("\\", output);

  // Experiment with Unicode escapes. Here are one-, two- and three-byte Unicode
  // characters.
  Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\U00024b62XX'", &output);
  EXPECT_EQ("$¢€𤭢XX", output);
  // Same thing encoded using UTF16.
  Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\ud852\\udf62XX'", &output);
  EXPECT_EQ("$¢€𤭢XX", output);
  // Here's some broken UTF16; there's a head surrogate with no tail surrogate.
  // We just output this as if it were UTF8; it's not a defined code point, but
  // it has a defined encoding.
  Tokenizer::ParseString("'\\ud852XX'", &output);
  EXPECT_EQ("\xed\xa1\x92XX", output);
  // Malformed escape: Demons may fly out of the nose.
  Tokenizer::ParseString("\\u0", &output);
  EXPECT_EQ("u0", output);

  // Test invalid strings that will never be tokenized as strings.
#ifdef PROTOBUF_HAS_DEATH_TEST  // death tests do not work on Windows yet
  EXPECT_DEBUG_DEATH(Tokenizer::ParseString("", &output),
    "passed text that could not have been tokenized as a string");
#endif  // PROTOBUF_HAS_DEATH_TEST
}

TEST_F(TokenizerTest, ParseStringAppend) {
  // Check that ParseString and ParseStringAppend differ.
  string output("stuff+");
  Tokenizer::ParseStringAppend("'hello'", &output);
  EXPECT_EQ("stuff+hello", output);
  Tokenizer::ParseString("'hello'", &output);
  EXPECT_EQ("hello", output);
}

// -------------------------------------------------------------------

// Each case parses some input text, ignoring the tokens produced, and
// checks that the error output matches what is expected.
struct ErrorCase {
  string input;
  bool recoverable;  // True if the tokenizer should be able to recover and
                     // parse more tokens after seeing this error.  Cases
                     // for which this is true must end with "foo" as
                     // the last token, which the test will check for.
  const char* errors;
};

inline std::ostream& operator<<(std::ostream& out, const ErrorCase& test_case) {
  return out << CEscape(test_case.input);
}

ErrorCase kErrorCases[] = {
  // String errors.
  { "'\\l' foo", true,
    "0:2: Invalid escape sequence in string literal.\n" },
  { "'\\X' foo", true,
    "0:2: Invalid escape sequence in string literal.\n" },
  { "'\\x' foo", true,
    "0:3: Expected hex digits for escape sequence.\n" },
  { "'foo", false,
    "0:4: Unexpected end of string.\n" },
  { "'bar\nfoo", true,
    "0:4: String literals cannot cross line boundaries.\n" },
  { "'\\u01' foo", true,
    "0:5: Expected four hex digits for \\u escape sequence.\n" },
  { "'\\u01' foo", true,
    "0:5: Expected four hex digits for \\u escape sequence.\n" },
  { "'\\uXYZ' foo", true,
    "0:3: Expected four hex digits for \\u escape sequence.\n" },

  // Integer errors.
  { "123foo", true,
    "0:3: Need space between number and identifier.\n" },

  // Hex/octal errors.
  { "0x foo", true,
    "0:2: \"0x\" must be followed by hex digits.\n" },
  { "0541823 foo", true,
    "0:4: Numbers starting with leading zero must be in octal.\n" },
  { "0x123z foo", true,
    "0:5: Need space between number and identifier.\n" },
  { "0x123.4 foo", true,
    "0:5: Hex and octal numbers must be integers.\n" },
  { "0123.4 foo", true,
    "0:4: Hex and octal numbers must be integers.\n" },

  // Float errors.
  { "1e foo", true,
    "0:2: \"e\" must be followed by exponent.\n" },
  { "1e- foo", true,
    "0:3: \"e\" must be followed by exponent.\n" },
  { "1.2.3 foo", true,
    "0:3: Already saw decimal point or exponent; can't have another one.\n" },
  { "1e2.3 foo", true,
    "0:3: Already saw decimal point or exponent; can't have another one.\n" },
  { "a.1 foo", true,
    "0:1: Need space between identifier and decimal point.\n" },
  // allow_f_after_float not enabled, so this should be an error.
  { "1.0f foo", true,
    "0:3: Need space between number and identifier.\n" },

  // Block comment errors.
  { "/*", false,
    "0:2: End-of-file inside block comment.\n"
    "0:0:   Comment started here.\n"},
  { "/*/*/ foo", true,
    "0:3: \"/*\" inside block comment.  Block comments cannot be nested.\n"},

  // Control characters.  Multiple consecutive control characters should only
  // produce one error.
  { "\b foo", true,
    "0:0: Invalid control characters encountered in text.\n" },
  { "\b\b foo", true,
    "0:0: Invalid control characters encountered in text.\n" },

  // Check that control characters at end of input don't result in an
  // infinite loop.
  { "\b", false,
    "0:0: Invalid control characters encountered in text.\n" },

  // Check recovery from '\0'.  We have to explicitly specify the length of
  // these strings because otherwise the string constructor will just call
  // strlen() which will see the first '\0' and think that is the end of the
  // string.
  { string("\0foo", 4), true,
    "0:0: Invalid control characters encountered in text.\n" },
  { string("\0\0foo", 5), true,
    "0:0: Invalid control characters encountered in text.\n" },

  // Check error from high order bits set
  { "\300foo", true,
    "0:0: Interpreting non ascii codepoint 192.\n" },
};

TEST_2D(TokenizerTest, Errors, kErrorCases, kBlockSizes) {
  // Set up the tokenizer.
  TestInputStream input(kErrorCases_case.input.data(),
                        kErrorCases_case.input.size(),
                        kBlockSizes_case);
  TestErrorCollector error_collector;
  Tokenizer tokenizer(&input, &error_collector);

  // Ignore all input, except remember if the last token was "foo".
  bool last_was_foo = false;
  while (tokenizer.Next()) {
    last_was_foo = tokenizer.current().text == "foo";
  }

  // Check that the errors match what was expected.
  EXPECT_EQ(kErrorCases_case.errors, error_collector.text_);

  // If the error was recoverable, make sure we saw "foo" after it.
  if (kErrorCases_case.recoverable) {
    EXPECT_TRUE(last_was_foo);
  }
}

// -------------------------------------------------------------------

TEST_1D(TokenizerTest, BackUpOnDestruction, kBlockSizes) {
  string text = "foo bar";
  TestInputStream input(text.data(), text.size(), kBlockSizes_case);

  // Create a tokenizer, read one token, then destroy it.
  {
    TestErrorCollector error_collector;
    Tokenizer tokenizer(&input, &error_collector);

    tokenizer.Next();
  }

  // Only "foo" should have been read.
  EXPECT_EQ(strlen("foo"), input.ByteCount());
}


}  // namespace
}  // namespace io
}  // namespace protobuf
}  // namespace google