Squashed 'third_party/protobuf/' content from commit e35e248
Change-Id: I6cbe123d09fe50fdcad0e51466665daeee7433c7
git-subtree-dir: third_party/protobuf
git-subtree-split: e35e24800fb8d694bdeea5fd63dc7d1b14d68723
diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py
new file mode 100755
index 0000000..8d25607
--- /dev/null
+++ b/python/google/protobuf/text_format.py
@@ -0,0 +1,1112 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc. All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Contains routines for printing protocol messages in text format.
+
+Simple usage example:
+
+ # Create a proto object and serialize it to a text proto string.
+ message = my_proto_pb2.MyMessage(foo='bar')
+ text_proto = text_format.MessageToString(message)
+
+ # Parse a text proto string.
+ message = text_format.Parse(text_proto, my_proto_pb2.MyMessage())
+"""
+
+__author__ = 'kenton@google.com (Kenton Varda)'
+
+import io
+import re
+
+import six
+
+if six.PY3:
+ long = int
+
+from google.protobuf.internal import type_checkers
+from google.protobuf import descriptor
+from google.protobuf import text_encoding
+
+__all__ = ['MessageToString', 'PrintMessage', 'PrintField',
+ 'PrintFieldValue', 'Merge']
+
+
+_INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),
+ type_checkers.Int32ValueChecker(),
+ type_checkers.Uint64ValueChecker(),
+ type_checkers.Int64ValueChecker())
+_FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?', re.IGNORECASE)
+_FLOAT_NAN = re.compile('nanf?', re.IGNORECASE)
+_FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT,
+ descriptor.FieldDescriptor.CPPTYPE_DOUBLE])
+_QUOTES = frozenset(("'", '"'))
+
+
+class Error(Exception):
+ """Top-level module error for text_format."""
+
+
+class ParseError(Error):
+ """Thrown in case of text parsing error."""
+
+
+class TextWriter(object):
+ def __init__(self, as_utf8):
+ if six.PY2:
+ self._writer = io.BytesIO()
+ else:
+ self._writer = io.StringIO()
+
+ def write(self, val):
+ if six.PY2:
+ if isinstance(val, six.text_type):
+ val = val.encode('utf-8')
+ return self._writer.write(val)
+
+ def close(self):
+ return self._writer.close()
+
+ def getvalue(self):
+ return self._writer.getvalue()
+
+
+def MessageToString(message, as_utf8=False, as_one_line=False,
+ pointy_brackets=False, use_index_order=False,
+ float_format=None):
+ """Convert protobuf message to text format.
+
+ Floating point values can be formatted compactly with 15 digits of
+ precision (which is the most that IEEE 754 "double" can guarantee)
+ using float_format='.15g'. To ensure that converting to text and back to a
+ proto will result in an identical value, float_format='.17g' should be used.
+
+ Args:
+ message: The protocol buffers message.
+ as_utf8: Produce text output in UTF8 format.
+ as_one_line: Don't introduce newlines between fields.
+ pointy_brackets: If True, use angle brackets instead of curly braces for
+ nesting.
+ use_index_order: If True, print fields of a proto message using the order
+ defined in source code instead of the field number. By default, use the
+ field number order.
+ float_format: If set, use this to specify floating point number formatting
+ (per the "Format Specification Mini-Language"); otherwise, str() is used.
+
+ Returns:
+ A string of the text formatted protocol buffer message.
+ """
+ out = TextWriter(as_utf8)
+ PrintMessage(message, out, as_utf8=as_utf8, as_one_line=as_one_line,
+ pointy_brackets=pointy_brackets,
+ use_index_order=use_index_order,
+ float_format=float_format)
+ result = out.getvalue()
+ out.close()
+ if as_one_line:
+ return result.rstrip()
+ return result
+
+
+def _IsMapEntry(field):
+ return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
+ field.message_type.has_options and
+ field.message_type.GetOptions().map_entry)
+
+
+def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False,
+ pointy_brackets=False, use_index_order=False,
+ float_format=None):
+ fields = message.ListFields()
+ if use_index_order:
+ fields.sort(key=lambda x: x[0].index)
+ for field, value in fields:
+ if _IsMapEntry(field):
+ for key in sorted(value):
+ # This is slow for maps with submessage entires because it copies the
+ # entire tree. Unfortunately this would take significant refactoring
+ # of this file to work around.
+ #
+ # TODO(haberman): refactor and optimize if this becomes an issue.
+ entry_submsg = field.message_type._concrete_class(
+ key=key, value=value[key])
+ PrintField(field, entry_submsg, out, indent, as_utf8, as_one_line,
+ pointy_brackets=pointy_brackets,
+ use_index_order=use_index_order, float_format=float_format)
+ elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
+ for element in value:
+ PrintField(field, element, out, indent, as_utf8, as_one_line,
+ pointy_brackets=pointy_brackets,
+ use_index_order=use_index_order,
+ float_format=float_format)
+ else:
+ PrintField(field, value, out, indent, as_utf8, as_one_line,
+ pointy_brackets=pointy_brackets,
+ use_index_order=use_index_order,
+ float_format=float_format)
+
+
+def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False,
+ pointy_brackets=False, use_index_order=False, float_format=None):
+ """Print a single field name/value pair. For repeated fields, the value
+ should be a single element.
+ """
+
+ out.write(' ' * indent)
+ if field.is_extension:
+ out.write('[')
+ if (field.containing_type.GetOptions().message_set_wire_format and
+ field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
+ field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL):
+ out.write(field.message_type.full_name)
+ else:
+ out.write(field.full_name)
+ out.write(']')
+ elif field.type == descriptor.FieldDescriptor.TYPE_GROUP:
+ # For groups, use the capitalized name.
+ out.write(field.message_type.name)
+ else:
+ out.write(field.name)
+
+ if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
+ # The colon is optional in this case, but our cross-language golden files
+ # don't include it.
+ out.write(': ')
+
+ PrintFieldValue(field, value, out, indent, as_utf8, as_one_line,
+ pointy_brackets=pointy_brackets,
+ use_index_order=use_index_order,
+ float_format=float_format)
+ if as_one_line:
+ out.write(' ')
+ else:
+ out.write('\n')
+
+
+def PrintFieldValue(field, value, out, indent=0, as_utf8=False,
+ as_one_line=False, pointy_brackets=False,
+ use_index_order=False,
+ float_format=None):
+ """Print a single field value (not including name). For repeated fields,
+ the value should be a single element."""
+
+ if pointy_brackets:
+ openb = '<'
+ closeb = '>'
+ else:
+ openb = '{'
+ closeb = '}'
+
+ if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
+ if as_one_line:
+ out.write(' %s ' % openb)
+ PrintMessage(value, out, indent, as_utf8, as_one_line,
+ pointy_brackets=pointy_brackets,
+ use_index_order=use_index_order,
+ float_format=float_format)
+ out.write(closeb)
+ else:
+ out.write(' %s\n' % openb)
+ PrintMessage(value, out, indent + 2, as_utf8, as_one_line,
+ pointy_brackets=pointy_brackets,
+ use_index_order=use_index_order,
+ float_format=float_format)
+ out.write(' ' * indent + closeb)
+ elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
+ enum_value = field.enum_type.values_by_number.get(value, None)
+ if enum_value is not None:
+ out.write(enum_value.name)
+ else:
+ out.write(str(value))
+ elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
+ out.write('\"')
+ if isinstance(value, six.text_type):
+ out_value = value.encode('utf-8')
+ else:
+ out_value = value
+ if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
+ # We need to escape non-UTF8 chars in TYPE_BYTES field.
+ out_as_utf8 = False
+ else:
+ out_as_utf8 = as_utf8
+ out.write(text_encoding.CEscape(out_value, out_as_utf8))
+ out.write('\"')
+ elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
+ if value:
+ out.write('true')
+ else:
+ out.write('false')
+ elif field.cpp_type in _FLOAT_TYPES and float_format is not None:
+ out.write('{1:{0}}'.format(float_format, value))
+ else:
+ out.write(str(value))
+
+
+def Parse(text, message, allow_unknown_extension=False):
+ """Parses an text representation of a protocol message into a message.
+
+ Args:
+ text: Message text representation.
+ message: A protocol buffer message to merge into.
+ allow_unknown_extension: if True, skip over missing extensions and keep
+ parsing
+
+ Returns:
+ The same message passed as argument.
+
+ Raises:
+ ParseError: On text parsing problems.
+ """
+ if not isinstance(text, str):
+ text = text.decode('utf-8')
+ return ParseLines(text.split('\n'), message, allow_unknown_extension)
+
+
+def Merge(text, message, allow_unknown_extension=False):
+ """Parses an text representation of a protocol message into a message.
+
+ Like Parse(), but allows repeated values for a non-repeated field, and uses
+ the last one.
+
+ Args:
+ text: Message text representation.
+ message: A protocol buffer message to merge into.
+ allow_unknown_extension: if True, skip over missing extensions and keep
+ parsing
+
+ Returns:
+ The same message passed as argument.
+
+ Raises:
+ ParseError: On text parsing problems.
+ """
+ return MergeLines(text.split('\n'), message, allow_unknown_extension)
+
+
+def ParseLines(lines, message, allow_unknown_extension=False):
+ """Parses an text representation of a protocol message into a message.
+
+ Args:
+ lines: An iterable of lines of a message's text representation.
+ message: A protocol buffer message to merge into.
+ allow_unknown_extension: if True, skip over missing extensions and keep
+ parsing
+
+ Returns:
+ The same message passed as argument.
+
+ Raises:
+ ParseError: On text parsing problems.
+ """
+ _ParseOrMerge(lines, message, False, allow_unknown_extension)
+ return message
+
+
+def MergeLines(lines, message, allow_unknown_extension=False):
+ """Parses an text representation of a protocol message into a message.
+
+ Args:
+ lines: An iterable of lines of a message's text representation.
+ message: A protocol buffer message to merge into.
+ allow_unknown_extension: if True, skip over missing extensions and keep
+ parsing
+
+ Returns:
+ The same message passed as argument.
+
+ Raises:
+ ParseError: On text parsing problems.
+ """
+ _ParseOrMerge(lines, message, True, allow_unknown_extension)
+ return message
+
+
+def _ParseOrMerge(lines,
+ message,
+ allow_multiple_scalars,
+ allow_unknown_extension=False):
+ """Converts an text representation of a protocol message into a message.
+
+ Args:
+ lines: Lines of a message's text representation.
+ message: A protocol buffer message to merge into.
+ allow_multiple_scalars: Determines if repeated values for a non-repeated
+ field are permitted, e.g., the string "foo: 1 foo: 2" for a
+ required/optional field named "foo".
+ allow_unknown_extension: if True, skip over missing extensions and keep
+ parsing
+
+ Raises:
+ ParseError: On text parsing problems.
+ """
+ tokenizer = _Tokenizer(lines)
+ while not tokenizer.AtEnd():
+ _MergeField(tokenizer, message, allow_multiple_scalars,
+ allow_unknown_extension)
+
+
+def _MergeField(tokenizer,
+ message,
+ allow_multiple_scalars,
+ allow_unknown_extension=False):
+ """Merges a single protocol message field into a message.
+
+ Args:
+ tokenizer: A tokenizer to parse the field name and values.
+ message: A protocol message to record the data.
+ allow_multiple_scalars: Determines if repeated values for a non-repeated
+ field are permitted, e.g., the string "foo: 1 foo: 2" for a
+ required/optional field named "foo".
+ allow_unknown_extension: if True, skip over missing extensions and keep
+ parsing
+
+ Raises:
+ ParseError: In case of text parsing problems.
+ """
+ message_descriptor = message.DESCRIPTOR
+ if (hasattr(message_descriptor, 'syntax') and
+ message_descriptor.syntax == 'proto3'):
+ # Proto3 doesn't represent presence so we can't test if multiple
+ # scalars have occurred. We have to allow them.
+ allow_multiple_scalars = True
+ if tokenizer.TryConsume('['):
+ name = [tokenizer.ConsumeIdentifier()]
+ while tokenizer.TryConsume('.'):
+ name.append(tokenizer.ConsumeIdentifier())
+ name = '.'.join(name)
+
+ if not message_descriptor.is_extendable:
+ raise tokenizer.ParseErrorPreviousToken(
+ 'Message type "%s" does not have extensions.' %
+ message_descriptor.full_name)
+ # pylint: disable=protected-access
+ field = message.Extensions._FindExtensionByName(name)
+ # pylint: enable=protected-access
+ if not field:
+ if allow_unknown_extension:
+ field = None
+ else:
+ raise tokenizer.ParseErrorPreviousToken(
+ 'Extension "%s" not registered.' % name)
+ elif message_descriptor != field.containing_type:
+ raise tokenizer.ParseErrorPreviousToken(
+ 'Extension "%s" does not extend message type "%s".' % (
+ name, message_descriptor.full_name))
+
+ tokenizer.Consume(']')
+
+ else:
+ name = tokenizer.ConsumeIdentifier()
+ field = message_descriptor.fields_by_name.get(name, None)
+
+ # Group names are expected to be capitalized as they appear in the
+ # .proto file, which actually matches their type names, not their field
+ # names.
+ if not field:
+ field = message_descriptor.fields_by_name.get(name.lower(), None)
+ if field and field.type != descriptor.FieldDescriptor.TYPE_GROUP:
+ field = None
+
+ if (field and field.type == descriptor.FieldDescriptor.TYPE_GROUP and
+ field.message_type.name != name):
+ field = None
+
+ if not field:
+ raise tokenizer.ParseErrorPreviousToken(
+ 'Message type "%s" has no field named "%s".' % (
+ message_descriptor.full_name, name))
+
+ if field and field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
+ is_map_entry = _IsMapEntry(field)
+ tokenizer.TryConsume(':')
+
+ if tokenizer.TryConsume('<'):
+ end_token = '>'
+ else:
+ tokenizer.Consume('{')
+ end_token = '}'
+
+ if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
+ if field.is_extension:
+ sub_message = message.Extensions[field].add()
+ elif is_map_entry:
+ sub_message = field.message_type._concrete_class()
+ else:
+ sub_message = getattr(message, field.name).add()
+ else:
+ if field.is_extension:
+ sub_message = message.Extensions[field]
+ else:
+ sub_message = getattr(message, field.name)
+ sub_message.SetInParent()
+
+ while not tokenizer.TryConsume(end_token):
+ if tokenizer.AtEnd():
+ raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token))
+ _MergeField(tokenizer, sub_message, allow_multiple_scalars,
+ allow_unknown_extension)
+
+ if is_map_entry:
+ value_cpptype = field.message_type.fields_by_name['value'].cpp_type
+ if value_cpptype == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
+ value = getattr(message, field.name)[sub_message.key]
+ value.MergeFrom(sub_message.value)
+ else:
+ getattr(message, field.name)[sub_message.key] = sub_message.value
+ elif field:
+ tokenizer.Consume(':')
+ if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and
+ tokenizer.TryConsume('[')):
+ # Short repeated format, e.g. "foo: [1, 2, 3]"
+ while True:
+ _MergeScalarField(tokenizer, message, field, allow_multiple_scalars)
+ if tokenizer.TryConsume(']'):
+ break
+ tokenizer.Consume(',')
+ else:
+ _MergeScalarField(tokenizer, message, field, allow_multiple_scalars)
+ else: # Proto field is unknown.
+ assert allow_unknown_extension
+ _SkipFieldContents(tokenizer)
+
+ # For historical reasons, fields may optionally be separated by commas or
+ # semicolons.
+ if not tokenizer.TryConsume(','):
+ tokenizer.TryConsume(';')
+
+
+def _SkipFieldContents(tokenizer):
+ """Skips over contents (value or message) of a field.
+
+ Args:
+ tokenizer: A tokenizer to parse the field name and values.
+ """
+ # Try to guess the type of this field.
+ # If this field is not a message, there should be a ":" between the
+ # field name and the field value and also the field value should not
+ # start with "{" or "<" which indicates the beginning of a message body.
+ # If there is no ":" or there is a "{" or "<" after ":", this field has
+ # to be a message or the input is ill-formed.
+ if tokenizer.TryConsume(':') and not tokenizer.LookingAt(
+ '{') and not tokenizer.LookingAt('<'):
+ _SkipFieldValue(tokenizer)
+ else:
+ _SkipFieldMessage(tokenizer)
+
+
+def _SkipField(tokenizer):
+ """Skips over a complete field (name and value/message).
+
+ Args:
+ tokenizer: A tokenizer to parse the field name and values.
+ """
+ if tokenizer.TryConsume('['):
+ # Consume extension name.
+ tokenizer.ConsumeIdentifier()
+ while tokenizer.TryConsume('.'):
+ tokenizer.ConsumeIdentifier()
+ tokenizer.Consume(']')
+ else:
+ tokenizer.ConsumeIdentifier()
+
+ _SkipFieldContents(tokenizer)
+
+ # For historical reasons, fields may optionally be separated by commas or
+ # semicolons.
+ if not tokenizer.TryConsume(','):
+ tokenizer.TryConsume(';')
+
+
+def _SkipFieldMessage(tokenizer):
+ """Skips over a field message.
+
+ Args:
+ tokenizer: A tokenizer to parse the field name and values.
+ """
+
+ if tokenizer.TryConsume('<'):
+ delimiter = '>'
+ else:
+ tokenizer.Consume('{')
+ delimiter = '}'
+
+ while not tokenizer.LookingAt('>') and not tokenizer.LookingAt('}'):
+ _SkipField(tokenizer)
+
+ tokenizer.Consume(delimiter)
+
+
+def _SkipFieldValue(tokenizer):
+ """Skips over a field value.
+
+ Args:
+ tokenizer: A tokenizer to parse the field name and values.
+
+ Raises:
+ ParseError: In case an invalid field value is found.
+ """
+ # String tokens can come in multiple adjacent string literals.
+ # If we can consume one, consume as many as we can.
+ if tokenizer.TryConsumeString():
+ while tokenizer.TryConsumeString():
+ pass
+ return
+
+ if (not tokenizer.TryConsumeIdentifier() and
+ not tokenizer.TryConsumeInt64() and
+ not tokenizer.TryConsumeUint64() and
+ not tokenizer.TryConsumeFloat()):
+ raise ParseError('Invalid field value: ' + tokenizer.token)
+
+
+def _MergeScalarField(tokenizer, message, field, allow_multiple_scalars):
+ """Merges a single protocol message scalar field into a message.
+
+ Args:
+ tokenizer: A tokenizer to parse the field value.
+ message: A protocol message to record the data.
+ field: The descriptor of the field to be merged.
+ allow_multiple_scalars: Determines if repeated values for a non-repeated
+ field are permitted, e.g., the string "foo: 1 foo: 2" for a
+ required/optional field named "foo".
+
+ Raises:
+ ParseError: In case of text parsing problems.
+ RuntimeError: On runtime errors.
+ """
+ value = None
+
+ if field.type in (descriptor.FieldDescriptor.TYPE_INT32,
+ descriptor.FieldDescriptor.TYPE_SINT32,
+ descriptor.FieldDescriptor.TYPE_SFIXED32):
+ value = tokenizer.ConsumeInt32()
+ elif field.type in (descriptor.FieldDescriptor.TYPE_INT64,
+ descriptor.FieldDescriptor.TYPE_SINT64,
+ descriptor.FieldDescriptor.TYPE_SFIXED64):
+ value = tokenizer.ConsumeInt64()
+ elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32,
+ descriptor.FieldDescriptor.TYPE_FIXED32):
+ value = tokenizer.ConsumeUint32()
+ elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64,
+ descriptor.FieldDescriptor.TYPE_FIXED64):
+ value = tokenizer.ConsumeUint64()
+ elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT,
+ descriptor.FieldDescriptor.TYPE_DOUBLE):
+ value = tokenizer.ConsumeFloat()
+ elif field.type == descriptor.FieldDescriptor.TYPE_BOOL:
+ value = tokenizer.ConsumeBool()
+ elif field.type == descriptor.FieldDescriptor.TYPE_STRING:
+ value = tokenizer.ConsumeString()
+ elif field.type == descriptor.FieldDescriptor.TYPE_BYTES:
+ value = tokenizer.ConsumeByteString()
+ elif field.type == descriptor.FieldDescriptor.TYPE_ENUM:
+ value = tokenizer.ConsumeEnum(field)
+ else:
+ raise RuntimeError('Unknown field type %d' % field.type)
+
+ if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
+ if field.is_extension:
+ message.Extensions[field].append(value)
+ else:
+ getattr(message, field.name).append(value)
+ else:
+ if field.is_extension:
+ if not allow_multiple_scalars and message.HasExtension(field):
+ raise tokenizer.ParseErrorPreviousToken(
+ 'Message type "%s" should not have multiple "%s" extensions.' %
+ (message.DESCRIPTOR.full_name, field.full_name))
+ else:
+ message.Extensions[field] = value
+ else:
+ if not allow_multiple_scalars and message.HasField(field.name):
+ raise tokenizer.ParseErrorPreviousToken(
+ 'Message type "%s" should not have multiple "%s" fields.' %
+ (message.DESCRIPTOR.full_name, field.name))
+ else:
+ setattr(message, field.name, value)
+
+
+class _Tokenizer(object):
+ """Protocol buffer text representation tokenizer.
+
+ This class handles the lower level string parsing by splitting it into
+ meaningful tokens.
+
+ It was directly ported from the Java protocol buffer API.
+ """
+
+ _WHITESPACE = re.compile('(\\s|(#.*$))+', re.MULTILINE)
+ _TOKEN = re.compile('|'.join([
+ r'[a-zA-Z_][0-9a-zA-Z_+-]*', # an identifier
+ r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*', # a number
+ ] + [ # quoted str for each quote mark
+ r'{qt}([^{qt}\n\\]|\\.)*({qt}|\\?$)'.format(qt=mark) for mark in _QUOTES
+ ]))
+
+ _IDENTIFIER = re.compile(r'\w+')
+
+ def __init__(self, lines):
+ self._position = 0
+ self._line = -1
+ self._column = 0
+ self._token_start = None
+ self.token = ''
+ self._lines = iter(lines)
+ self._current_line = ''
+ self._previous_line = 0
+ self._previous_column = 0
+ self._more_lines = True
+ self._SkipWhitespace()
+ self.NextToken()
+
+ def LookingAt(self, token):
+ return self.token == token
+
+ def AtEnd(self):
+ """Checks the end of the text was reached.
+
+ Returns:
+ True iff the end was reached.
+ """
+ return not self.token
+
+ def _PopLine(self):
+ while len(self._current_line) <= self._column:
+ try:
+ self._current_line = next(self._lines)
+ except StopIteration:
+ self._current_line = ''
+ self._more_lines = False
+ return
+ else:
+ self._line += 1
+ self._column = 0
+
+ def _SkipWhitespace(self):
+ while True:
+ self._PopLine()
+ match = self._WHITESPACE.match(self._current_line, self._column)
+ if not match:
+ break
+ length = len(match.group(0))
+ self._column += length
+
+ def TryConsume(self, token):
+ """Tries to consume a given piece of text.
+
+ Args:
+ token: Text to consume.
+
+ Returns:
+ True iff the text was consumed.
+ """
+ if self.token == token:
+ self.NextToken()
+ return True
+ return False
+
+ def Consume(self, token):
+ """Consumes a piece of text.
+
+ Args:
+ token: Text to consume.
+
+ Raises:
+ ParseError: If the text couldn't be consumed.
+ """
+ if not self.TryConsume(token):
+ raise self._ParseError('Expected "%s".' % token)
+
+ def TryConsumeIdentifier(self):
+ try:
+ self.ConsumeIdentifier()
+ return True
+ except ParseError:
+ return False
+
+ def ConsumeIdentifier(self):
+ """Consumes protocol message field identifier.
+
+ Returns:
+ Identifier string.
+
+ Raises:
+ ParseError: If an identifier couldn't be consumed.
+ """
+ result = self.token
+ if not self._IDENTIFIER.match(result):
+ raise self._ParseError('Expected identifier.')
+ self.NextToken()
+ return result
+
+ def ConsumeInt32(self):
+ """Consumes a signed 32bit integer number.
+
+ Returns:
+ The integer parsed.
+
+ Raises:
+ ParseError: If a signed 32bit integer couldn't be consumed.
+ """
+ try:
+ result = ParseInteger(self.token, is_signed=True, is_long=False)
+ except ValueError as e:
+ raise self._ParseError(str(e))
+ self.NextToken()
+ return result
+
+ def ConsumeUint32(self):
+ """Consumes an unsigned 32bit integer number.
+
+ Returns:
+ The integer parsed.
+
+ Raises:
+ ParseError: If an unsigned 32bit integer couldn't be consumed.
+ """
+ try:
+ result = ParseInteger(self.token, is_signed=False, is_long=False)
+ except ValueError as e:
+ raise self._ParseError(str(e))
+ self.NextToken()
+ return result
+
+ def TryConsumeInt64(self):
+ try:
+ self.ConsumeInt64()
+ return True
+ except ParseError:
+ return False
+
+ def ConsumeInt64(self):
+ """Consumes a signed 64bit integer number.
+
+ Returns:
+ The integer parsed.
+
+ Raises:
+ ParseError: If a signed 64bit integer couldn't be consumed.
+ """
+ try:
+ result = ParseInteger(self.token, is_signed=True, is_long=True)
+ except ValueError as e:
+ raise self._ParseError(str(e))
+ self.NextToken()
+ return result
+
+ def TryConsumeUint64(self):
+ try:
+ self.ConsumeUint64()
+ return True
+ except ParseError:
+ return False
+
+ def ConsumeUint64(self):
+ """Consumes an unsigned 64bit integer number.
+
+ Returns:
+ The integer parsed.
+
+ Raises:
+ ParseError: If an unsigned 64bit integer couldn't be consumed.
+ """
+ try:
+ result = ParseInteger(self.token, is_signed=False, is_long=True)
+ except ValueError as e:
+ raise self._ParseError(str(e))
+ self.NextToken()
+ return result
+
+ def TryConsumeFloat(self):
+ try:
+ self.ConsumeFloat()
+ return True
+ except ParseError:
+ return False
+
+ def ConsumeFloat(self):
+ """Consumes an floating point number.
+
+ Returns:
+ The number parsed.
+
+ Raises:
+ ParseError: If a floating point number couldn't be consumed.
+ """
+ try:
+ result = ParseFloat(self.token)
+ except ValueError as e:
+ raise self._ParseError(str(e))
+ self.NextToken()
+ return result
+
+ def ConsumeBool(self):
+ """Consumes a boolean value.
+
+ Returns:
+ The bool parsed.
+
+ Raises:
+ ParseError: If a boolean value couldn't be consumed.
+ """
+ try:
+ result = ParseBool(self.token)
+ except ValueError as e:
+ raise self._ParseError(str(e))
+ self.NextToken()
+ return result
+
+ def TryConsumeString(self):
+ try:
+ self.ConsumeString()
+ return True
+ except ParseError:
+ return False
+
+ def ConsumeString(self):
+ """Consumes a string value.
+
+ Returns:
+ The string parsed.
+
+ Raises:
+ ParseError: If a string value couldn't be consumed.
+ """
+ the_bytes = self.ConsumeByteString()
+ try:
+ return six.text_type(the_bytes, 'utf-8')
+ except UnicodeDecodeError as e:
+ raise self._StringParseError(e)
+
+ def ConsumeByteString(self):
+ """Consumes a byte array value.
+
+ Returns:
+ The array parsed (as a string).
+
+ Raises:
+ ParseError: If a byte array value couldn't be consumed.
+ """
+ the_list = [self._ConsumeSingleByteString()]
+ while self.token and self.token[0] in _QUOTES:
+ the_list.append(self._ConsumeSingleByteString())
+ return b''.join(the_list)
+
+ def _ConsumeSingleByteString(self):
+ """Consume one token of a string literal.
+
+ String literals (whether bytes or text) can come in multiple adjacent
+ tokens which are automatically concatenated, like in C or Python. This
+ method only consumes one token.
+
+ Returns:
+ The token parsed.
+ Raises:
+ ParseError: When the wrong format data is found.
+ """
+ text = self.token
+ if len(text) < 1 or text[0] not in _QUOTES:
+ raise self._ParseError('Expected string but found: %r' % (text,))
+
+ if len(text) < 2 or text[-1] != text[0]:
+ raise self._ParseError('String missing ending quote: %r' % (text,))
+
+ try:
+ result = text_encoding.CUnescape(text[1:-1])
+ except ValueError as e:
+ raise self._ParseError(str(e))
+ self.NextToken()
+ return result
+
+ def ConsumeEnum(self, field):
+ try:
+ result = ParseEnum(field, self.token)
+ except ValueError as e:
+ raise self._ParseError(str(e))
+ self.NextToken()
+ return result
+
+ def ParseErrorPreviousToken(self, message):
+ """Creates and *returns* a ParseError for the previously read token.
+
+ Args:
+ message: A message to set for the exception.
+
+ Returns:
+ A ParseError instance.
+ """
+ return ParseError('%d:%d : %s' % (
+ self._previous_line + 1, self._previous_column + 1, message))
+
+ def _ParseError(self, message):
+ """Creates and *returns* a ParseError for the current token."""
+ return ParseError('%d:%d : %s' % (
+ self._line + 1, self._column + 1, message))
+
+ def _StringParseError(self, e):
+ return self._ParseError('Couldn\'t parse string: ' + str(e))
+
+ def NextToken(self):
+ """Reads the next meaningful token."""
+ self._previous_line = self._line
+ self._previous_column = self._column
+
+ self._column += len(self.token)
+ self._SkipWhitespace()
+
+ if not self._more_lines:
+ self.token = ''
+ return
+
+ match = self._TOKEN.match(self._current_line, self._column)
+ if match:
+ token = match.group(0)
+ self.token = token
+ else:
+ self.token = self._current_line[self._column]
+
+
+def ParseInteger(text, is_signed=False, is_long=False):
+ """Parses an integer.
+
+ Args:
+ text: The text to parse.
+ is_signed: True if a signed integer must be parsed.
+ is_long: True if a long integer must be parsed.
+
+ Returns:
+ The integer value.
+
+ Raises:
+ ValueError: Thrown Iff the text is not a valid integer.
+ """
+ # Do the actual parsing. Exception handling is propagated to caller.
+ try:
+ # We force 32-bit values to int and 64-bit values to long to make
+ # alternate implementations where the distinction is more significant
+ # (e.g. the C++ implementation) simpler.
+ if is_long:
+ result = long(text, 0)
+ else:
+ result = int(text, 0)
+ except ValueError:
+ raise ValueError('Couldn\'t parse integer: %s' % text)
+
+ # Check if the integer is sane. Exceptions handled by callers.
+ checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)]
+ checker.CheckValue(result)
+ return result
+
+
+def ParseFloat(text):
+ """Parse a floating point number.
+
+ Args:
+ text: Text to parse.
+
+ Returns:
+ The number parsed.
+
+ Raises:
+ ValueError: If a floating point number couldn't be parsed.
+ """
+ try:
+ # Assume Python compatible syntax.
+ return float(text)
+ except ValueError:
+ # Check alternative spellings.
+ if _FLOAT_INFINITY.match(text):
+ if text[0] == '-':
+ return float('-inf')
+ else:
+ return float('inf')
+ elif _FLOAT_NAN.match(text):
+ return float('nan')
+ else:
+ # assume '1.0f' format
+ try:
+ return float(text.rstrip('f'))
+ except ValueError:
+ raise ValueError('Couldn\'t parse float: %s' % text)
+
+
+def ParseBool(text):
+ """Parse a boolean value.
+
+ Args:
+ text: Text to parse.
+
+ Returns:
+ Boolean values parsed
+
+ Raises:
+ ValueError: If text is not a valid boolean.
+ """
+ if text in ('true', 't', '1'):
+ return True
+ elif text in ('false', 'f', '0'):
+ return False
+ else:
+ raise ValueError('Expected "true" or "false".')
+
+
+def ParseEnum(field, value):
+ """Parse an enum value.
+
+ The value can be specified by a number (the enum value), or by
+ a string literal (the enum name).
+
+ Args:
+ field: Enum field descriptor.
+ value: String value.
+
+ Returns:
+ Enum value number.
+
+ Raises:
+ ValueError: If the enum value could not be parsed.
+ """
+ enum_descriptor = field.enum_type
+ try:
+ number = int(value, 0)
+ except ValueError:
+ # Identifier.
+ enum_value = enum_descriptor.values_by_name.get(value, None)
+ if enum_value is None:
+ raise ValueError(
+ 'Enum type "%s" has no value named %s.' % (
+ enum_descriptor.full_name, value))
+ else:
+ # Numeric value.
+ enum_value = enum_descriptor.values_by_number.get(number, None)
+ if enum_value is None:
+ raise ValueError(
+ 'Enum type "%s" has no value with number %d.' % (
+ enum_descriptor.full_name, number))
+ return enum_value.number