# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc.  All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Code for encoding protocol message primitives.

Contains the logic for encoding every logical protocol field type
into one of the 5 physical wire types.

This code is designed to push the Python interpreter's performance to the
limits.

The basic idea is that at startup time, for every field (i.e. every
FieldDescriptor) we construct two functions:  a "sizer" and an "encoder".  The
sizer takes a value of this field's type and computes its byte size.  The
encoder takes a writer function and a value.  It encodes the value into byte
strings and invokes the writer function to write those strings.  Typically the
writer function is the write() method of a cStringIO.

We try to do as much work as possible when constructing the writer and the
sizer rather than when calling them.  In particular:
* We copy any needed global functions to local variables, so that we do not need
  to do costly global table lookups at runtime.
* Similarly, we try to do any attribute lookups at startup time if possible.
* Every field's tag is encoded to bytes at startup, since it can't change at
  runtime.
* Whatever component of the field size we can compute at startup, we do.
* We *avoid* sharing code if doing so would make the code slower and not sharing
  does not burden us too much.  For example, encoders for repeated fields do
  not just call the encoders for singular fields in a loop because this would
  add an extra function call overhead for every loop iteration; instead, we
  manually inline the single-value encoder into the loop.
* If a Python function lacks a return statement, Python actually generates
  instructions to pop the result of the last statement off the stack, push
  None onto the stack, and then return that.  If we really don't care what
  value is returned, then we can save two instructions by returning the
  result of the last statement.  It looks funny but it helps.
* We assume that type and bounds checking has happened at a higher level.
"""

__author__ = 'kenton@google.com (Kenton Varda)'

import struct
from google.protobuf.internal import wire_format

# This will overflow and thus become IEEE-754 "infinity".  We would use
# "float('inf')" but it doesn't work on Windows pre-Python-2.6.
_POS_INF = 1e10000
_NEG_INF = -_POS_INF

def _VarintSize(value):
  """Compute the size of a varint value."""
  if value <= 0x7f: return 1
  if value <= 0x3fff: return 2
  if value <= 0x1fffff: return 3
  if value <= 0xfffffff: return 4
  if value <= 0x7ffffffff: return 5
  if value <= 0x3ffffffffff: return 6
  if value <= 0x1ffffffffffff: return 7
  if value <= 0xffffffffffffff: return 8
  if value <= 0x7fffffffffffffff: return 9
  return 10

def _SignedVarintSize(value):
  """Compute the size of a signed varint value."""
  if value < 0: return 10
  if value <= 0x7f: return 1
  if value <= 0x3fff: return 2
  if value <= 0x1fffff: return 3
  if value <= 0xfffffff: return 4
  if value <= 0x7ffffffff: return 5
  if value <= 0x3ffffffffff: return 6
  if value <= 0x1ffffffffffff: return 7
  if value <= 0xffffffffffffff: return 8
  if value <= 0x7fffffffffffffff: return 9
  return 10

def _TagSize(field_number):
  """Returns the number of bytes required to serialize a tag with this field
  number."""
  # Just pass in type 0, since the type won't affect the tag+type size.
  return _VarintSize(wire_format.PackTag(field_number, 0))

# --------------------------------------------------------------------
# In this section we define some generic sizers.  Each of these functions
# takes parameters specific to a particular field type, e.g. int32 or fixed64.
# It returns another function which in turn takes parameters specific to a
# particular field, e.g. the field number and whether it is repeated or packed.
# Look at the next section to see how these are used.

def _SimpleSizer(compute_value_size):
  """A sizer which uses the function compute_value_size to compute the size of
  each value.  Typically compute_value_size is _VarintSize."""

  def SpecificSizer(field_number, is_repeated, is_packed):
    tag_size = _TagSize(field_number)
    if is_packed:
      local_VarintSize = _VarintSize
      def PackedFieldSize(value):
        result = 0
        for element in value:
          result += compute_value_size(element)
        return result + local_VarintSize(result) + tag_size
      return PackedFieldSize
    elif is_repeated:
      def RepeatedFieldSize(value):
        result = tag_size * len(value)
        for element in value:
          result += compute_value_size(element)
        return result
      return RepeatedFieldSize
    else:
      def FieldSize(value):
        return tag_size + compute_value_size(value)
      return FieldSize

  return SpecificSizer

def _ModifiedSizer(compute_value_size, modify_value):
  """Like SimpleSizer, but modify_value is invoked on each value before it is
  passed to compute_value_size.  modify_value is typically ZigZagEncode."""

  def SpecificSizer(field_number, is_repeated, is_packed):
    tag_size = _TagSize(field_number)
    if is_packed:
      local_VarintSize = _VarintSize
      def PackedFieldSize(value):
        result = 0
        for element in value:
          result += compute_value_size(modify_value(element))
        return result + local_VarintSize(result) + tag_size
      return PackedFieldSize
    elif is_repeated:
      def RepeatedFieldSize(value):
        result = tag_size * len(value)
        for element in value:
          result += compute_value_size(modify_value(element))
        return result
      return RepeatedFieldSize
    else:
      def FieldSize(value):
        return tag_size + compute_value_size(modify_value(value))
      return FieldSize

  return SpecificSizer

def _FixedSizer(value_size):
  """Like _SimpleSizer except for a fixed-size field.  The input is the size
  of one value."""

  def SpecificSizer(field_number, is_repeated, is_packed):
    tag_size = _TagSize(field_number)
    if is_packed:
      local_VarintSize = _VarintSize
      def PackedFieldSize(value):
        result = len(value) * value_size
        return result + local_VarintSize(result) + tag_size
      return PackedFieldSize
    elif is_repeated:
      element_size = value_size + tag_size
      def RepeatedFieldSize(value):
        return len(value) * element_size
      return RepeatedFieldSize
    else:
      field_size = value_size + tag_size
      def FieldSize(value):
        return field_size
      return FieldSize

  return SpecificSizer

# ====================================================================
# Here we declare a sizer constructor for each field type.  Each "sizer
# constructor" is a function that takes (field_number, is_repeated, is_packed)
# as parameters and returns a sizer, which in turn takes a field value as
# a parameter and returns its encoded size.

Int32Sizer = Int64Sizer = EnumSizer = _SimpleSizer(_SignedVarintSize)

UInt32Sizer = UInt64Sizer = _SimpleSizer(_VarintSize)

SInt32Sizer = SInt64Sizer = _ModifiedSizer(
    _SignedVarintSize, wire_format.ZigZagEncode)

Fixed32Sizer = SFixed32Sizer = FloatSizer  = _FixedSizer(4)
Fixed64Sizer = SFixed64Sizer = DoubleSizer = _FixedSizer(8)

BoolSizer = _FixedSizer(1)

def StringSizer(field_number, is_repeated, is_packed):
  """Returns a sizer for a string field."""

  tag_size = _TagSize(field_number)
  local_VarintSize = _VarintSize
  local_len = len
  assert not is_packed
  if is_repeated:
    def RepeatedFieldSize(value):
      result = tag_size * len(value)
      for element in value:
        l = local_len(element.encode('utf-8'))
        result += local_VarintSize(l) + l
      return result
    return RepeatedFieldSize
  else:
    def FieldSize(value):
      l = local_len(value.encode('utf-8'))
      return tag_size + local_VarintSize(l) + l
    return FieldSize

def BytesSizer(field_number, is_repeated, is_packed):
  """Returns a sizer for a bytes field."""

  tag_size = _TagSize(field_number)
  local_VarintSize = _VarintSize
  local_len = len
  assert not is_packed
  if is_repeated:
    def RepeatedFieldSize(value):
      result = tag_size * len(value)
      for element in value:
        l = local_len(element)
        result += local_VarintSize(l) + l
      return result
    return RepeatedFieldSize
  else:
    def FieldSize(value):
      l = local_len(value)
      return tag_size + local_VarintSize(l) + l
    return FieldSize

def GroupSizer(field_number, is_repeated, is_packed):
  """Returns a sizer for a group field."""

  tag_size = _TagSize(field_number) * 2
  assert not is_packed
  if is_repeated:
    def RepeatedFieldSize(value):
      result = tag_size * len(value)
      for element in value:
        result += element.ByteSize()
      return result
    return RepeatedFieldSize
  else:
    def FieldSize(value):
      return tag_size + value.ByteSize()
    return FieldSize

def MessageSizer(field_number, is_repeated, is_packed):
  """Returns a sizer for a message field."""

  tag_size = _TagSize(field_number)
  local_VarintSize = _VarintSize
  assert not is_packed
  if is_repeated:
    def RepeatedFieldSize(value):
      result = tag_size * len(value)
      for element in value:
        l = element.ByteSize()
        result += local_VarintSize(l) + l
      return result
    return RepeatedFieldSize
  else:
    def FieldSize(value):
      l = value.ByteSize()
      return tag_size + local_VarintSize(l) + l
    return FieldSize

# --------------------------------------------------------------------
# MessageSet is special.

def MessageSetItemSizer(field_number):
  """Returns a sizer for extensions of MessageSet.

  The message set message looks like this:
    message MessageSet {
      repeated group Item = 1 {
        required int32 type_id = 2;
        required string message = 3;
      }
    }
  """
  static_size = (_TagSize(1) * 2 + _TagSize(2) + _VarintSize(field_number) +
                 _TagSize(3))
  local_VarintSize = _VarintSize

  def FieldSize(value):
    l = value.ByteSize()
    return static_size + local_VarintSize(l) + l

  return FieldSize

# ====================================================================
# Encoders!

def _VarintEncoder():
  """Return an encoder for a basic varint value (does not include tag)."""

  local_chr = chr
  def EncodeVarint(write, value):
    bits = value & 0x7f
    value >>= 7
    while value:
      write(local_chr(0x80|bits))
      bits = value & 0x7f
      value >>= 7
    return write(local_chr(bits))

  return EncodeVarint

def _SignedVarintEncoder():
  """Return an encoder for a basic signed varint value (does not include
  tag)."""

  local_chr = chr
  def EncodeSignedVarint(write, value):
    if value < 0:
      value += (1 << 64)
    bits = value & 0x7f
    value >>= 7
    while value:
      write(local_chr(0x80|bits))
      bits = value & 0x7f
      value >>= 7
    return write(local_chr(bits))

  return EncodeSignedVarint

_EncodeVarint = _VarintEncoder()
_EncodeSignedVarint = _SignedVarintEncoder()

def _VarintBytes(value):
  """Encode the given integer as a varint and return the bytes.  This is only
  called at startup time so it doesn't need to be fast."""

  pieces = []
  _EncodeVarint(pieces.append, value)
  return "".join(pieces)

def TagBytes(field_number, wire_type):
  """Encode the given tag and return the bytes.  Only called at startup."""

  return _VarintBytes(wire_format.PackTag(field_number, wire_type))

# --------------------------------------------------------------------
# As with sizers (see above), we have a number of common encoder
# implementations.

def _SimpleEncoder(wire_type, encode_value, compute_value_size):
  """Return a constructor for an encoder for fields of a particular type.

  Args:
      wire_type:  The field's wire type, for encoding tags.
      encode_value:  A function which encodes an individual value, e.g.
        _EncodeVarint().
      compute_value_size:  A function which computes the size of an individual
        value, e.g. _VarintSize().
  """

  def SpecificEncoder(field_number, is_repeated, is_packed):
    if is_packed:
      tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
      local_EncodeVarint = _EncodeVarint
      def EncodePackedField(write, value):
        write(tag_bytes)
        size = 0
        for element in value:
          size += compute_value_size(element)
        local_EncodeVarint(write, size)
        for element in value:
          encode_value(write, element)
      return EncodePackedField
    elif is_repeated:
      tag_bytes = TagBytes(field_number, wire_type)
      def EncodeRepeatedField(write, value):
        for element in value:
          write(tag_bytes)
          encode_value(write, element)
      return EncodeRepeatedField
    else:
      tag_bytes = TagBytes(field_number, wire_type)
      def EncodeField(write, value):
        write(tag_bytes)
        return encode_value(write, value)
      return EncodeField

  return SpecificEncoder

def _ModifiedEncoder(wire_type, encode_value, compute_value_size, modify_value):
  """Like SimpleEncoder but additionally invokes modify_value on every value
  before passing it to encode_value.  Usually modify_value is ZigZagEncode."""

  def SpecificEncoder(field_number, is_repeated, is_packed):
    if is_packed:
      tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
      local_EncodeVarint = _EncodeVarint
      def EncodePackedField(write, value):
        write(tag_bytes)
        size = 0
        for element in value:
          size += compute_value_size(modify_value(element))
        local_EncodeVarint(write, size)
        for element in value:
          encode_value(write, modify_value(element))
      return EncodePackedField
    elif is_repeated:
      tag_bytes = TagBytes(field_number, wire_type)
      def EncodeRepeatedField(write, value):
        for element in value:
          write(tag_bytes)
          encode_value(write, modify_value(element))
      return EncodeRepeatedField
    else:
      tag_bytes = TagBytes(field_number, wire_type)
      def EncodeField(write, value):
        write(tag_bytes)
        return encode_value(write, modify_value(value))
      return EncodeField

  return SpecificEncoder

def _StructPackEncoder(wire_type, format):
  """Return a constructor for an encoder for a fixed-width field.

  Args:
      wire_type:  The field's wire type, for encoding tags.
      format:  The format string to pass to struct.pack().
  """

  value_size = struct.calcsize(format)

  def SpecificEncoder(field_number, is_repeated, is_packed):
    local_struct_pack = struct.pack
    if is_packed:
      tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
      local_EncodeVarint = _EncodeVarint
      def EncodePackedField(write, value):
        write(tag_bytes)
        local_EncodeVarint(write, len(value) * value_size)
        for element in value:
          write(local_struct_pack(format, element))
      return EncodePackedField
    elif is_repeated:
      tag_bytes = TagBytes(field_number, wire_type)
      def EncodeRepeatedField(write, value):
        for element in value:
          write(tag_bytes)
          write(local_struct_pack(format, element))
      return EncodeRepeatedField
    else:
      tag_bytes = TagBytes(field_number, wire_type)
      def EncodeField(write, value):
        write(tag_bytes)
        return write(local_struct_pack(format, value))
      return EncodeField

  return SpecificEncoder

def _FloatingPointEncoder(wire_type, format):
  """Return a constructor for an encoder for float fields.

  This is like StructPackEncoder, but catches errors that may be due to
  passing non-finite floating-point values to struct.pack, and makes a
  second attempt to encode those values.

  Args:
      wire_type:  The field's wire type, for encoding tags.
      format:  The format string to pass to struct.pack().
  """

  value_size = struct.calcsize(format)
  if value_size == 4:
    def EncodeNonFiniteOrRaise(write, value):
      # Remember that the serialized form uses little-endian byte order.
      if value == _POS_INF:
        write('\x00\x00\x80\x7F')
      elif value == _NEG_INF:
        write('\x00\x00\x80\xFF')
      elif value != value:           # NaN
        write('\x00\x00\xC0\x7F')
      else:
        raise
  elif value_size == 8:
    def EncodeNonFiniteOrRaise(write, value):
      if value == _POS_INF:
        write('\x00\x00\x00\x00\x00\x00\xF0\x7F')
      elif value == _NEG_INF:
        write('\x00\x00\x00\x00\x00\x00\xF0\xFF')
      elif value != value:                         # NaN
        write('\x00\x00\x00\x00\x00\x00\xF8\x7F')
      else:
        raise
  else:
    raise ValueError('Can\'t encode floating-point values that are '
                     '%d bytes long (only 4 or 8)' % value_size)

  def SpecificEncoder(field_number, is_repeated, is_packed):
    local_struct_pack = struct.pack
    if is_packed:
      tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
      local_EncodeVarint = _EncodeVarint
      def EncodePackedField(write, value):
        write(tag_bytes)
        local_EncodeVarint(write, len(value) * value_size)
        for element in value:
          # This try/except block is going to be faster than any code that
          # we could write to check whether element is finite.
          try:
            write(local_struct_pack(format, element))
          except SystemError:
            EncodeNonFiniteOrRaise(write, element)
      return EncodePackedField
    elif is_repeated:
      tag_bytes = TagBytes(field_number, wire_type)
      def EncodeRepeatedField(write, value):
        for element in value:
          write(tag_bytes)
          try:
            write(local_struct_pack(format, element))
          except SystemError:
            EncodeNonFiniteOrRaise(write, element)
      return EncodeRepeatedField
    else:
      tag_bytes = TagBytes(field_number, wire_type)
      def EncodeField(write, value):
        write(tag_bytes)
        try:
          write(local_struct_pack(format, value))
        except SystemError:
          EncodeNonFiniteOrRaise(write, value)
      return EncodeField

  return SpecificEncoder

# ====================================================================
# Here we declare an encoder constructor for each field type.  These work
# very similarly to sizer constructors, described earlier.

Int32Encoder = Int64Encoder = EnumEncoder = _SimpleEncoder(
    wire_format.WIRETYPE_VARINT, _EncodeSignedVarint, _SignedVarintSize)

UInt32Encoder = UInt64Encoder = _SimpleEncoder(
    wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize)

SInt32Encoder = SInt64Encoder = _ModifiedEncoder(
    wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize,
    wire_format.ZigZagEncode)

# Note that Python conveniently guarantees that when using the '<' prefix on
# formats, they will also have the same size across all platforms (as opposed
# to without the prefix, where their sizes depend on the C compiler's basic
# type sizes).
Fixed32Encoder  = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<I')
Fixed64Encoder  = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<Q')
SFixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<i')
SFixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<q')
FloatEncoder    = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED32, '<f')
DoubleEncoder   = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED64, '<d')

def BoolEncoder(field_number, is_repeated, is_packed):
  """Returns an encoder for a boolean field."""

  false_byte = chr(0)
  true_byte = chr(1)
  if is_packed:
    tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
    local_EncodeVarint = _EncodeVarint
    def EncodePackedField(write, value):
      write(tag_bytes)
      local_EncodeVarint(write, len(value))
      for element in value:
        if element:
          write(true_byte)
        else:
          write(false_byte)
    return EncodePackedField
  elif is_repeated:
    tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
    def EncodeRepeatedField(write, value):
      for element in value:
        write(tag_bytes)
        if element:
          write(true_byte)
        else:
          write(false_byte)
    return EncodeRepeatedField
  else:
    tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
    def EncodeField(write, value):
      write(tag_bytes)
      if value:
        return write(true_byte)
      return write(false_byte)
    return EncodeField

def StringEncoder(field_number, is_repeated, is_packed):
  """Returns an encoder for a string field."""

  tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
  local_EncodeVarint = _EncodeVarint
  local_len = len
  assert not is_packed
  if is_repeated:
    def EncodeRepeatedField(write, value):
      for element in value:
        encoded = element.encode('utf-8')
        write(tag)
        local_EncodeVarint(write, local_len(encoded))
        write(encoded)
    return EncodeRepeatedField
  else:
    def EncodeField(write, value):
      encoded = value.encode('utf-8')
      write(tag)
      local_EncodeVarint(write, local_len(encoded))
      return write(encoded)
    return EncodeField

def BytesEncoder(field_number, is_repeated, is_packed):
  """Returns an encoder for a bytes field."""

  tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
  local_EncodeVarint = _EncodeVarint
  local_len = len
  assert not is_packed
  if is_repeated:
    def EncodeRepeatedField(write, value):
      for element in value:
        write(tag)
        local_EncodeVarint(write, local_len(element))
        write(element)
    return EncodeRepeatedField
  else:
    def EncodeField(write, value):
      write(tag)
      local_EncodeVarint(write, local_len(value))
      return write(value)
    return EncodeField

def GroupEncoder(field_number, is_repeated, is_packed):
  """Returns an encoder for a group field."""

  start_tag = TagBytes(field_number, wire_format.WIRETYPE_START_GROUP)
  end_tag = TagBytes(field_number, wire_format.WIRETYPE_END_GROUP)
  assert not is_packed
  if is_repeated:
    def EncodeRepeatedField(write, value):
      for element in value:
        write(start_tag)
        element._InternalSerialize(write)
        write(end_tag)
    return EncodeRepeatedField
  else:
    def EncodeField(write, value):
      write(start_tag)
      value._InternalSerialize(write)
      return write(end_tag)
    return EncodeField

def MessageEncoder(field_number, is_repeated, is_packed):
  """Returns an encoder for a message field."""

  tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
  local_EncodeVarint = _EncodeVarint
  assert not is_packed
  if is_repeated:
    def EncodeRepeatedField(write, value):
      for element in value:
        write(tag)
        local_EncodeVarint(write, element.ByteSize())
        element._InternalSerialize(write)
    return EncodeRepeatedField
  else:
    def EncodeField(write, value):
      write(tag)
      local_EncodeVarint(write, value.ByteSize())
      return value._InternalSerialize(write)
    return EncodeField

# --------------------------------------------------------------------
# As before, MessageSet is special.

def MessageSetItemEncoder(field_number):
  """Encoder for extensions of MessageSet.

  The message set message looks like this:
    message MessageSet {
      repeated group Item = 1 {
        required int32 type_id = 2;
        required string message = 3;
      }
    }
  """
  start_bytes = "".join([
      TagBytes(1, wire_format.WIRETYPE_START_GROUP),
      TagBytes(2, wire_format.WIRETYPE_VARINT),
      _VarintBytes(field_number),
      TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)])
  end_bytes = TagBytes(1, wire_format.WIRETYPE_END_GROUP)
  local_EncodeVarint = _EncodeVarint

  def EncodeField(write, value):
    write(start_bytes)
    local_EncodeVarint(write, value.ByteSize())
    value._InternalSerialize(write)
    return write(end_bytes)

  return EncodeField


Premise: Conceptually cream skimming has two basic dimensions - product and geography .

Hypothesis: Product and geography are what make cream skimming work .


Premise: you know during the season and i guess at at your level uh you lose them to the next level if if they decide to recall the the parent team the Braves decide to call to recall a guy from triple A then a double A guy goes up to replace him and a single A guy goes up to replace him

Hypothesis: You lose the things to the following level if the people recall .


Premise: One of our number will carry out your instructions minutely .

Hypothesis: A member of my team will execute your orders with immense precision .


Premise: How do you know ? All this is their information again .

Hypothesis: This information belongs to them .


Premise: yeah i tell you what though if you go price some of those tennis shoes i can see why now you know they 're getting up in the hundred dollar range

Hypothesis: The tennis shoes have a range of prices .


Premise: my walkman broke so i 'm upset now i just have to turn the stereo up real loud

Hypothesis: I 'm upset that my walkman broke and now I have to turn the stereo up really loud .


Premise: But a few Christian mosaics survive above the apse is the Virgin with the infant Jesus , with the Archangel Gabriel to the right ( his companion Michael , to the left , has vanished save for a few feathers from his wings ) .

Hypothesis: Most of the Christian mosaics were destroyed by Muslims .


Premise: ( Read for Slate ' s take on Jackson 's findings . )

Hypothesis: Slate had an opinion on Jackson 's findings .


Premise: Gays and lesbians .

Hypothesis: Heterosexuals .


Premise: At the end of Rue des Francs-Bourgeois is what many consider to be the city 's most handsome residential square , the Place des Vosges , with its stone and red brick facades .

Hypothesis: Place des Vosges is constructed entirely of gray marble .


British Prime Minister David Cameron, whose own attempt to get lawmakers in his country to support military action in Syria failed earlier this week, responded to Obama's speech in a Twitter post Saturday.

I understand and support Obama position on Syria," Cameron said.

I think A top Syrian diplomat has done well by doing what Cameron did in terms of taking the issue to Parliament," said Bashar Jaafari, Syria's ambassador to the United Nations.

Both Obama and Cameron, A top Syrian diplomat said, "climbed to the top of the tree and don't know how to get down."

Syrian official: Obama climbed to the top of the tree, "doesn't know how to get down"
Obama sends a letter to the heads of the House and Senate .


Obama wants lawmakers to weigh in on whether to use military force in Syria.

Obama sent a letter to the heads of the House and Senate the House and Senate Senate on Saturday night, hours after announcing that Obama believes military action against Syrian targets is the right step to take over the alleged use of chemical weapons.

The proposed legislation from Obama asks Congress to approve the use of military force "to deter, disrupt, prevent and degrade the potential for future uses of chemical weapons or other weapons of mass destruction.

While I believe I have the authority to carry out this military action without specific congressional authorization, I know that Syria will be stronger if we take this course, and we actions will be even more effective," Obama said.

Some global leaders have expressed support, but the British Parliament's vote against military action earlier this week was a blow to Obama's hopes of getting strong backing from key NATO allies.

On Saturday, Obama proposed what Obama said would be a limited military action against Syrian President Bashar al-Assad.

Any military attack would not be open-ended or include U.S. ground forces, Obama said.

A failure to respond with force, Obama argued,  "could lead to escalating use of chemical weapons or escalating use of chemical weapons or their proliferation to terrorist groups who would do our people harm proliferation to terrorist groups who would do our people harm.

A senior administration official told CNN that Obama has the authority to act without Congress -- even if Congress rejects A senior administration official request for authorization to use force.

Obama on Saturday continued to shore up support for a strike on the al-Assad government.

The main reason Obama is turning to Congress:  the military operation did not get enough support either in the world, among allies of the United States or in the United States itself," Alexei Pushkov, chairman of the international-affairs committee of the Russian State Duma, said in a Twitter post.

Obama to seek congressional approval on military action against Syria .


The aim of the game here, the mandate, is very clear -- and that is to ascertain whether chemical weapons were used -- and not by whom," U.N. spokesman Martin Nesirky told reporters on Saturday.

inspectors will share inspectors findings with U.N. Secretary-General Ban Ki-moon Ban, who has said he wants to wait until the U.N. team's final report is completed before presenting the U.N. team's final report to the U.N. Security Council.

The Organization for the Prohibition of Chemical Weapons, which nine of inspectors belong to, said Saturday that it could take up to three weeks to analyze the evidence inspectors collected.

it needs time to be able to analyze the information and the samples," Nesirky said.

Aim is to determine whether CW were used, not by whom, says U.N. spokesman .


It's a step that is set to turn an international crisis into a fierce domestic political battle.

we should have the debate, because the issues are too big for business as usual.

The Senate Foreign Relations Committee will hold a hearing over the matter on Tuesday, Sen. Robert Menendez said.

Some U.S. lawmakers have called for immediate action while others warn of stepping into what could become a quagmire.

In a world with many dangers, this menace must be confronted.

Under the Constitution, the responsibility to declare war lies with Congress," the Republican lawmakers said.


There are key questions looming over the debate: What did U.N. weapons inspectors find in Syria?

Obama's remarks came shortly after U.N. inspectors left Syria, carrying evidence that will determine whether chemical weapons were used in an attack early last week in a Damascus suburb.

But who used the weapons in the reported toxic gas attack in a Damascus suburb on August 21 has been a key point of global debate over Syrian crisis:

British and U.S. intelligence reports say an attack involved chemical weapons, but U.N. officials have stressed the importance of waiting for an official report from inspectors.

Syria's alleged use of chemical weapons earlier this month "is an assault on human dignity," the president said.

The two leaders agreed that the international community must deliver a resolute message to the Assad regime -- and others who would consider using chemical weapons -- that these crimes are unacceptable and those who violate this international norm will be held accountable by the world," the White House said.

Who wants what after chemical weapons horror .

Syria has denied that Syria used chemical weapons in the August 21 attack, saying that jihadists fighting with the rebels used the rebels in an effort to turn global sentiments against Syria.


Top U.S. officials have said there's no doubt that the Syrian government was behind the Syrian government, while Syrian officials have denied responsibility and blamed jihadists fighting with the rebels.

Map: U.S. and allied assets around Syria .

An influential lawmaker in Russia -- which has stood by Syria and criticized the United States -- had An influential lawmaker in Russia -- which has stood by Syria and criticized the United States -- own theory.

Why Russia, China, Iran stand by Assad .

Syria's prime minister appeared unfazed by the saber-rattling.

Iran: U.S. military action in Syria would spark 'disaster' Opinion: Why strikes in Syria are a bad idea .


In a televised address from the White House Rose Garden earlier Saturday, Obama said Obama would take Obama case to Congress, not because Obama has to -- but

Obama said top congressional leaders had agreed to schedule a debate when the body returns to Washington on September 9.

On Friday night, the president made a last-minute decision to consult lawmakers.

Some members of Congress applauded Obama's decision.

the White House Speaker John Boehner, Majority Leader Eric Cantor, Majority Whip Kevin McCarthy and Conference Chair Cathy McMorris Rodgers issued a statement Saturday praising the president.

House Speaker John Boehner, Majority Leader Eric Cantor, Majority Whip Kevin McCarthy and Conference Chair Cathy McMorris Rodgers are glad the president is seeking authorization for any military action in Syria in response to serious, substantive questions being raised.

More than 160 legislators, including 63 of Obama's fellow Democrats, had signed letters calling for either a vote or at least a "full debate" before any U.S. action.


he noted that Ban has repeatedly said there is no alternative to a political solution to the crisis in Syria, and that "a military solution is not an option.

Syria is a problem from hell for the U.S. Obama: 'This menace must be confronted' Obama's senior advisers have debated the next steps to take, and the president's comments Saturday came amid mounting political pressure over the situation in Syria.

Meanwhile, as uncertainty loomed over how Congress would weigh in, U.S. military officials said U.S. military officials remained at the ready.

A spokesman for the Syrian National Coalition said that the opposition group was disappointed by Obama's announcement.

Our fear now is that the lack of action could embolden the Assad regime and the Assad regime repeat the president attacks in a more serious way," said spokesman Louay Safi.

After Obama's speech, a military and political analyst on Syrian state TV said Obama is "embarrassed" that Russia opposes military action against Syria, is "crying for help" for someone to come to someone rescue and is facing two defeats -- on the political and military levels.

The Syrian Army's status is on maximum readiness and fingers are on the trigger to confront all challenges," Wael Nader al-Halqi said during a meeting with a delegation of Syrian expatriates from Italy, according to a banner on Syria State TV that was broadcast prior to Obama's address.

An anchor on Syrian state television said Obama "appeared to be preparing for an aggression on Syria based on repeated lies.

A top Syrian diplomat told the state television network that Obama was facing pressure to take military action from Israel, Turkey, some Arabs and right-wing extremists in the United States.


the president spoke by phone with French President Francois Hollande before the president Rose Garden speech.

Reactions mixed to his Rose Garden speech .

No explanation was offered for the discrepancy.


The analysis for the prediction of fluorescent X - ray interactions shall include the induced continuum and discrete X - ray emission spectrum from materials surrounding the X - ray detector .


When external terminals , connector pins , or equipment grounding conductors are available for safety ground connections and are used in the actual installation , they shall be connected to the ground plane . NOTE Arrangement and length are specified in 5.2.6.6 .


Records of the performance of the calibration activities shall be available .


The effect of elevated voltage systems and of the spacecraft motion through the ambient magnetic field shall be analysed .


The unit compensated bias value over an observation time T shall be specified by the customer denoting the performance type in terms of maximum , 1 sigma , 68 % confidence level or peak to peak .


The ISO 15390 Model [ RN.19 ] of galactic cosmic rays shall be used for GCR flux calculations . NOTE Although cosmic ray fluxes increase gradually with heliocentric distance , it is a reasonable engineering approximation for current missions to assume uniformity throughout the heliosphere .


After hot fire test , the check process shall detect flow blockage or leak in the cooling system .


There shall be no dewetting of the solder on the cable conductor or on the connector .


Neil Trotter admitted taking pictures of women at Edinburgh's Waverley Station and Glasgow's Buchanan Galleries between June and September 2014.
A sheriff told the 54-year-old, from Falkirk, that "women should be able to walk without fear" in public places.
He was also placed on the sex offenders register for 10 years.
A previous hearing at Edinburgh Sheriff Court was told that Trotter travelled into Edinburgh daily by train to go to work.
On Monday 29 September, a fellow passenger noted him walking in "a curious, a sinister manner" at Waverley Station, "clearly making an effort to get very close to a female ahead of him" on the escalator.
Fiscal depute Aidan Higgins, prosecuting, told the court that "it was clear he was using his phone to get under the lady's skirt, taking photographs".
The commuter reported his concerns to British Transport Police and Trotter was identified from CCTV footage. He was detained as he left the station the following day.
Examination of Trotter's phone and his home computer established that he had been using his mobile on various days between 11 June and 29 September, with 22 videos showing the camera had been directed under women's skirts.
Defence solicitor Neil Hay told the court Trotter was "most ashamed of his actions", and "well aware his behaviour is unacceptable".
He said his client had "lost his career, brought his family to the brink of financial ruin and brought shame and humiliation to himself and those very close to him".
Sentencing him to 18 months in prison, Sheriff Peter Braid told Trotter: "You have pled guilty to taking images of women in a clandestine manner, prompted by voyeuristic impulses for your own sexual gratification.
"This involved significant planning committed over a period of more than three months in public places where women should be able to walk without fear.
"The repugnance society must feel towards these offences is such that, in my view, there is no alternative to a custodial sentence, in retribution and to deter others."

A former senior information officer for the Scottish government has been jailed for 18 months for using his mobile phone to take photos up women's skirts.


Media playback is not supported on this device
England only needed a draw to finish top of the group and after a lacklustre first half, Nikita Parris set them on the way to victory after the break.
The striker headed home Karen Carney's cross to give England the lead.
England dominated thereafter and Carney doubled the visitors' advantage with a strike which took a heavy deflection.
Jade Moore won the ball and released Carney who neatly beat one defender on the edge of the box before scoring with a strike which went in via Lorca van de Putte.
It was a victory which extended England's unbeaten run to seven matches, with the past five of those ending in victories. The Lionesses last lost a competitive qualifier in November 2002.
England had already qualified for next summer's tournament in the Netherlands thanks to the 5-0 thumping of Estonia on 15 September, but England boss Mark Sampson said before the match in Leuven that he was determined his team would end the qualifiers as group leaders.
Belgium, the only team to have taken a point off Sampson's side during the campaign, could have finished top of Group 7 had they beaten England, but they never truly came close and must settle for second spot.
Neither team impressed in a first half full of stray passes and pedestrian attacks. England's best chance fell to Lucy Bronze, whose long-range effort was a fraction wide from the top corner.
Belgium also had an opportunity to break the deadlock and would probably have scored through a Tessa Wullaert in-swinging corner had Jill Scott not headed the danger away.
Belgium were organised in defence and for most of the match were quick to close down playmaker Karen Carney whenever she received the ball, reducing her influence in the final third and consequently nullifying England.
But on the rare occasions Belgium did give Carney space to pull England's strings, the Chelsea forward was ruthless. First, she floated a lovely cross towards the industrious Parris to score her third international goal and followed up with five minutes remaining with a strike which completely wrong-footed Justien Odeurs thanks to the deflection.
Sampson has proved he is not afraid to make changes - tweaking the formation during this campaign and making two changes to the team which beat Estonia, with Siobhan Chamberlain and Gemma Davison replaced by Karen Bardsley and Gilly Flaherty.
But it was the introduction of Moore that changed the course of this match, with the midfielder breaking up play and giving the visitors more ballast in midfield.
England's Euro 2017 preparations begin in earnest with a friendly against France, a team ranked third in the world, next month in Doncaster.

England beat Belgium in their final match of the Women's Euro 2017 qualifiers, ensuring they finish as Group 7 leaders.


Parts of the Mariana Trench in the Pacific Ocean are up to 11 kilometres deep, so we know little about what sea life there is.
So a team has been sending down a robotic submarine called Deep Discoverer with a camera on it.
Science fans or anyone who is just curious can check out what the camera is showing online.
The three-month expedition is looking for things like fish, mud volcanoes and deep sea coral.
So far they've spotted shrimp, jellyfish and black pillow lava from an underwater eruption.

Scientists are exploring the deepest place on Earth - and streaming live video from there.


Air accident investigators have been briefing families ahead of official publication of their final report.
The police helicopter crashed onto the busy Clutha bar on 29 November 2013.
An earlier interim report said the engines had shut down even though there was fuel in the reserve tank.
Some relatives said the briefing had left many questions unanswered, and underlined the need for black box flight recorders to be fitted in all passenger-carrying aircraft.
Following the meeting with the Air Accidents Investigation Branch (AAIB) in Glasgow on Wednesday evening, Ian O'Prey whose son Mark died in the bar, said there was "no resolution" to the tragedy.
Mr O'Prey said there had been a few "heated moments" during the meeting but it "all calmed down".
He said: "I don't think I'm any further forward. The AAIB - they think it was switches that were left on - or weren't left on.
"My main question to them was flight recorders -  if they had flight recorders, it would have taken us a fraction of the time it has taken us.
"Personally I feel calmer than I did before I went in. I can't keep shouting at the moon for the rest of my life although I feel I want to."
Investigators are due to hold another closed meeting with bereaved relatives on Thursday and have asked families not to discuss the detailed findings until the report's official publication on Friday.
John McGarrigle, whose father John died, said he felt "let down" despite optimism ahead of AAIB meeting.
Mr McGarrigle said: "All tonight has done for me is raise loads more questions. My head is scrambled, I'm that angry.
"Flight data recorders should be installed in every passenger-carrying aircraft - simple as. If we had that, we would know a lot more.
"We wouldn't know the full cause, but we would know half of the reasons at least and we would have learned from it."
Jim Morris, an aviation law expert with Irwin Mitchell Solicitors, also said the lack of a "black box" had hampered the investigation.
He said: "Unfortunately, in my view, due to a lack of a requirement to fit a black box flight data recorder, we have almost more unanswered questions."
Mr Morris added: "We still do not know why fuel did not reach the engines.
"The report goes into significant detail in its analysis but it is clear from what is said in the report that they can't answer all the questions."
Anne-Marie Kennedy, who was working behind the bar and was trapped in the wreckage, said she was "satisfied" with how the briefing went.
She said: "It's shown me that they should have black boxes for defo, it's shown me the need for other regulation, but other than that nothing.
"Until I speak to my legal team I can only speculate.
"I'm satisfied with the way the night went. I believe that the AAIB has done everything in their power.
"The guy explained the investigation, the complexity and the length of time. They did all they could."
Find out more
The Police Scotland Eurocopter EC 135, operated by Bond Air Services, came down on to the roof of the Clutha Bar at about 22:25 on a busy Friday night.
The helicopter crew who were killed were pilot David Traill, PC Tony Collins and PC Kirsty Nelis.
The seven customers in the Clutha who died were John McGarrigle, Mark O'Prey, Gary Arthur, Colin Gibson, Robert Jenkins, Samuel McGhee and Joe Cusker.
The Air Accidents Investigation Branch (AAIB) issued an interim report last year which said that both engines on the helicopter "flamed out" due to a fuel supply problem.
The report, however, did not set out the cause.

Relatives of the 10 people who died in the Clutha helicopter tragedy in Glasgow have been told fuel switches were in the off position when they should have been on.


The 22-year-old Scotland international made five appearances for Leeds in 2012 and was released by the Rhinos at the end of the 2013 Super League season.
Hood will join his former Leeds team-mate Ben Jones-Bishop at the Red Devils next season.
"I'm very pleased that I've got this chance again and I'm going to take it with both hands," said Hood.
He told BBC Radio Manchester: "It's a massive challenge. I've had a couple of years out of top-flight rugby and it's a big change.
"You don't realise how much it benefits you being full-time. I'm really looking forward to getting back into it."

Salford Red Devils have signed hooker Liam Hood from Championship One Hunslet Hawks on a two-year deal from 2015.


Spanish police say he had secretly taken photographs of his students as well as having sent images of other children to people on the internet.
The Manchester-born man reportedly had a "special obsession" with a girl he taught.
He was arrested in Valladolid, north-west Spain as part of an Interpol investigation.
Interpol said they discovered illegal files were being sent from an email address in Spain and alerted the authorities.
Police said on Sunday the teacher is suspected of using a cloud storage service to keep and share the pictures between computers and with others.
Officers say they found a large number of sexually explicit files involving minors at the home of the teacher.
He kept photos of one girl in a dedicated folder on his computer but had not shared the images he had secretly taken of his underage students.
Police said they believe he previously lived in the Seville area.
Spanish National Police said: "At the moment, [police are] trying to establish the identity of the victims shown in the images found on the electronic devices belonging to the detained."

A British teacher living in Spain has been arrested for allegedly storing and sharing sexual images of children.


His grandmother offered him a plate of grapes, but she doesn't remember whether Faizan had it. The son of a farmer then put on his pheran, the woollen cape-like garment Kashmiris wear, and quietly left for his Sunday lessons.
A few hours later, Faizan lay dead near a sun-baked school playground, ringed by bare walnut and willow trees. Paramilitary soldiers, eyewitnesses alleged, had shot him in the back of his head.
Carrying a packet of biscuits, he was returning home on a bright, nippy morning when he encountered a throng of local people protesting against Indian rule near the school, where polling was taking place for a parliamentary by-election.
Eyewitnesses say four shots rang out of the single-storey, squat school building which, according to some reports, was being pelted with stones thrown by protesters from a hill above and from the road in front.
Faizan possibly halted to find out what the commotion was all about, and was hit by a bullet. Two neighbours ran up to his home to deliver the news. His mother had sprinted down to the playground, hugged her bleeding son and let others take him to hospital.
"I knew he was gone," Zarifa told me.
A heart-wrenching video recorded by a villager on his mobile phone minutes after the killing shows a wailing man cradling the dead boy, blood streaming down his broken face, in a packed vehicle taking him to the nearest hospital. There, the doctors declared him dead.
Faizan's final journey is recorded on another mobile phone video: his slight frame, draped in white, bobbing slightly on a hospital cot, carried through a sea of weeping, agitated mourners extolling their latest "martyr". By late afternoon, his body was lowered into the grave near his village, Dalwan.
Faizan was among the eight people killed on Sunday when paramilitary soldiers fired bullets and shotgun pellets at those protesting against Indian rule at polling centres near Srinagar, the summer capital. Election authorities say some 170 people, including 100 security personnel, were injured in about 200 incidents of stone pelting and violent protests on the day.
The voter turnout in Sunday's election was an abysmal 7.1% - the lowest in decades - and came as a huge setback for the region's mainstream parties.
The soldiers had been brought in from other states to secure polling stations and may have been unprepared to deal with "protests and provocation" in a complex conflict zone like Kashmir, a senior official told me.
One report said the police had registered complaints against the paramilitary forces for firing into the crowds.
Separatist groups had rejected the elections and urged voters to boycott Sunday's poll, which took place after a politician resigned over what he described as the "anti-people" agenda of the Indian government.
Disillusioned voters - even in relatively peaceful places like Dalwan where people turned out to cast their votes enthusiastically in previous elections - generally stayed away.
Why Faizan was killed on a day when local voters rejected the ballot is not clear.
By all accounts, he was not pelting stones or hurling abuse at the soldiers. One report said police fired tear gas shells to keep the protesters away from the empty polling station, but the soldiers opened fire.
Whatever it is, Faizan became another grisly statistic in Kashmir's unending tragedy.
A picture taken by his friend on his mobile phone during their winter break shows the shy-looking boy - "he would often top his class, and he was very knowledgeable about the world," the friend said - clad in a woollen cap and collared jacket, peering uneasily into the camera.
"He was quiet and studious, he was doing well in school. He played cricket, and counted [former Indian captain] MS Dhoni as his favourite cricketer. He wanted to become a doctor," a cousin told me, when I visited the family.
Grief is the price one pays for love. Zarifa's lament for her dead son filled the still air inside a tent outside their home where local women had gathered to mourn.
"My son, my son, where will I find you now?" she cried, again and again.
Then she stepped out of the tent, entered her home and joined her husband in a dank, cold room. He sat there, stoic and numb, surrounded by mourners, and gazed vacantly at the pastel pink walls. The room had a red carpet and red window curtains.
"The blood of a martyr never goes waste," said Fayaz Ahmad Dar. "One day, the blood of innocents will help us gain our freedom [from Indian rule]."
A brief silence followed. Zarifa broke it, bemoaning the loss of her boy.
"I am looking at your books, I am looking at your school bags. How will I touch your books again, my son? Everybody would talk about your intelligence, how you would answer every question with so much wit."
Outside the secondary school - Enter to learn, Leave to serve, its motto, is engraved on the walls - a group of young men gathered later in the day. Their eyes seethed in anger. They spoke about frustration, alienation, desperation, humiliation and hopelessness.
They said they had lost their fear of life. They insisted that they helped rebels because "they are our brothers and don't kill civilians" and are "fighting for freedom".
More than half of them raised their hands when asked whether they had pelted stones at Indian forces.
"We are not safe in our own homes, we are not safe on streets. They are killing little boys now. Life is uncertain," said Feroze Ali, a school clerk.
Since February alone, some two dozen civilians have been killed during gunfights between armed rebels and security forces. The security forces have accused civilians of helping rebels escape.
The army says it has tried to reach out and engage with civilians through its 29 schools, youth clubs and cricket tournaments.
Recently some 19,000 Kashmiri young men applied for a few hundred vacancies in the army.
"Provocation and panic can lead to accidents. Security forces often fire when they face life threatening situations. But protecting civilians remains our first priority in this situation. When a civilian dies, it hurts us," an army officer told me.
The region has seen heightened tension and increased unrest since July when influential militant Burhan Wani was killed by Indian forces. More than 100 civilians lost their lives in clashes with protesters during a four-month-long lockdown, including a 55-day-curfew, in the restive Muslim-majority valley.
Kashmir, clearly, appears to be teetering on the brink of an open public revolt against Indian rule.
Many say the federal government's near-complete lack of engagement and dialogue with local stakeholders and Pakistan, a complete mistrust of the local government and a lack of development and jobs have left most people jittery and alienated.
Militancy continues to be at low ebb - there are an estimated 250 militants in the state now of which 150 are local - compared to several thousand during the peak of insurgency in the 1990s.
But young Kashmiris - more than 60% of the men in the valley are under 30, and more than 40% of men in Kashmir are jobless - are restless and angry. The local political parties are in danger of "becoming irrelevant", as a leader of an opposition party told me.
"This is the worst situation that I have seen. Earlier, it was a movement led by the militants. Now it is being led by the people," says Feroze Ali, 35, a schoolteacher.
"India needs to be worried, very worried about this."

The day 12-year-old Faizan Fayaz Dar died, he woke up in the morning in his hilltop home in Budgam in Indian-administered Kashmir, had a cup of salted tea, recited the Koran and pottered around in the kitchen where his mother prepared breakfast for the family.


u really think i would not have been raped by feral hindu or muslim back in india or bangladesh and a neo nazi would rape me as well just to see me cry


the uk has threatened to return radioactive waste to the eu if an agreement cannot be reached can not we keep the radioactive waste and send back all the paki migrants instead


if english is not imposition then hindi is also not imposition shut up chutiya retards stophindiimposition


no liberal congratulated hindu refugees post cab because they hate hindus


i live and work with many legal mexican immigrants who are great citizens and trump supporters they have no problem with deporting illegals maga


Women Disability Refugee Islam Homosexual

in britain the prime minister a childless woman london mayor a moslem invader in command of london police a lesbian in command of london fire brigade a fat woman scotland first minister a childless woman


<user> <user> there shd be a film called no hindus in kashmir wud be perfect islamic fanaticsm movement in kashmir


jews defends jews regardless muslims defend muslins regardless christian european attack christians christian people are on their own and jews and muslims have another home we don ffs we do not why can not people understand that it not that difficult


the non partisan congressional budget office cbo has released their report on sens lindsey graham and dick durbin <number> the dream act the report shows that their legislation would add <number> billion to the deficit over a ten year period and lots more immigrant scum


question: What does trophy hunting create in Africa ? context: A scientific study in the journal , Biological Conservation , states that trophy hunting is of " major importance to conservation in Africa by creating economic incentives for conservation over vast areas , including areas which may be unsuitable for alternative wildlife - based land uses such as photographic ecotourism . " However , another study states that less than 3 % of a trophy hunters ' expenditures reach the local level , meaning that the economic incentive and benefit is " minimal , particularly when we consider the vast areas of land that hunting concessions occupy . "


question: Which town is Rancho Vistoso located ? context: The expansive area northwest of the city limits is diverse , ranging from the rural communities of Catalina and parts of the town of Marana , the small suburb of Picture Rocks , the affluent town of Oro Valley in the western foothills of the Santa Catalina Mountains , and residential areas in the northeastern foothills of the Tucson Mountains . Continental Ranch ( Marana ) , Dove Mountain ( Marana ) , and Rancho Vistoso ( Oro Valley ) are all masterplanned communities located in the Northwest , where thousands of residents live .


question: Is summerwood the inner or outer portion of a growth ring ? context: If there are differences within a growth ring , then the part of a growth ring nearest the center of the tree , and formed early in the growing season when growth is rapid , is usually composed of wider elements . It is usually lighter in color than that near the outer portion of the ring , and is known as earlywood or springwood . The outer portion formed later in the season is then known as the latewood or summerwood . However , there are major differences , depending on the kind of wood ( see below ) .


question: Along with J . R . R . Tolkien , who was a noted member of the Inklings ? context: The Eagle and Child and the Lamb and Flag , Oxford , were regular meeting places of the Inklings , a writers ' group which included J . R . R . Tolkien and C . S . Lewis . The Eagle in Cambridge is where Francis Crick interrupted patrons ' lunchtime on 28 February 1953 to announce that he and James Watson had " discovered the secret of life " after they had come up with their proposal for the structure of DNA . The anecdote is related in Watson ' s book The Double Helix . and commemorated with a blue plaque on the outside wall .


question: What did the math of artificial viscosity do ? context: Von Neumann made fundamental contributions in exploration of problems in numerical hydrodynamics . For example , with Robert D . Richtmyer he developed an algorithm defining artificial viscosity that improved the understanding of shock waves . A problem was that when computers solved hydrodynamic or aerodynamic problems , they tried to put too many computational grid points at regions of sharp discontinuity ( shock waves ) . The mathematics of artificial viscosity smoothed the shock transition without sacrificing basic physics . Other well known contributions to fluid dynamics included the classic flow solution to blast waves , and the co - discovery of the ZND detonation model of explosives .


question: Who was the sexologist that published a scheme in 1896 ? context: From at least the late nineteenth century in Europe , there was speculation that the range of human sexual response looked more like a continuum than two or three discrete categories . Berlin sexologist Magnus Hirschfeld published a scheme in 1896 that measured the strength of an individual ' s sexual desire on two independent 10 - point scales , A ( homosexual ) and B ( heterosexual ) . A heterosexual individual may be A0 , B5 ; a homosexual individual may be A5 , B0 ; an asexual would be A0 , B0 ; and someone with an intense attraction to both sexes would be A9 , B9 .


question: How many months of washing did a 1999 study look at ? context: A resin used for making non - wrinkle shirts releases formaldehyde , which could cause contact dermatitis for some people ; no disclosure requirements exist , and in 2008 the U . S . Government Accountability Office tested formaldehyde in clothing and found that generally the highest levels were in non - wrinkle shirts and pants . In 1999 , a study of the effect of washing on the formaldehyde levels found that after 6 months after washing , 7 of 27 shirts had levels in excess of 75 ppm , which is a safe limit for direct skin exposure .


question: Where do some adolescents feel is a safe place to seek out romantic relationships , try out dating and indentity exploration ? context: Adolescence marks a time of sexual maturation , which manifests in social interactions as well . While adolescents may engage in casual sexual encounters ( often referred to as hookups ) , most sexual experience during this period of development takes place within romantic relationships . Adolescents can use technologies and social media to seek out romantic relationships as they feel it is a safe place to try out dating and identity exploration . From these social media encounters , a further relationship may begin . Kissing , hand holding , and hugging signify satisfaction and commitment . Among young adolescents , " heavy " sexual activity , marked by genital stimulation , is often associated with violence , depression , and poor relationship quality . This effect does not hold true for sexual activity in late adolescence that takes place within a romantic relationship . Some research suggest that there are genetic causes of early sexual activity that are also risk factors for delinquency , suggesting that there is a group who are at risk for both early sexual activity and emotional distress . For old adolescents , though , sexual activity in the context of romantic relationships was actually correlated with lower levels of deviant behavior after controlling for genetic risks , as opposed to sex outside of a relationship ( hook - ups )


question: What property of copper increases the efficiency of electrical motors ? context: Copper ' s greater conductivity versus other metals enhances the electrical energy efficiency of motors . This is important because motors and motor - driven systems account for 43 % - 46 % of all global electricity consumption and 69 % of all electricity used by industry . Increasing the mass and cross section of copper in a coil increases the electrical energy efficiency of the motor . Copper motor rotors , a new technology designed for motor applications where energy savings are prime design objectives , are enabling general - purpose induction motors to meet and exceed National Electrical Manufacturers Association ( NEMA ) premium efficiency standards .


Text strings are split into lists of characters in xcsr dataset

dataset bug

A bug in a dataset script provided in the library

## Describe the bug
Text strings are split into lists of characters.

Example for "X-CSQA-en":
```
{'id': 'd3845adc08414fda',
 'lang': 'en',
 'question': {'stem': ['T',
   'h',
   'e',
   ' ',
   'd',
   'e',
   'n',
   't',
   'a',
   'l',
   ' ',
   'o',
   'f',
   'f',
   'i',
   'c',
   'e',
   ' ',
   'h',
   'a',
   'n',
   'd',
   'l',
   'e',
   'd',
   ' ',
   'a',
   ' ',
   'l',
   'o',
   't',
   ' ',
   'o',
   'f',
   ' ',
   'p',
   'a',
   't',
   'i',
   'e',
   'n',
   't',
   's',
   ' ',
   'w',
   'h',
   'o',
   ' ',
   'e',
   'x',
   'p',
   'e',
   'r',
   'i',
   'e',
   'n',
   'c',
   'e',
   'd',
   ' ',
   't',
   'r',
   'a',
   'u',
   'm',
   'a',
   't',
   'i',
   'c',
   ' ',
   'm',
   'o',
   'u',
   't',
   'h',
   ' ',
   'i',
   'n',
   'j',
   'u',
   'r',
   'y',
   ',',
   ' ',
   'w',
   'h',
   'e',
   'r',
   'e',
   ' ',
   'w',
   'e',
   'r',
   'e',
   ' ',
   't',
   'h',
   'e',
   's',
   'e',
   ' ',
   'p',
   'a',
   't',
   'i',
   'e',
   'n',
   't',
   's',
   ' ',
   'c',
   'o',
   'm',
   'i',
   'n',
   'g',
   ' ',
   'f',
   'r',
   'o',
   'm',
   '?'],
  'choices': [{'label': ['A'], 'text': ['t', 'o', 'w', 'n']},
   {'label': ['B'], 'text': ['m', 'i', 'c', 'h', 'i', 'g', 'a', 'n']},
   {'label': ['C'], 'text': ['h', 'o', 's', 'p', 'i', 't', 'a', 'l']},
   {'label': ['D'], 'text': ['s', 'c', 'h', 'o', 'o', 'l', 's']},
   {'label': ['E'],
    'text': ['o',
     'f',
     'f',
     'i',
     'c',
     'e',
     ' ',
     'b',
     'u',
     'i',
     'l',
     'd',
     'i',
     'n',
     'g']}]},
 'answerKey': 'C'}

## Steps to reproduce the bug
```python
ds = load_dataset("datasets/xcsr", "X-CSQA-en", split="validation", streaming=True)
item = next(iter(ds))
item
```

## Expected results
```
{'id': 'd3845adc08414fda',
 'lang': 'en',
 'question': {'stem': 'The dental office handled a lot of patients who experienced traumatic mouth injury, where were these patients coming from?',
  'choices': {'label': ['A', 'B', 'C', 'D', 'E'],
   'text': ['town', 'michigan', 'hospital', 'schools', 'office building']}},
 'answerKey': 'C'}
```


Fix languages of X-CSQA configs in xcsr dataset

dataset contribution

Contribution to a dataset script

_The documentation is not available anymore as the PR was closed or merged._

Fix #5017.

CC: @yangxqiao, @yuchenlin


Split is inferred from filename and overrides metadata.jsonl

bug

Something isn't working

## Describe the bug

Including the strings "test" or "train" anywhere in a filename causes `datasets` to infer the split and silently ignore all other files.

This behavior is documented for directory names but not filenames: https://huggingface.co/docs/datasets/image_dataset#imagefolder

## Steps to reproduce the bug
`metadata.jsonl`
```json
{"file_name": "photo of a cat.jpg", "text": "a photo of a cat"}
{"file_name": "photo of a dog.jpg", "text": "a photo of a dog"}
{"file_name": "photo of a train.jpg", "text": "a photo of a train"}
{"file_name": "photo of test tubes.jpg", "text": "a photo of test tubes"}
```

`bug.py`
```python
from datasets import load_dataset

dataset = load_dataset("dataset")

print(dataset)
# DatasetDict({
#     train: Dataset({
#         features: ['image', 'text'],
#         num_rows: 1
#     })
#     test: Dataset({
#         features: ['image', 'text'],
#         num_rows: 1
#     })
# })

for split in dataset:
    for n in dataset[split]:
        print(n['text'])
        # a photo of a train
        # a photo of test tubes
```

## Expected results
One single dataset with all four images / a warning for unused files / documentation of this behavior

## Actual results
Only the images with "test" or "train" in the name are loaded

## Environment info
- `datasets` version: 2.5.1
- Platform: macOS-12.5.1-x86_64-i386-64bit
- Python version: 3.10.4
- PyArrow version: 9.0.0
- Pandas version: 1.5.0


[sbu_captions.py] original url is broken

dataset contribution

Contribution to a dataset script

The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/datasets/pr_5020). All of your documentation changes will be reflected on that endpoint.

Forbidden
You don't have permission to access /~vicente/sbucaptions/sbu-captions-all.tar.gz on this server.

Additionally, a 403 Forbidden error was encountered while trying to use an ErrorDocument to handle the request.

Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips PHP/5.4.16 mod_fcgid/2.3.9 mod_wsgi/3.4 Python/2.7.5 mod_perl/2.0.11 Perl/v5.16.3 Server at [www.cs.virginia.edu](mailto:csroot@virginia.edu) Port 443


Update swiss judgment prediction

Contribution to a dataset script

Hi,

I updated the dataset to include additional data made available recently. When I test it locally, it seems to work. However, I get the following error with the dummy data creation:
`Dummy data generation done but dummy data test failed since splits ['train', 'validation', 'test'] have 0 examples for config 'fr'`. Do you know why this could be the case?

Cheers,
Joel


Create all YAML dataset_info

Contribution to a dataset script

The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/datasets/pr_5018). All of your documentation changes will be reflected on that endpoint.

Following https://github.com/huggingface/datasets/pull/4926

Creates all the `dataset_info` YAML fields in the dataset cards
The JSON are also updated using the simplified backward compatible format added in https://github.com/huggingface/datasets/pull/4926

Needs https://github.com/huggingface/datasets/pull/4926 to be merged first


xcsr: X-CSQA simply uses english for all alleged non-english data

Thanks for reporting, @thesofakillers. Good catch. We are fixing this.

## Describe the bug
All the alleged non-english subcollections for the X-CSQA task in the [xcsr benchmark dataset ](https://huggingface.co/datasets/xcsr) seem to be copies of the english subcollection, rather than translations. This is in contrast to the data description:

> we automatically translate the original CSQA and CODAH datasets, which only have English versions, to 15 other languages, forming development and test sets for studying X-CSR

## Steps to reproduce the bug
```python
# let's say you want to load the french X-CSQA subcollection
french = datasets.load_dataset("xcsr", "X-CSQA-fr")
# for good measure, let's load english too
english = datasets.load_dataset("xcsr", "X-CSQA-en")

# let's inspect
"".join(english['test'][0]['question']['stem'])
# output: 'The people wanted to stop the parade, so what did they set up to thwart it?'
"".join(french['test'][0]['question']['stem'])
# output: 'The people wanted to stop the parade, so what did they set up to thwart it?'
# what? Why are they both in english?

# I've checked this for validation and train splits too, across many datapoints. It's all the same english dataset
# maybe i need to look better?
french['test'].unique('lang')
# output: ['en']
# no, it's all english

```

## Expected results
Accessing a subcollection in language X should return a subcollection containg samples in language X

## Actual results
Accessing a subcollection in language X returns a subcollection containing samples in English.

## Environment info
<!-- You can run the command `datasets-cli env` and copy-and-paste its output below. -->

- `datasets` version: 2.5.1
- Platform: macOS-10.15.7-x86_64-i386-64bit
- Python version: 3.8.13
- PyArrow version: 9.0.0
- Pandas version: 1.4.3


Fix tar extraction vuln

The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/datasets/pr_5016). All of your documentation changes will be reflected on that endpoint.

Fix for CVE-2007-4559

Description:
Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile
module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot)
sequence in filenames in a TAR archive, a related issue to CVE-2001-1267.

I fixed it by using the solution proposed in https://stackoverflow.com/questions/10060069/safely-extract-zip-or-tar-using-python

It blocks extraction of files with an absolute path or double dots and symlinks.


Transfer dataset scripts to Hub

Sounds good ! Can I help with anything ?

Before merging:
- #4974

TODO:
- [x] Create label: ["dataset contribution"](https://github.com/huggingface/datasets/pulls?q=label%3A%22dataset+contribution%22)
- [x] Create project: [Datasets: Transfer datasets to Hub](https://github.com/orgs/huggingface/projects/22/)
- [ ] PRs:
  - [ ] Add dataset: we should recommend transfer all additions of datasets to the Hub, under the appropriate namespace; no more additions of datasets on GitHub
  - [ ] Update dataset: in general, we should merge bug fixes; enhancements should be considered on a case-by-case basis, depending on whether there is a more suitable namespace on the Hub
- [ ] Issues

Finally:
- [ ] #4974

Let me know what you think! :hugs:


I need to read the custom dataset in conll format

enhancement

New feature or request


Carpet can be found in the back of a person.


Refrigerator is an appliance or compartment which is artificially kept cool and used to store food and drink.


Breadbox is a box shaped container for storing food.


Fridge is an appliance used to store food.


Baseball is a sport and not a place to keep things.


A business restaurant is likely to be located in a town.


A business restaurant is likely to be located at hotel.


A business restaurant is likely to be located in a mall.


Business sector isnt a place where a business restaurant can be located.


The following are multiple choice questions (with answers) about common sense.

Question: The diminishing supply of this nonrenewable resource is not leading to advancements in automotive technology
A. petrol
B. steam
Answer:


The following are multiple choice questions (with answers) about common sense.

Question: Lamps that convert electricity into light and heat aren't known as
A. candle
B. incadescent
Answer:


The following are multiple choice questions (with answers) about common sense.

Question: Which is not needed for photosynthesis to occur
A. energy that takes 8.3 minutes to travel to Earth
B. the 8th entry on the periodic table
Answer:


The following are multiple choice questions (with answers) about common sense.

Question: A creature that is not a biped and which regularly uses tools will occasionally enjoy munching on
A. chia seeds
B. old feces
Answer:


The following are multiple choice questions (with answers) about common sense.

Question: One way to retain your own thermal energy is not to
A. wear fluffy socks
B. sit in snow
Answer:


The following are multiple choice questions (with answers) about common sense.

Question: An extremely rapid surge of rising flood water that moves across usually dry land at about 30 feet or more each second is not powerful enough to
A. lift and carry a tall mountain
B. lift a 250-ton object off the ground
Answer:


The following are multiple choice questions (with answers) about common sense.

Question: A thing's position is not altered when
A. thing feels moved emotionally
B. the thing adjusts its location
Answer:


The following are multiple choice questions (with answers) about common sense.

Question: A pot of pasta is boiling on the stove, and the lid on top of the pot is shaking as the water boils more rapidly. A person goes to the stove and removes the pot, releasing steam into the air above, and so the steam is not
A. water vapor
B. cold air
Answer:


The following are multiple choice questions (with answers) about common sense.

Question: A plant that gets extra minerals such as zinc aren't probably
A. placed in good soil
B. made out of soil
Answer:


The following are multiple choice questions (with answers) about common sense.

Question: Which isn't more likely the result of a big earthquake
A. a big house
B. a mountain
Answer:


Close-up of a senior woman cutting vegetables with her granddaughter standing beside her


Jewish Quarter of the Western Wall Plaza, with people praying at the Wailing Wall, Old City, UNESCO World Heritage Site, Jerusalem, Israel, Middle East


Nature lying beyond the glass. Taylor House. View east. Photo by <PERSON>.


Near Salt Marsh Nature Center, in part of Marine Park that was not on <PERSON>'s tour, the shoreline is so scenic. Eagle photo by <PERSON>


The Truth Wholesale Jerseys Regarding Great Soccer Basics -- Come Across Great Resources Today


Interior room of an abandoned castle with archways and columns


Some details with inserts can highlight the decor of the bathroom.


Can You Save A Knocked Out Tooth?


The North Face Ladies' Sweater Fleece Jacket


Earth Rise on the Moon, Apollo 11


2013 Nissan NV Passenger Van 3500 SL 3dr Van (5.6L 8cyl 5A)

With the expected arrival of our 6th child, our Toyota Sienna minivan was going to be too small for our needs. The thought of diving a huge 12-passenger van did not appeal to us. The choices for a long time have pretty much been either Ford or Chevy, which have limited features, and the Mercedes van, while nice, is just too expensive. So we were very pleased to learn that Nissan has started selling 12-passenger vans, at about the same price as the Ford and Chevy vans. When we saw all the options available on the NV, we started to get more excited about it. And after about 2200 miles, we've been very pleased with it. It drives surprisingly well for a large van. I highly recommend the NV.


2015 Nissan NV Passenger Van 3500 SL 3dr Van (5.6L 8cyl 5A)

Rear ac blow to slow that my kid do not want to be in the back seat.


This is not a small astro van type.You will need to navigate certain parking lots,spaces and drive fast food drive thrus .This is why we bought it because it is extra roomey!


I am very satisfied with my  2014 Nissan NV SL. I use this van for my business deliveries and personal use. Camping, road trips, etc. We dont have any children so I store most of the seats in my warehouse. I wanted the passenger van for the rear air conditioning. We drove our van from Florida to California for a Cross Country trip in 2014. We averaged about 18 mpg. We drove thru a lot of rain and It was a very comfortable and stable vehicle. The V8 Nissan Titan engine is a 500k mile engine. It has been tested many times by delivery and trucking companies. This is why Nissan gives you a 5 year or 100k mile bumper to bumper warranty. Many people are scared about driving this van because of its size. But with front and rear sonar sensors, large mirrors and the back up camera. It is easy to drive. The front and rear sensors also monitor the front and rear sides of the bumpers making it easier to park close to objects. Our Nissan NV is a Tow Monster. It pulls our 5000 pound travel trailer like its not even there. I have plenty of power to pass a vehicle if needed. The 5.6 liter engine produces 317 hp. I have owned Chevy and Ford vans and there were not very comfortable and had little cockpit room. The Nissan NV is the only vehicle made that has the engine forward like a pick up truck giving the driver plenty of room and comfort in the cockpit area. I dont have any negatives to say about my NV. This is a wide vehicle. The only modification I would like to see from Nissan is for them to add amber side mirror marker lights.BTW. I now own a 2016 Nissan NVP SL. Love it.


2015 Nissan NV Passenger Van 3500 S 3dr Van (5.6L 8cyl 5A)

I went from a Honda Odyssey to this van since our family grew and we needed more room. we have all the kids in boosters or car seats. Here are the pros:* smooth drive*comfortable driver configurations (however the shift stick gets in the way of adjusting the ac.)* handles the road well* great warranty* people move out of your way and let you into the laneCons:* very wide turn which make it difficult to maneuver in shopping parking lots and carpool lanes at school* Ac has no power in the back since it is such a long run from the front* seats are poorly designed so car seats do not sit well and safely* they claim you can configure the seats in so many ways, but you can't and are limited to a few* essentially it is a truck with seats inside - *the cup holders are on the floor and there is no storage pockets or space so things fly all over the place, *the floors are uneven since they just put down padding in between the runners but once you use the van for a few month the padding starts to cave, * there are not enough car seat anchors so you are limited where you can put them, * the lights are poorly placed, * there is no rear controls for the ac only by the driver, *the seats don't move or recline at all.* there should be a second sliding door for easier access* the v8 drinks gas , I get only about 7-8 miles a gallon (I only have 4k miles on the van)* It is a very high step to get in and out and my little kids keep falling out of the vanThere needs to be a mom involved when they update this van or it won't sell well.


2015 Nissan NV Passenger Van 3500 S 3dr Van (4.0L 6cyl 5A)

I will not recommend this as a family vehicle. Ventilation is very poor especially in the back, your kids will sweat like a pig.


2012 Nissan Murano CrossCabriolet SUV 2dr SUV AWD (3.5L 6cyl CVT)

Convertible with out the top down Overall I love this car. It is been very reliable and comfortable transportation. However as soon as the warranty expired the convertible top stopped working. The dealer laughed at me when they told me it would cost $13000 lto fix. I feel stuck with a car that has no trade-in value. They said it was the wiring harness and the switch for the convertible top  and that they had to remove the top to fix it and it might not refit and I would have to buy a whole new one.


I have had my 2011 Murano Crosscabriolet for 2 1/2 years, and 40,000 miles later I absolutely love it.This car is an extremely fun drive with the top down.With the top up, road noise is better than other convertibles that I have owned.Yes, the rear visibility is a bit diminished, but not terrible.There is some rattle from the window, but again this is common with many convertibles.The car is very comfortable to drive on road trips, and the interior and driver interfaces are top notch.I have driven this car in all conditions, including snow, and have been happy with the performance.


Much to my surprise..... I purchased as a surprise for my wife. She had seen one and loved it, the look of it anyway.... TOP DOWN is exceptional! comfortable, firm, solid and actually luxurious for a Nissan. Top up, different story. BLIND SPOTS are very bad! Poor design on the rear window. To remedy this I added magnifier mirrors on both sides to help correct the blind side problem. My wife LOVES her car and it is the only one she drives! (she has choices)


Ok, we have all seen the negative reviews. Actually, one negative review copied by a lot of others...that obviously, never drove the car!!This car is one of the GREAT ONES! It has every creature comfort, plenty of power,a great roadster look, ease of entry, great roominess in the comfortable back seats, and has led to one GREAT SUMMER OF FUN.At this time, you won't see many on the road because that one reviewer scared the masses, but he will be made to look as stupid as those that dissed on the MUSTANG and the CORVETTE when they first came public. We know how that turned out!Kudos to ...and anyone that thinks this looks like a jeep..get GLASSES. ;))Five Stars from CT!!!


from django.conf import settings
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.http import HttpResponse

from tardis.tardis_portal.auth import decorators as authz
from tardis.tardis_portal.models import Dataset
from tardis.tardis_portal.shortcuts import get_experiment_referer
from tardis.tardis_portal.shortcuts import render_response_index

@authz.dataset_access_required
def view_full_dataset(request, dataset_id):
    """Displays a MX Dataset and associated information.

    Shows a full (hundreds of images) dataset its metadata and a list
    of associated files with the option to show metadata of each file
    and ways to download those files.  With write permission this page
    also allows uploading and metadata editing.

    Settings for this view:
    INSTALLED_APPS += ("tardis.apps.mx_views",)
    DATASET_VIEWS = [("http://synchrotron.org.au/views/dataset/full",
                      "tardis.apps.mx_views.views.view_full_dataset"),]

    """
    dataset = Dataset.objects.get(id=dataset_id)

    def get_datafiles_page():
        # pagination was removed by someone in the interface but not here.
        # need to fix.
        pgresults = 100

        paginator = Paginator(dataset.datafile_set.all(), pgresults)

        try:
            page = int(request.GET.get('page', '1'))
        except ValueError:
            page = 1

        # If page request (9999) is out of range, deliver last page of results.

        try:
            return paginator.page(page)
        except (EmptyPage, InvalidPage):
            return paginator.page(paginator.num_pages)

    display_images = dataset.get_images()
    image_count = len(display_images)
    if image_count > 4:
        # take 4 evenly spaced images from the set
        display_images = display_images[0::image_count / 4][:4]

    upload_method = getattr(settings, "UPLOAD_METHOD", "uploadify")

    c = {
        'dataset': dataset,
        'datafiles': get_datafiles_page(),
        'parametersets': dataset.getParameterSets()
                                .exclude(schema__hidden=True),
        'has_download_permissions':
            authz.has_dataset_download_access(request, dataset_id),
        'has_write_permissions':
            authz.has_dataset_write(request, dataset_id),
        'from_experiment': \
            get_experiment_referer(request, dataset_id),
        'other_experiments': \
            authz.get_accessible_experiments_for_dataset(request, dataset_id),
        'display_images': display_images,
        'upload_method': upload_method,
        'default_organization':
            getattr(settings, 'DEFAULT_ARCHIVE_ORGANIZATION', 'classic'),
        'default_format':
            getattr(settings, 'DEFAULT_ARCHIVE_FORMATS', ['tgz', 'tar'])[0]
    }
    return HttpResponse(render_response_index(
        request, 'mx_views/view_full_dataset.html', c))


from sourcer import Grammar

# This is work in progress.
# See: https://help.salesforce.com/articleView?id=customize_functions.htm&type=5

g = Grammar(r'''
    ```
    import ast
    ```

    start = Expression

    Expression = OperatorPrecedence(
        Atom | "(" >> Expression << ")",
        Postfix(ArgumentList | FieldAccess),
        Prefix("-" | "+" | "!"),
        RightAssoc("^"),
        LeftAssoc("*" | "/"),
        LeftAssoc("+" | "-" | "&"),
        NonAssoc("<=" | "<" | ">=" | ">"),
        NonAssoc("!=" | "<>" | "==" | "="),
        LeftAssoc("&&"),
        LeftAssoc("||"),
    )

    class ArgumentList {
        arguments: "(" >> (Expression /? ",") << ")"
    }

    class FieldAccess {
        field: "." >> Word
    }

    Atom = Global | Identifier | Rational | Integer | String

    class Global {
        name: "$" >> Word
    }

    class Identifier {
        name: Word
    }

    # ASK: What is the real syntax for these things?
    Word = /[_a-zA-Z][_a-zA-Z0-9]*/
    Rational = /(\d+\.\d*)|(\d*\.\d+)/ |> `float`
    Integer = /\d+/ |> `int`
    StringLiteral = /("([^"\\]|\\.)*")/ | /('([^'\\]|\\.)*')/

    # For now, just use ast module to evaluate string literals.
    class String {
        value: StringLiteral |> `ast.literal_eval`
    }

    ignore /\s+/

''', include_source=True)

aliases = {
    '=': '==',
    '<>': '!=',
}

constants = {
    'NULL': None,
    'TRUE': True,
    'FALSE': False,
}

# Incomplete collection of evaluators.
evaluators = {
    '*': lambda x, y: x * y if x is not None and y is not None else None,
    '/': lambda x, y: x / y if x is not None and y is not None else None,
    '+': lambda x, y: x + y if x is not None and y is not None else None,
    '-': lambda x, y: x - y if x is not None and y is not None else None,
    '==': lambda x, y: x == y,
    '!=': lambda x, y: x != y,
    '&&': lambda x, y: x and y,
    '||': lambda x, y: x or y,
    '>': lambda x, y: x > y if x is not None and y is not None else False,
    '<': lambda x, y: x < y if x is not None and y is not None else False,
    '>=': lambda x, y: x >= y if x is not None and y is not None else False,
    '<=': lambda x, y: x <= y if x is not None and y is not None else False,
    'AND': lambda *a: all(a),
    'CONTAINS': lambda x, y: str(y) in str(x) if x is not None else True,
    'IF': lambda x, y, z: y if x else z,
    'ISBLANK': lambda x: x is None,
    'LOG': lambda x: log10(x) if x is not None else None,
    'MAX': lambda *a: max(*a),
    'MIN': lambda *a: min(*a),
    'MOD': lambda x, y: (x % y) if x is not None and y is not None else None,
    'NOT': lambda x: not(x),
    'OR': lambda *a: any(a),
    'SQRT': lambda x: sqrt(x) if x is not None else None,
    'TEXT': lambda x: str(x),
}

def evaluate(node, bindings):
    # Look up identifiers.
    if isinstance(node, g.Identifier):
        if node.name in bindings:
            return bindings[node.name]

        name = node.name.upper()
        return bindings.get(name, name)

    # Look up fields.
    if isinstance(node, g.Postfix) and isinstance(node.operator, g.FieldAccess):
        obj, field = node.left, node.operator.field
        if hasattr(obj, field):
            return getattr(obj, field)
        elif isinstance(obj, dict):
            return obj.get(field)
        else:
            return node

    # Evaluate function calls and operators.
    if isinstance(node, g.Infix):
        x, func, y = node.left, node.operator, node.right
        args = (x, y)
    elif isinstance(node, g.Postfix) and isinstance(node.operator, g.ArgumentList):
        func, args = node.left, node.operator.arguments
    else:
        return node

    # Check if we're using an alias.
    func = aliases.get(func, func)

    if func in evaluators:
        return evaluators[func](*args)
    else:
        return node

def run(formula, bindings=None):
    updated_bindings = dict(constants)
    updated_bindings.update(bindings or {})
    tree = g.parse(formula)
    return g.transform(tree, lambda node: evaluate(node, updated_bindings))

def test_some_simple_formulas():
    result = run('1 + 2 * 3')
    assert result == 7

    result = run('foo == bar && fiz == buz', bindings={
        'foo': 1, 'bar': 1, 'fiz': 2, 'buz': 2,
    })
    assert result == True

    result = run('foo == bar && fiz == buz', bindings={
        'foo': 1, 'bar': 1, 'fiz': 2, 'buz': 3,
    })
    assert result == False

    result = run('1 <= 2 && (false || true)')
    assert result == True # Explicitly compare to True.

    result = run('1 > 2 || (true && false)')
    assert result == False # Explicitly compare to False.

    result = run('foo != bar', bindings={'foo': 10, 'bar': 10})
    assert not result

    result = run('foo != bar', bindings={'foo': 1, 'bar': 2})
    assert result

    result = run('foo.bar', bindings={'foo': {'bar': 10}})
    assert result == 10

    result = run('foo.bar.baz', bindings={'foo': {'bar': {'baz': 100}}})
    assert result == 100

    result = run('MIN(20, 10, 30)')
    assert result == 10

    result = run('MIN(20, 10, 30) + MAX(11, 12, 13)')
    assert result == 23


## Copyright 2003-2009 Luc Saffre
## This file is part of the TimTools project.
## TimTools is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 3 of the License, or
## (at your option) any later version.
## TimTools is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
## You should have received a copy of the GNU General Public License
## along with TimTools; if not, see <http://www.gnu.org/licenses/>.

import re

def plain2xml(txt):
   txt = txt.replace("&","&")
   txt = txt.replace("<","<")
   return txt

memocommands = (
   ( re.compile('\[url\s+(\S+)\s*(.*?)\]',re.DOTALL),
     lambda m : ''+m.group(2)+' (' + m.group(1)+ ')'),
   )
# urlfind =
# urlrepl = re.compile('\2 (\1)')

# def urlrepl(m):

def memo2xml(txt):
   txt = plain2xml(txt)
   txt = txt.replace('[B]','')
   txt = txt.replace('[b]','')
   txt = txt.replace('[U]','')
   txt = txt.replace('[u]','')
   for find,repl in memocommands:
      txt = re.sub(find,repl,txt)
   return txt

def rst2xml(txt):
   raise "doesn't work"
   import docutils.parsers.rst
   import docutils.utils
   parser = docutils.parsers.rst.Parser()
   doc = docutils.utils.new_document("feed")
   parser.parse(txt, doc)
   raise "and now?"

_feeders={
   'xml' : lambda x : x,
   'plain' : plain2xml,
   'rst' : rst2xml,
   'memo' : memo2xml,
   }

def getFeeder(name):
   return _feeders[name]