signature:
public class HadoopUtils { /** * MapReduce counters are available only with { @ link TaskInputOutputContext } ,
 * but most interfaces use super classes , though the actual object is a
 * subclass ( e . g . Mapper . Context ) .
 * < br >
 * This utility method checks the type and returns the appropriate counter .
 * In the rare ( may be unexpected ) case where ctx is not a
 * TaskInputOutputContext , a dummy counter is returned after printing
 * a warning . */
public static Counter getCounter ( JobContext ctx , String group , String counter ) { } }

implementation: if ( ctx instanceof TaskInputOutputContext < ? , ? , ? , ? > ) { Counter c = HadoopCompat . getCounter ( ( TaskInputOutputContext < ? , ? , ? , ? > ) ctx , group , counter ) ; if ( c != null ) { return c ; } } String name = group + ":" + counter ; LOG . warn ( "Using a dummy counter for " + name + " because it does not already exist." ) ; return HadoopCompat . newGenericCounter ( name , name , 0 ) ;


signature:
public class HadoopUtils { /** * A helper to set configuration to class name .
 * Throws a RuntimeExcpetion if the
 * configuration is already set to a different class name . */
public static void setClassConf ( Configuration conf , String configKey , Class < ? > clazz ) { } }

implementation: String existingClass = conf . get ( configKey ) ; String className = clazz . getName ( ) ; if ( existingClass != null && ! existingClass . equals ( className ) ) { throw new RuntimeException ( "Already registered a different thriftClass for " + configKey + ". old: " + existingClass + " new: " + className ) ; } else { conf . set ( configKey , className ) ; }


signature:
public class HadoopUtils { /** * Writes an object into a configuration by converting it to a base64 encoded string .
 * This is done by serializing the object to bytes via an { @ link ObjectOutputStream } ,
 * gzip compressing those bytes , and then base64 encoding the compressed bytes .
 * NOTE : obj must implement { @ link java . io . Serializable }
 * @ param key for the configuration
 * @ param obj to write ( must be Serializable )
 * @ param conf to write to
 * @ throws IOException */
public static void writeObjectToConfAsBase64 ( String key , Object obj , Configuration conf ) throws IOException { } }

implementation: ByteArrayOutputStream baos = null ; GZIPOutputStream gos = null ; ObjectOutputStream oos = null ; try { baos = new ByteArrayOutputStream ( ) ; gos = new GZIPOutputStream ( baos ) ; oos = new ObjectOutputStream ( gos ) ; oos . writeObject ( obj ) ; } finally { Closeables . close ( oos , false ) ; Closeables . close ( gos , false ) ; Closeables . close ( baos , false ) ; } conf . set ( key , new String ( Base64 . encodeBase64 ( baos . toByteArray ( ) ) , Charsets . UTF_8 ) ) ;


signature:
public class HadoopUtils { /** * Reads an object ( that was written using
 * { @ link # writeObjectToConfAsBase64 } ) from a configuration .
 * @ param key for the configuration
 * @ param conf to read from
 * @ return the read object , or null if key is not present in conf
 * @ throws IOException */
@ SuppressWarnings ( "unchecked" ) public static < T > T readObjectFromConfAsBase64 ( String key , Configuration conf ) throws IOException { } }

implementation: String b64 = conf . get ( key ) ; if ( b64 == null ) { return null ; } byte [ ] bytes = Base64 . decodeBase64 ( b64 . getBytes ( Charsets . UTF_8 ) ) ; ByteArrayInputStream bais = null ; GZIPInputStream gis = null ; ObjectInputStream ois = null ; try { bais = new ByteArrayInputStream ( bytes ) ; gis = new GZIPInputStream ( bais ) ; ois = new ObjectInputStream ( gis ) ; return ( T ) ois . readObject ( ) ; } catch ( ClassNotFoundException e ) { LOG . error ( "Could not read object from config with key " + key , e ) ; throw new IOException ( e ) ; } catch ( ClassCastException e ) { LOG . error ( "Couldn't cast object read from config with key " + key , e ) ; throw new IOException ( e ) ; } finally { Closeables . close ( ois , false ) ; Closeables . close ( gis , false ) ; Closeables . close ( bais , false ) ; }


signature:
public class HadoopUtils { /** * Writes a list of strings into a configuration by converting it to a json array
 * @ param key for the configuration
 * @ param list to write
 * @ param conf to write to */
public static void writeStringListToConfAsJson ( String key , List < String > list , Configuration conf ) { } }

implementation: Preconditions . checkNotNull ( list ) ; conf . set ( key , JSONArray . toJSONString ( list ) ) ;


signature:
public class HadoopUtils { /** * Reads a list of strings stored as a json array from a configuration
 * @ param key for the configuration
 * @ param conf to read from
 * @ return the read list of strings , or null if key is not present in conf */
@ SuppressWarnings ( "unchecked" ) public static List < String > readStringListFromConfAsJson ( String key , Configuration conf ) { } }

implementation: String json = conf . get ( key ) ; if ( json == null ) { return null ; } return Lists . < String > newArrayList ( ( ( JSONArray ) JSONValue . parse ( json ) ) ) ;


signature:
public class HadoopUtils { /** * Writes a list of strings into a configuration by base64 encoding them and separating
 * them with commas
 * @ param key for the configuration
 * @ param list to write
 * @ param conf to write to */
public static void writeStringListToConfAsBase64 ( String key , List < String > list , Configuration conf ) { } }

implementation: Preconditions . checkNotNull ( list ) ; Iterator < String > iter = list . iterator ( ) ; StringBuilder sb = new StringBuilder ( ) ; while ( iter . hasNext ( ) ) { byte [ ] bytes = Base64 . encodeBase64 ( iter . next ( ) . getBytes ( Charsets . UTF_8 ) , false ) ; sb . append ( new String ( bytes , Charsets . UTF_8 ) ) ; if ( iter . hasNext ( ) ) { sb . append ( ',' ) ; } } conf . set ( key , sb . toString ( ) ) ;


signature:
public class HadoopUtils { /** * Reads a list of strings stored as comma separated base64
 * @ param key for the configuration
 * @ param conf to read from
 * @ return the read list of strings , or null if key is not present in conf */
@ SuppressWarnings ( "unchecked" ) public static List < String > readStringListFromConfAsBase64 ( String key , Configuration conf ) { } }

implementation: String b64List = conf . get ( key ) ; if ( b64List == null ) { return null ; } List < String > strings = Lists . newArrayList ( ) ; for ( String b64 : COMMA_SPLITTER . split ( b64List ) ) { byte [ ] bytes = Base64 . decodeBase64 ( b64 . getBytes ( Charsets . UTF_8 ) ) ; strings . add ( new String ( bytes , Charsets . UTF_8 ) ) ; } return strings ;


signature:
public class W3CLogParser { /** * < FieldName , Value > */
public Map < String , String > parse ( String line ) throws IOException { } }

implementation:
Map < String , String > map = new HashMap < String , String > ( ) ; // Get the version CRC and find the field definition
List < String > fields = Arrays . asList ( line . split ( DELIMITER ) ) ; List < String > fieldNames = fieldDef_ . get ( fields . get ( 0 ) ) ; if ( fieldNames == null ) { throw new IOException ( "cannot find matching field definition for CRC " + fields . get ( 0 ) ) ; } if ( fieldNames . size ( ) != fields . size ( ) ) { throw new IOException ( "W3C field definition and input line for CRC " + fields . get ( 0 ) + " does not match:\n" + line ) ; } // Map values to field names
for ( int fieldNum = 1 ; fieldNum < fieldNames . size ( ) ; fieldNum ++ ) { map . put ( fieldNames . get ( fieldNum ) , fields . get ( fieldNum ) ) ; } return map ;


signature:
public class ThriftUtils { /** * Verify that clazz is a Thrift class . i . e . is a subclass of TBase */
private static void verifyAncestry ( Class < ? > tClass ) { } }

implementation: if ( ! TBase . class . isAssignableFrom ( tClass ) ) { Utils . ensureClassLoaderConsistency ( TBase . class , tClass . getClassLoader ( ) ) ; throw new ClassCastException ( tClass . getName ( ) + " is not a Thrift class" ) ; }


signature:
public class ThriftUtils { /** * Returns TypeRef for the Thrift class that was set using setClass ( jobConf ) ; */
public static < M extends TBase < ? , ? > > TypeRef < M > getTypeRef ( Configuration jobConf , Class < ? > genericClass ) { } }

implementation: String className = jobConf . get ( CLASS_CONF_PREFIX + genericClass . getName ( ) ) ; if ( className == null ) { throw new RuntimeException ( CLASS_CONF_PREFIX + genericClass . getName ( ) + " is not set" ) ; } Class < ? > tClass = null ; try { tClass = jobConf . getClassByName ( className ) ; } catch ( ClassNotFoundException e ) { throw new RuntimeException ( e ) ; } verifyAncestry ( tClass ) ; return new TypeRef < M > ( tClass ) { } ;


signature:
public class ThriftUtils { /** * returns TypeRef for thrift class . Verifies that the class is
 * a Thrift class ( i . e extends TBase ) . */
public static < M extends TBase < ? , ? > > TypeRef < M > getTypeRef ( Class < ? > tClass ) { } }

implementation: verifyAncestry ( tClass ) ; return new TypeRef < M > ( tClass ) { } ;


signature:
public class ThriftUtils { /** * returns TypeRef for a thrift class . */
public static < M extends TBase < ? , ? > > TypeRef < M > getTypeRef ( String thriftClassName , ClassLoader classLoader ) { } }

implementation: try { Class < ? > tClass = classLoader == null ? Class . forName ( thriftClassName ) : Class . forName ( thriftClassName , true , classLoader ) ; verifyAncestry ( tClass ) ; return new TypeRef < M > ( tClass ) { } ; } catch ( ClassNotFoundException e ) { throw new RuntimeException ( e ) ; }


signature:
public class ThriftUtils { /** * returns TypeRef for a thrift class . */
public static < M extends TBase < ? , ? > > TypeRef < M > getTypeRef ( String thriftClassName ) { } }

implementation: return getTypeRef ( thriftClassName , null ) ;


signature:
public class ThriftUtils { /** * Returns value of a fieldName in an object . */
public static < M > M getFieldValue ( Object containingObject , String fieldName , Class < M > fieldClass ) { } }

implementation: return getFieldValue ( containingObject . getClass ( ) , containingObject , fieldName , fieldClass ) ;


signature:
public class ThriftUtils { /** * Returns value of a static field with given name in containingClass . */
public static < M > M getFieldValue ( Class < ? > containingClass , String fieldName , Class < M > fieldClass ) { } }

implementation: return getFieldValue ( containingClass , null , fieldName , fieldClass ) ;


signature:
public class ThriftUtils { /** * Returns generic type for a field in a Thrift class . The type is the return
 * type for the accessor method for the field ( e . g . < code > isFieldName ( ) < / code >
 * for a boolean type or < code > getFieldName < / code > for other types ) . The return
 * type works for both structs and unions . Reflecting directly based on
 * fields does not work for unions .
 * @ return generic { @ link Type } of the thrift field . */
public static Type getFieldType ( Class < ? > containingClass , String fieldName ) { } }

implementation:
String suffix = // uppercase first letter
fieldName . substring ( 0 , 1 ) . toUpperCase ( ) + fieldName . substring ( 1 ) ; // look for getFieldName ( ) or isFieldName ( )
for ( String prefix : new String [ ] { "get" , "is" } ) { try { Method method = containingClass . getDeclaredMethod ( prefix + suffix ) ; return method . getGenericReturnType ( ) ; } catch ( NoSuchMethodException e ) { } } // look for bean style accessors get _ fieldName and is _ fieldName
for ( String prefix : new String [ ] { "get_" , "is_" } ) { try { Method method = containingClass . getDeclaredMethod ( prefix + fieldName ) ; return method . getGenericReturnType ( ) ; } catch ( NoSuchMethodException e ) { } } throw new RuntimeException ( "could not find type for " + fieldName + " in " + containingClass ) ;


signature:
public class ThriftUtils { /** * Returns the value class of the given thrift field .
 * @ param field thrift field .
 * @ return class of field value , or null in the case of thrift types { @ link TType # STOP } or
 * { @ link TType # VOID } . */
public static Class < ? > getFieldValueType ( Field field ) { } }

implementation: switch ( field . getType ( ) ) { case TType . BOOL : return Boolean . class ; case TType . BYTE : return Byte . class ; case TType . DOUBLE : return Double . class ; case TType . ENUM : return field . getEnumClass ( ) ; case TType . I16 : return Short . class ; case TType . I32 : return Integer . class ; case TType . I64 : return Long . class ; case TType . LIST : return List . class ; case TType . MAP : return Map . class ; case TType . SET : return Set . class ; case TType . STOP : return null ; case TType . STRING : return String . class ; case TType . STRUCT : return field . gettStructDescriptor ( ) . getThriftClass ( ) ; case TType . VOID : return null ; } return null ;


signature:
public class ThriftUtils { /** * Serializes a single field of a thrift struct .
 * @ throws TException */
public static void writeFieldNoTag ( TProtocol proto , Field field , Object value ) throws TException { } }

implementation:
if ( value == null ) { return ; } Field innerField = null ; switch ( field . getType ( ) ) { case TType . LIST : innerField = field . getListElemField ( ) ; break ; case TType . SET : innerField = field . getSetElemField ( ) ; break ; case TType . MAP : innerField = field . getMapKeyField ( ) ; break ; default : writeSingleFieldNoTag ( proto , field , value ) ; return ; } // a map or a collection :
if ( field . getType ( ) == TType . MAP ) { Field valueField = field . getMapValueField ( ) ; Map < ? , ? > map = ( Map < ? , ? > ) value ; proto . writeByte ( innerField . getType ( ) ) ; proto . writeByte ( valueField . getType ( ) ) ; proto . writeI32 ( map . size ( ) ) ; for ( Entry < ? , ? > entry : map . entrySet ( ) ) { writeSingleFieldNoTag ( proto , innerField , entry . getKey ( ) ) ; writeSingleFieldNoTag ( proto , valueField , entry . getValue ( ) ) ; } } else { // SET or LIST
Collection < ? > coll = ( Collection < ? > ) value ; proto . writeByte ( innerField . getType ( ) ) ; proto . writeI32 ( coll . size ( ) ) ; for ( Object v : coll ) { writeSingleFieldNoTag ( proto , innerField , v ) ; } }


signature:
public class ThriftUtils { /** * Deserializes a thrift field that was serilized with
 * { @ link # writeFieldNoTag ( TProtocol , Field , Object ) } .
 * @ throws TException in case of any Thrift errors . */
public static Object readFieldNoTag ( TProtocol proto , Field field ) throws TException { } }

implementation:
Collection < Object > coll = null ; Field innerField = null ; switch ( field . getType ( ) ) { case TType . LIST : innerField = field . getListElemField ( ) ; coll = Lists . newArrayList ( ) ; break ; case TType . SET : innerField = field . getSetElemField ( ) ; coll = Sets . newHashSet ( ) ; break ; case TType . MAP : innerField = field . getMapKeyField ( ) ; break ; default : return readSingleFieldNoTag ( proto , field ) ; } // collection or a map :
if ( field . getType ( ) == TType . MAP ) { proto . readByte ( ) ; proto . readByte ( ) ; int nEntries = proto . readI32 ( ) ; Map < Object , Object > map = Maps . newHashMap ( ) ; Field valueField = field . getMapValueField ( ) ; for ( int i = 0 ; i < nEntries ; i ++ ) { map . put ( readFieldNoTag ( proto , innerField ) , readFieldNoTag ( proto , valueField ) ) ; } return map ; } else { // SET or LIST
proto . readByte ( ) ; int nEntries = proto . readI32 ( ) ; for ( int i = 0 ; i < nEntries ; i ++ ) { coll . add ( readFieldNoTag ( proto , innerField ) ) ; } return coll ; }


signature:
public class PigToProtobuf { /** * Turn a Tuple into a Message with the given type .
 * @ param builder a builder for the Message type the tuple will be converted to
 * @ param tuple the tuple
 * @ return a message representing the given tuple */
public static Message tupleToMessage ( Builder builder , Tuple tuple ) { } }

implementation: return tupleToMessage ( builder , builder . getDescriptorForType ( ) . getFields ( ) , tuple ) ;


signature:
public class PigToProtobuf { /** * For a given < code > ResourceSchema < / code > , generate a protobufs < code > Descriptor < / code > with analogous field names
 * and types .
 * @ param schema Pig schema .
 * @ param extraFields optionally pass a List of extra fields ( Pairs of name : type ) to be included .
 * @ return Protobufs Descriptor
 * @ throws Descriptors . DescriptorValidationException */
public static Descriptor schemaToProtoDescriptor ( ResourceSchema schema , List < Pair < String , Type > > extraFields ) throws DescriptorValidationException { } }

implementation:
// init protobufs
DescriptorProto . Builder desBuilder = DescriptorProto . newBuilder ( ) ; int count = 0 ; for ( ResourceFieldSchema fieldSchema : schema . getFields ( ) ) { // Pig types
int position = ++ count ; String fieldName = fieldSchema . getName ( ) ; byte dataTypeId = fieldSchema . getType ( ) ; // determine and add protobuf types
Type protoType = pigTypeToProtoType ( dataTypeId ) ; LOG . info ( "Mapping Pig field " + fieldName + " of type " + dataTypeId + " to protobuf type: " + protoType ) ; addField ( desBuilder , fieldName , position , protoType ) ; } if ( count == 0 ) { throw new IllegalArgumentException ( "ResourceSchema does not have any fields" ) ; } // If extra fields are needed , let ' s add them
if ( extraFields != null ) { for ( Pair < String , Type > extraField : extraFields ) { addField ( desBuilder , extraField . first , ++ count , extraField . second ) ; } } desBuilder . setName ( "PigToProtobufDynamicBuilder" ) ; return Protobufs . makeMessageDescriptor ( desBuilder . build ( ) ) ;


signature:
public class PigToProtobuf { /** * Converts a DataBag into a List of objects with the type in the given FieldDescriptor . DataBags
 * don ' t map cleanly to repeated protobuf types , so each Tuple has to be unwrapped ( by taking the
 * first element if the type is primitive or by converting the Tuple to a Message if the type is
 * MESSAGE ) , and the contents have to be appended to a List .
 * @ param containingMessageBuilder a Message builder for the Message that contains this repeated field
 * @ param fieldDescriptor a FieldDescriptor for this repeated field
 * @ param bag the DataBag being serialized
 * @ return a protobuf - friendly List of fieldDescriptor - type objects */
private static List < Object > dataBagToRepeatedField ( Builder containingMessageBuilder , FieldDescriptor fieldDescriptor , DataBag bag ) { } }

implementation: ArrayList < Object > bagContents = new ArrayList < Object > ( ( int ) bag . size ( ) ) ; Iterator < Tuple > bagIter = bag . iterator ( ) ; while ( bagIter . hasNext ( ) ) { Tuple tuple = bagIter . next ( ) ; if ( fieldDescriptor . getType ( ) == FieldDescriptor . Type . MESSAGE ) { Builder nestedMessageBuilder = containingMessageBuilder . newBuilderForField ( fieldDescriptor ) ; bagContents . add ( tupleToMessage ( ( Builder ) nestedMessageBuilder , tuple ) ) ; } else { try { bagContents . add ( tupleFieldToSingleField ( fieldDescriptor , tuple . get ( 0 ) ) ) ; } catch ( ExecException e ) { LOG . warn ( "Could not add a value for repeated field with descriptor " + fieldDescriptor ) ; } } } return bagContents ;


signature:
public class PigToProtobuf { /** * Converts a tupleField string to its corresponding protobuf enum type if necessary , otherwise
 * returns the tupleField as is .
 * @ param fieldDescriptor the FieldDescriptor for the given tuple field
 * @ param tupleField the tupleField being converted to a protobuf field
 * @ return the protobuf type for the given tupleField . This will be the tupleField itself unless it ' s an enum , in which case this will return the enum type for the field . */
private static Object tupleFieldToSingleField ( FieldDescriptor fieldDescriptor , Object tupleField ) { } }

implementation:
// type convertion should match with ProtobufToPig . getPigScriptDataType
switch ( fieldDescriptor . getType ( ) ) { case ENUM : return toEnumValueDescriptor ( fieldDescriptor , ( String ) tupleField ) ; case BOOL : return Boolean . valueOf ( ( Integer ) tupleField != 0 ) ; case BYTES : return ByteString . copyFrom ( ( ( DataByteArray ) tupleField ) . get ( ) ) ; default : return tupleField ; }


signature:
public class PigToProtobuf { /** * Add a field to a protobuf builder */
private static void addField ( DescriptorProto . Builder builder , String name , int fieldId , Type type ) { } }

implementation: FieldDescriptorProto . Builder fdBuilder = FieldDescriptorProto . newBuilder ( ) . setName ( name ) . setNumber ( fieldId ) . setType ( type ) ; builder . addField ( fdBuilder . build ( ) ) ;


signature:
public class PigToProtobuf { /** * For a given Pig type , return the protobufs type that maps to it . */
private static Type pigTypeToProtoType ( byte pigTypeId ) { } }

implementation: switch ( pigTypeId ) { case DataType . BOOLEAN : return Type . TYPE_BOOL ; case DataType . INTEGER : return Type . TYPE_INT32 ; case DataType . LONG : return Type . TYPE_INT64 ; case DataType . FLOAT : return Type . TYPE_FLOAT ; case DataType . DOUBLE : return Type . TYPE_DOUBLE ; case DataType . CHARARRAY : return Type . TYPE_STRING ; case DataType . BYTEARRAY : return Type . TYPE_BYTES ; default : throw new IllegalArgumentException ( "Unsupported Pig type passed (" + pigTypeId + ") where a simple type is expected while converting Pig to a dynamic Protobuf" ) ; }


signature:
public class JsonLoader { /** * Return every non - null line as a single - element tuple to Pig . */
@ Override public Tuple getNext ( ) throws IOException { } }

implementation:
if ( reader == null ) { return null ; } // nested load can be enabled through a pig property
if ( "true" . equals ( jobConf . get ( NESTED_LOAD_KEY ) ) ) isNestedLoadEnabled = true ; try { while ( reader . nextKeyValue ( ) ) { Text value = ( Text ) reader . getCurrentValue ( ) ; incrCounter ( JsonLoaderCounters . LinesRead , 1L ) ; Tuple t = parseStringToTuple ( value . toString ( ) ) ; if ( t != null ) { incrCounter ( JsonLoaderCounters . LinesJsonDecoded , 1L ) ; return t ; } } return null ; } catch ( InterruptedException e ) { int errCode = 6018 ; String errMsg = "Error while reading input" ; throw new ExecException ( errMsg , errCode , PigException . REMOTE_ENVIRONMENT , e ) ; }


signature:
public class PigToThrift { /** * Construct a Thrift object from the tuple . */
@ SuppressWarnings ( "unchecked" ) private static TBase < ? , ? > toThrift ( TStructDescriptor tDesc , Tuple tuple ) { } }

implementation:
int size = tDesc . getFields ( ) . size ( ) ; int tupleSize = tuple . size ( ) ; @ SuppressWarnings ( "rawtypes" ) TBase tObj = newTInstance ( tDesc . getThriftClass ( ) ) ; for ( int i = 0 ; i < size && i < tupleSize ; i ++ ) { Object pObj ; try { pObj = tuple . get ( i ) ; } catch ( ExecException e ) { throw new RuntimeException ( e ) ; } if ( pObj != null ) { Field field = tDesc . getFieldAt ( i ) ; try { tObj . setFieldValue ( field . getFieldIdEnum ( ) , toThriftValue ( field , pObj ) ) ; } catch ( Exception e ) { String value = String . valueOf ( tObj ) ; final int max_length = 100 ; if ( max_length < value . length ( ) ) { value = value . substring ( 0 , max_length - 3 ) + "..." ; } String type = tObj == null ? "unknown" : tObj . getClass ( ) . getName ( ) ; throw new RuntimeException ( String . format ( "Failed to set field '%s' using tuple value '%s' of type '%s' at index %d" , field . getName ( ) , value , type , i ) , e ) ; } } // if tDesc is a union , at least one field needs to be non - null .
// user is responsible for ensuring that .
} return tObj ;


signature:
public class PigToThrift { /** * For a given Pig value , return a Thrift object of the same type as the Thrift field passed . The
 * thrift field is expected to be compatible with the value passed . If it is not , a warning will
 * be logged and a null value will be returned .
 * @ param thriftField the Thrift field used to determine the type of the response object
 * @ param pigValue the value to convert to Thrift
 * @ return a Thrift object */
@ SuppressWarnings ( "unchecked" ) public static Object toThriftValue ( Field thriftField , Object pigValue ) { } }

implementation:
try { switch ( thriftField . getType ( ) ) { case TType . BOOL : return Boolean . valueOf ( ( ( Integer ) pigValue ) != 0 ) ; case TType . BYTE : return ( ( Integer ) pigValue ) . byteValue ( ) ; case TType . I16 : return Short . valueOf ( ( ( Integer ) pigValue ) . shortValue ( ) ) ; case TType . STRING : return toStringType ( pigValue ) ; case TType . STRUCT : return toThrift ( thriftField . gettStructDescriptor ( ) , ( Tuple ) pigValue ) ; case TType . MAP : return toThriftMap ( thriftField , ( Map < String , Object > ) pigValue ) ; case TType . SET : return toThriftSet ( thriftField . getSetElemField ( ) , ( DataBag ) pigValue ) ; case TType . LIST : return toThriftList ( thriftField . getListElemField ( ) , ( DataBag ) pigValue ) ; case TType . ENUM : return toThriftEnum ( thriftField , ( String ) pigValue ) ; default : // standard types : I32 , I64 , DOUBLE , etc .
return pigValue ; } } catch ( Exception e ) { // mostly a schema mismatch .
LOG . warn ( String . format ( "Failed to set field '%s' of type '%s' with value '%s' of type '%s'" , thriftField . getName ( ) , ThriftUtils . getFieldValueType ( thriftField ) . getName ( ) , pigValue , pigValue . getClass ( ) . getName ( ) ) , e ) ; } return null ;


signature:
public class PigToThrift { /** * / * TType . STRING could be either a DataByteArray or a String */
private static Object toStringType ( Object value ) { } }

implementation:
if ( value instanceof String ) { return value ; } else if ( value instanceof DataByteArray ) { byte [ ] buf = ( ( DataByteArray ) value ) . get ( ) ; // mostly there is no need to copy .
return ByteBuffer . wrap ( Arrays . copyOf ( buf , buf . length ) ) ; } return null ;


signature:
public class PigToThrift { /** * return an instance assuming tClass is a Thrift class */
private static TBase < ? , ? > newTInstance ( Class < ? > tClass ) { } }

implementation:
try { return ( TBase < ? , ? > ) tClass . newInstance ( ) ; } catch ( Exception e ) { // not expected .
throw new RuntimeException ( e ) ; }


signature:
public class DelegateCombineFileInputFormat { /** * This sets configures the delegate , though it does not configure DelegateCombineFileInputFormat . */
public static void setCombinedInputFormatDelegate ( Configuration conf , Class < ? extends InputFormat > clazz ) { } }

implementation: HadoopUtils . setClassConf ( conf , COMBINED_INPUT_FORMAT_DELEGATE , clazz ) ;


signature:
public class LuceneIndexInputFormat { /** * Creates splits with multiple indexes per split
 * ( if they are smaller than maxCombinedIndexSizePerSplit ) .
 * It is possible for a split to be larger than maxCombinedIndexSizePerSplit ,
 * if it consists of a single index that is
 * larger than maxCombinedIndexSizePerSplit .
 * All inputPaths will be searched for indexes recursively
 * The bin - packing problem of combining splits is solved naively :
 * < ol >
 * < li > Sort all indexes by size < / li >
 * < li > Begin packing indexes into splits until adding the next split would cause the split to
 * exceed maxCombinedIndexSizePerSplit < / li >
 * < li > Begin packing subsequent indexes into the next split , and so on < / li >
 * < / ol > */
@ Override public List < InputSplit > getSplits ( JobContext job ) throws IOException , InterruptedException { } }

implementation:
// load settings from job conf
loadConfig ( HadoopCompat . getConfiguration ( job ) ) ; // find all the index dirs and create a split for each
PriorityQueue < LuceneIndexInputSplit > splits = findSplits ( HadoopCompat . getConfiguration ( job ) ) ; // combine the splits based on maxCombineSplitSize
List < InputSplit > combinedSplits = combineSplits ( splits , maxCombinedIndexSizePerSplit , maxNumIndexesPerSplit ) ; return combinedSplits ;


signature:
public class LuceneIndexInputFormat { /** * Finds and creates all the index splits based on the input paths set in conf
 * @ param conf job conf
 * @ return a priority queue of the splits , default is sorted by size
 * @ throws IOException */
protected PriorityQueue < LuceneIndexInputSplit > findSplits ( Configuration conf ) throws IOException { } }

implementation:
PriorityQueue < LuceneIndexInputSplit > splits = new PriorityQueue < LuceneIndexInputSplit > ( ) ; List < Path > indexDirs = Lists . newLinkedList ( ) ; // find all indexes nested under all the input paths
// ( which happen to be directories themselves )
for ( Path path : inputPaths ) { HdfsUtils . collectPaths ( path , path . getFileSystem ( conf ) , indexDirPathFilter , indexDirs ) ; } // compute the size of each index
// and create a single split per index
for ( Path indexDir : indexDirs ) { long size = HdfsUtils . getDirectorySize ( indexDir , indexDir . getFileSystem ( conf ) ) ; splits . add ( new LuceneIndexInputSplit ( Lists . newLinkedList ( Arrays . asList ( indexDir ) ) , size ) ) ; } return splits ;


signature:
public class LuceneIndexInputFormat { /** * Set the queries to run over the indexes
 * These can be Strings suitable for parsing with a QueryParser , or they can be
 * a custom serialized Query object . They have to be Strings so that they can be
 * written to the job conf . They will be deserialized / parsed by the abstract
 * method { @ link LuceneIndexRecordReader # deserializeQuery ( String ) }
 * @ param queries queries to run over the indexes
 * @ param conf job conf
 * @ throws IOException */
public static void setQueries ( List < String > queries , Configuration conf ) throws IOException { } }

implementation: Preconditions . checkNotNull ( queries ) ; Preconditions . checkArgument ( ! queries . isEmpty ( ) ) ; HadoopUtils . writeStringListToConfAsBase64 ( QUERIES_KEY , queries , conf ) ;


signature:
public class LuceneIndexInputFormat { /** * Get the queries to run over the indexes
 * These are the queries in the same form as they were passed to { @ link # setQueries } above
 * @ param conf job conf
 * @ return queries as passed to { @ link # setQueries }
 * @ throws IOException */
public static List < String > getQueries ( Configuration conf ) throws IOException { } }

implementation: return Preconditions . checkNotNull ( HadoopUtils . readStringListFromConfAsBase64 ( QUERIES_KEY , conf ) , "You must call LuceneIndexInputFormat.setQueries()" ) ;


signature:
public class LuceneIndexInputFormat { /** * Sets the input paths for for this input format .
 * All paths will be searched for indexes recursively
 * @ param paths the input paths
 * @ param conf the job conf
 * @ throws IOException */
public static void setInputPaths ( List < Path > paths , Configuration conf ) throws IOException { } }

implementation: Preconditions . checkNotNull ( paths ) ; Preconditions . checkArgument ( ! paths . isEmpty ( ) ) ; String [ ] pathStrs = new String [ paths . size ( ) ] ; int i = 0 ; for ( Path p : paths ) { FileSystem fs = p . getFileSystem ( conf ) ; pathStrs [ i ++ ] = fs . makeQualified ( p ) . toString ( ) ; } conf . setStrings ( INPUT_PATHS_KEY , pathStrs ) ;


signature:
public class LuceneIndexInputFormat { /** * Gets the input paths for this input format
 * @ param conf the job conf */
public static Path [ ] getInputPaths ( Configuration conf ) { } }

implementation: String [ ] pathStrs = Preconditions . checkNotNull ( conf . getStrings ( INPUT_PATHS_KEY ) , "You must call LuceneIndexInputFormat.setInputPaths()" ) ; Path [ ] paths = new Path [ pathStrs . length ] ; for ( int i = 0 ; i < pathStrs . length ; i ++ ) { paths [ i ] = new Path ( pathStrs [ i ] ) ; } return paths ;


signature:
public class LuceneIndexCollectAllRecordReader { /** * Applies { @ link # docToValue ( Document ) } to every document
 * found by executing query over searcher
 * @ param searcher the index searcher to query
 * @ param query the query to run
 * @ return a list of values to be emitted as records ( one by one ) by this record reader
 * @ throws IOException */
@ Override protected Iterator < T > search ( final IndexSearcher searcher , final Query query ) throws IOException { } }

implementation:
// grow the bit set if needed
docIds . set ( searcher . getIndexReader ( ) . maxDoc ( ) ) ; // clear it
docIds . clear ( ) ; searcher . search ( query , new Collector ( ) { private int docBase ; @ Override public void setScorer ( Scorer scorer ) { } @ Override public boolean acceptsDocsOutOfOrder ( ) { return true ; } @ Override public void collect ( int doc ) { docIds . set ( doc + docBase ) ; } public void setNextReader ( AtomicReaderContext context ) { this . docBase = context . docBase ; } } ) ; return new AbstractIterator < T > ( ) { private int doc = docIds . nextSetBit ( 0 ) ; @ Override protected T computeNext ( ) { ( ( Progressable ) context ) . progress ( ) ; // casting to avoid Hadoop 2 incompatibility
if ( doc < 0 ) { return endOfData ( ) ; } try { T ret = docToValue ( searcher . doc ( doc ) ) ; doc = docIds . nextSetBit ( doc + 1 ) ; return ret ; } catch ( IOException e ) { throw new RuntimeException ( e ) ; } } } ;


signature:
public class CompositeInputSplit { /** * Add an InputSplit to this collection . */
public void add ( InputSplit split ) throws IOException , InterruptedException { } }

implementation: splits . add ( split ) ; totalSplitSizes += split . getLength ( ) ; locations = null ;


signature:
public class CompositeInputSplit { /** * Collect a set of hosts from all child InputSplits . Note that this is just a hint to the MapReduce framework
 * for where the task should go , so we return the top 5 , as returning too many can be expensive on the MR side
 * of things . */
public String [ ] getLocations ( ) throws IOException , InterruptedException { } }

implementation: if ( locations == null ) { Map < String , Integer > hosts = new HashMap < String , Integer > ( ) ; for ( InputSplit s : splits ) { String [ ] hints = s . getLocations ( ) ; if ( hints != null ) { for ( String host : hints ) { Integer value = hosts . get ( host ) ; if ( value == null ) { value = 0 ; } value ++ ; hosts . put ( host , value ) ; } } } if ( hosts . size ( ) < 5 ) { locations = hosts . keySet ( ) . toArray ( new String [ hosts . size ( ) ] ) ; } else { Queue < Pair < String , Integer > > queue = new PriorityQueue < Pair < String , Integer > > ( hosts . size ( ) , new Comparator < Pair < String , Integer > > ( ) { public int compare ( Pair < String , Integer > o1 , Pair < String , Integer > o2 ) { return - o1 . getSecond ( ) . compareTo ( o2 . getSecond ( ) ) ; } } ) ; for ( Map . Entry < String , Integer > entry : hosts . entrySet ( ) ) { queue . add ( new Pair < String , Integer > ( entry . getKey ( ) , entry . getValue ( ) ) ) ; } locations = new String [ ] { queue . remove ( ) . getFirst ( ) , queue . remove ( ) . getFirst ( ) , queue . remove ( ) . getFirst ( ) , queue . remove ( ) . getFirst ( ) , queue . remove ( ) . getFirst ( ) } ; } } return locations ;


signature:
public class CompositeInputSplit { /** * Write splits in the following format .
 * { @ code
 * < count > < class1 > < split1 > < class2 > < split2 > . . . < classn > < splitn > */
@ SuppressWarnings ( "unchecked" ) public void write ( DataOutput out ) throws IOException { } }

implementation: if ( conf == null ) { throw new IOException ( "Hadoop Configuration not set via setConf" ) ; } WritableUtils . writeVInt ( out , splits . size ( ) ) ; for ( InputSplit split : splits ) { SplitUtil . serializeInputSplit ( conf , ( DataOutputStream ) out , split ) ; }


signature:
public class CompositeInputSplit { /** * { @ inheritDoc }
 * @ throws IOException If the child InputSplit cannot be read , typically
 * for failing access checks . */
@ SuppressWarnings ( "unchecked" ) // Generic array assignment
public void readFields ( DataInput in ) throws IOException { } }

implementation: if ( conf == null ) { throw new IOException ( "Hadoop Configuration not set via setConf" ) ; } int card = WritableUtils . readVInt ( in ) ; splits = new ArrayList < InputSplit > ( ) ; for ( int i = 0 ; i < card ; ++ i ) { splits . add ( SplitUtil . deserializeInputSplit ( conf , ( DataInputStream ) in ) ) ; }


signature:
public class PigTokenHelper { /** * Parse an input delimiter string , as with PigStorage , and return the byte it represents .
 * @ param inputDelimiter the string passed in from the pig script .
 * @ return the corresponding byte that will serve as the field separator . */
public static byte evaluateDelimiter ( String inputDelimiter ) { } }

implementation: if ( inputDelimiter . length ( ) == 1 ) { return inputDelimiter . getBytes ( ) [ 0 ] ; } else if ( inputDelimiter . length ( ) > 1 && inputDelimiter . charAt ( 0 ) == '\\' ) { switch ( inputDelimiter . charAt ( 1 ) ) { case 't' : return ( byte ) '\t' ; case 'x' : case 'u' : return Integer . valueOf ( inputDelimiter . substring ( 2 ) ) . byteValue ( ) ; default : throw new IllegalArgumentException ( "Unknown delimiter " + inputDelimiter ) ; } } else { throw new IllegalArgumentException ( "LzoTokenizedStorage delimeter must be a single character" ) ; }


signature:
public class ResourceSchemaUtil { /** * Creates a new ResourceFieldSchema which reflects data from an input RequiredField .
 * @ param field
 * @ return new ResourceFieldSchema which reflects { @ code field } .
 * @ throws IOException */
public static ResourceFieldSchema createResourceFieldSchema ( RequiredField field ) throws IOException { } }

implementation: ResourceFieldSchema schema = new ResourceFieldSchema ( ) . setName ( field . getAlias ( ) ) . setType ( field . getType ( ) ) ; List < RequiredField > subFields = field . getSubFields ( ) ; if ( subFields != null && ! subFields . isEmpty ( ) ) { ResourceFieldSchema [ ] subSchemaFields = new ResourceFieldSchema [ subFields . size ( ) ] ; int i = 0 ; for ( RequiredField subField : subFields ) { subSchemaFields [ i ++ ] = createResourceFieldSchema ( subField ) ; } ResourceSchema subSchema = new ResourceSchema ( ) ; subSchema . setFields ( subSchemaFields ) ; schema . setSchema ( subSchema ) ; } return schema ;


signature:
public class ProtobufComparator { /** * Method used to read a protobuf input stream into a byte array
 * @ param in the protobuf input stream
 * @ param out the byte output stream
 * @ param buffer the array out output bytes */
public static void readFully ( InputStream in , ByteArrayOutputStream out , byte [ ] buffer ) { } }

implementation: try { int numRead = 0 ; while ( ( numRead = in . read ( buffer , 0 , buffer . length ) ) != - 1 ) { out . write ( buffer , 0 , numRead ) ; } out . flush ( ) ; } catch ( IOException e ) { throw new RuntimeException ( e ) ; }


signature:
public class LzoBinaryB64LineRecordReader { /** * Read the next key , value pair .
 * A small fraction of bad records in input are tolerated .
 * See { @ link LzoRecordReader } for more information on error handling .
 * @ return true if a key / value pair was read
 * @ throws IOException
 * @ throws InterruptedException */
@ Override public boolean nextKeyValue ( ) throws IOException , InterruptedException { } }

implementation:
// Since the lzop codec reads everything in lzo blocks , we can ' t stop if pos = = end .
// Instead we wait for the next block to be read in , when pos will be > end .
while ( pos_ <= end_ ) { key_ . set ( pos_ ) ; int newSize = lineReader_ . readLine ( line_ , maxLineLen ) ; if ( newSize == 0 ) { return false ; } HadoopCompat . incrementCounter ( linesReadCounter , 1 ) ; pos_ = getLzoFilePos ( ) ; if ( line_ . getLength ( ) == 0 || line_ . charAt ( 0 ) == '\n' ) { HadoopCompat . incrementCounter ( emptyLinesCounter , 1 ) ; continue ; } if ( line_ . getLength ( ) >= maxLineLen ) { HadoopCompat . incrementCounter ( truncatedLinesCounter , 1 ) ; } M protoValue = null ; errorTracker . incRecords ( ) ; try { protoValue = converter_ . fromBytes ( Base64Codec . decodeFast ( line_ . getBytes ( ) , line_ . getLength ( ) ) ) ; } catch ( DecodeException t1 ) { HadoopCompat . incrementCounter ( recordErrorsCounter , 1 ) ; errorTracker . incErrors ( t1 ) ; } if ( protoValue == null ) { HadoopCompat . incrementCounter ( recordsSkippedCounter , 1 ) ; continue ; } HadoopCompat . incrementCounter ( recordsReadCounter , 1 ) ; value_ . set ( protoValue ) ; return true ; } return false ;


signature:
public class LzoBaseRegexLoader { /** * Read the file line by line , returning lines the match the regex in Tuples
 * based on the regex match groups . */
@ Override public Tuple getNext ( ) throws IOException { } }

implementation:
if ( reader == null ) { return null ; } Pattern pattern = getPattern ( ) ; Matcher matcher = pattern . matcher ( "" ) ; Object lineObj ; String line ; Tuple t = null ; // Read lines until a match is found , making sure there ' s no reading past the
// end of the assigned byte range .
try { while ( reader . nextKeyValue ( ) ) { lineObj = reader . getCurrentValue ( ) ; if ( lineObj == null ) { break ; } line = lineObj . toString ( ) ; matcher = matcher . reset ( line ) ; // Increment counters for the number of matched and unmatched lines .
if ( matcher . find ( ) ) { incrCounter ( LzoBaseRegexLoaderCounters . MatchedRegexLines , 1L ) ; t = tupleFactory_ . newTuple ( matcher . groupCount ( ) ) ; for ( int i = 1 ; i <= matcher . groupCount ( ) ; i ++ ) { if ( matcher . group ( i ) != null ) { t . set ( i - 1 , matcher . group ( i ) ) ; } else { t . set ( i - 1 , "" ) ; } } break ; } else { incrCounter ( LzoBaseRegexLoaderCounters . UnmatchedRegexLines , 1L ) ; // TODO : stop doing this , as it can slow down the job .
LOG . debug ( "No match for line " + line ) ; } // If the read has walked beyond the end of the split , move on .
} } catch ( InterruptedException e ) { int errCode = 6018 ; String errMsg = "Error while reading input" ; throw new ExecException ( errMsg , errCode , PigException . REMOTE_ENVIRONMENT , e ) ; } return t ;


signature:
public class Codecs { /** * Get a instance of standard base64 implementation from apache
 * commons - codec library
 * @ return standard base64 instance */
public static Base64 createStandardBase64 ( ) { } }

implementation:
/* with constructor Base64 ( ) in commons - codec - 1.4
 * encode ( ) inserts a newline after every 76 characters .
 * Base64(0 ) disables that incompatibility . */
try { return Base64 . class . getConstructor ( int . class ) . newInstance ( 0 ) ; } catch ( SecurityException e ) { } catch ( NoSuchMethodException e ) { } catch ( IllegalArgumentException e ) { } catch ( InstantiationException e ) { } catch ( IllegalAccessException e ) { } catch ( InvocationTargetException e ) { } return new Base64 ( ) ;


signature:
public class RCFileProtobufInputFormat { /** * Stores supplied class name in configuration . This configuration is
 * read on the remote tasks to initialize the input format correctly . */
public static void setClassConf ( Class < ? extends Message > protoClass , Configuration conf ) { } }

implementation: Protobufs . setClassConf ( conf , RCFileProtobufInputFormat . class , protoClass ) ;


signature:
public class RCFileThriftOutputFormat { /** * Stores supplied class name in configuration . This configuration is
 * read on the remote tasks to initialize the output format correctly . */
public static void setClassConf ( Class < ? extends TBase < ? , ? > > thriftClass , Configuration conf ) { } }

implementation: ThriftUtils . setClassConf ( conf , RCFileThriftOutputFormat . class , thriftClass ) ;


signature:
public class ProtobufConverter { /** * Returns a ProtobufConverter for a given Protobuf class . */
public static < M extends Message > ProtobufConverter < M > newInstance ( Class < M > protoClass ) { } }

implementation: return new ProtobufConverter < M > ( new TypeRef < M > ( protoClass ) { } ) ;


signature:
public class LzoThriftSource { /** * Factory method for creating a new { @ code LzoThriftSource } from a given path and Thrift
 * record class .
 * @ param path path to the data
 * @ param thriftClass the Thrift class to read
 * @ return a new { @ code LzoThriftSource } */
public static < S extends TBase < ? , ? > > LzoThriftSource < S > at ( Path path , Class < S > thriftClass ) { } }

implementation: return new LzoThriftSource < S > ( path , EBTypes . thrifts ( thriftClass ) ) ;


signature:
public class LzoThriftSource { /** * Factory method for creating a new { @ code LzoThriftSource } from the given paths and Thrift
 * record class .
 * @ param paths list of paths to data
 * @ param thriftClass the Thrift class to read
 * @ return a new { @ code LzoThriftSource } */
public static < S extends TBase < ? , ? > > LzoThriftSource < S > at ( List < Path > paths , Class < S > thriftClass ) { } }

implementation: return new LzoThriftSource < S > ( paths , EBTypes . thrifts ( thriftClass ) ) ;


signature:
public class BinaryBlockReader { /** * Returns next deserialized object . null indicates end of stream or a
 * deserialization error . Use { @ link # readNext ( BinaryWritable ) } to
 * distinguish betwen end of stream or deserialization error . */
public M readNext ( ) throws IOException { } }

implementation: byte [ ] blob = readNextProtoBytes ( ) ; return blob == null ? null : protoConverter_ . fromBytes ( blob ) ;


signature:
public class BinaryBlockReader { /** * Returns true if new proto object was read into writable , false other wise . */
public boolean readNext ( BinaryWritable < M > writable ) throws IOException { } }

implementation: byte [ ] blob = readNextProtoBytes ( ) ; if ( blob != null ) { writable . set ( protoConverter_ . fromBytes ( blob ) ) ; return true ; } return false ;


signature:
public class BinaryBlockReader { /** * Return byte blob for the next proto object . null indicates end of stream ; */
public byte [ ] readNextProtoBytes ( ) throws IOException { } }

implementation: while ( true ) { if ( ! setupNewBlockIfNeeded ( ) ) { return null ; } int blobIndex = curBlobs_ . size ( ) - numLeftToReadThisBlock_ ; numLeftToReadThisBlock_ -- ; byte [ ] blob = curBlobs_ . get ( blobIndex ) . toByteArray ( ) ; if ( blob . length == 0 && skipEmptyRecords ) { continue ; } return blob ; }


signature:
public class BinaryBlockReader { /** * returns true if bytes for next object are written to writable , false
 * other wise . */
public boolean readNextProtoBytes ( BytesWritable writable ) throws IOException { } }

implementation: byte [ ] blob = readNextProtoBytes ( ) ; if ( blob != null ) { writable . set ( blob , 0 , blob . length ) ; return true ; } return false ;


signature:
public class BinaryBlockReader { /** * Finds next block marker and reads the block . If skipIfStartingOnBoundary is set
 * skips the the first block if the marker starts exactly at the current position .
 * ( i . e . there were no bytes from previous block before the start of the marker ) . */
public List < ByteString > parseNextBlock ( boolean skipIfStartingOnBoundary ) throws IOException { } }

implementation:
LOG . debug ( "BlockReader: none left to read, skipping to sync point" ) ; long skipped = skipToNextSyncPoint ( ) ; if ( skipped <= - 1 ) { LOG . debug ( "BlockReader: SYNC point eof" ) ; // EOF if there are no more sync markers .
return null ; } int blockSize = readInt ( ) ; LOG . debug ( "BlockReader: found sync point, next block has size " + blockSize ) ; if ( blockSize < 0 ) { LOG . debug ( "ProtobufReader: reading size after sync point eof" ) ; // EOF if the size cannot be read .
return null ; } if ( skipIfStartingOnBoundary && skipped == Protobufs . KNOWN_GOOD_POSITION_MARKER . length ) { // skip the current current block
in_ . skip ( blockSize ) ; return parseNextBlock ( false ) ; } SerializedBlock block = SerializedBlock . parseFrom ( new BoundedInputStream ( in_ , blockSize ) , blockSize ) ; curBlobs_ = block . getProtoBlobs ( ) ; numLeftToReadThisBlock_ = curBlobs_ . size ( ) ; LOG . debug ( "ProtobufReader: number in next block is " + numLeftToReadThisBlock_ ) ; return curBlobs_ ;


signature:
public class ThriftBinaryProtocol { /** * Ensures that an element type in a for container ( List , Set , Map ) is
 * a valid container .
 * @ param type
 * @ throws TException if the type is not one of the expected type . */
public static void checkContainerElemType ( byte type ) throws TException { } }

implementation:
// only valid types for an element in a container ( List , Map , Set )
// are the ones that are considered in TProtocolUtil . skip ( )
switch ( type ) { case TType . BOOL : case TType . BYTE : case TType . I16 : case TType . I32 : case TType . I64 : case TType . DOUBLE : case TType . STRING : case TType . STRUCT : case TType . MAP : case TType . SET : case TType . LIST : break ; // list other known types , but not expected
case TType . STOP : case TType . VOID : case TType . ENUM : // would be I32 on the wire
default : throw new TException ( "Unexpected type " + type + " in a container" ) ; }


signature:
public class HadoopCompat { /** * Creates JobContext from a JobConf and jobId using the correct constructor
 * for based on Hadoop version . < code > jobId < / code > could be null . */
public static JobContext newJobContext ( Configuration conf , JobID jobId ) { } }

implementation: return ( JobContext ) newInstance ( JOB_CONTEXT_CONSTRUCTOR , conf , jobId ) ;


signature:
public class HadoopCompat { /** * Creates TaskAttempContext from a JobConf and jobId using the correct
 * constructor for based on Hadoop version . */
public static TaskAttemptContext newTaskAttemptContext ( Configuration conf , TaskAttemptID taskAttemptId ) { } }

implementation: return ( TaskAttemptContext ) newInstance ( TASK_CONTEXT_CONSTRUCTOR , conf , taskAttemptId ) ;


signature:
public class HadoopCompat { /** * Instantiates MapContext under Hadoop 1 and MapContextImpl under Hadoop 2. */
public static MapContext newMapContext ( Configuration conf , TaskAttemptID taskAttemptID , RecordReader recordReader , RecordWriter recordWriter , OutputCommitter outputCommitter , StatusReporter statusReporter , InputSplit inputSplit ) { } }

implementation: return ( MapContext ) newInstance ( MAP_CONTEXT_CONSTRUCTOR , conf , taskAttemptID , recordReader , recordWriter , outputCommitter , statusReporter , inputSplit ) ;


signature:
public class HadoopCompat { /** * Invoke getCounter ( ) on TaskInputOutputContext . Works with both
 * Hadoop 1 and 2. */
public static Counter getCounter ( TaskInputOutputContext context , String groupName , String counterName ) { } }

implementation: return ( Counter ) invoke ( GET_COUNTER_METHOD , context , groupName , counterName ) ;


signature:
public class HadoopCompat { /** * Hadoop 1 & 2 compatible context . getCounter ( )
 * @ return { @ link TaskInputOutputContext # getCounter ( Enum key ) } */
public static Counter getCounter ( TaskInputOutputContext context , Enum < ? > key ) { } }

implementation: return ( Counter ) invoke ( GET_COUNTER_ENUM_METHOD , context , key ) ;


signature:
public class HadoopCompat { /** * This method invokes getDefaultBlocksize ( ) without path on Hadoop 1 and
 * with Path on Hadoop 2.
 * Older versions of Hadoop 1 do not have
 * { @ link FileSystem # getDefaultBlocksize ( Path ) } . This api exists in Hadoop 2
 * and recent versions of Hadoop 1 . Path argument is required for viewfs filesystem
 * in Hadoop 2 . < p > */
public static long getDefaultBlockSize ( FileSystem fs , Path path ) { } }

implementation: return ( Long ) ( HadoopCompat . isVersion2x ( ) ? invoke ( GET_DEFAULT_BLOCK_SIZE_METHOD , fs , path ) : invoke ( GET_DEFAULT_BLOCK_SIZE_METHOD , fs ) ) ;


signature:
public class HadoopCompat { /** * This method invokes getDefaultReplication ( ) without path on Hadoop 1 and
 * with Path on Hadoop 2.
 * Older versions of Hadoop 1 do not have
 * { @ link FileSystem # getDefaultReplication ( Path ) } . This api exists in Hadoop 2
 * and recent versions of Hadoop 1 . Path argument is required for viewfs filesystem
 * in Hadoop 2 . < p > */
public static short getDefaultReplication ( FileSystem fs , Path path ) { } }

implementation: return ( Short ) ( HadoopCompat . isVersion2x ( ) ? invoke ( GET_DEFAULT_REPLICATION_METHOD , fs , path ) : invoke ( GET_DEFAULT_REPLICATION_METHOD , fs ) ) ;


signature:
public class ThriftWritable { /** * Returns a ThriftWritable for a given Thrift class . */
public static < M extends TBase < ? , ? > > ThriftWritable < M > newInstance ( Class < M > tClass ) { } }

implementation: return new ThriftWritable < M > ( new TypeRef < M > ( tClass ) { } ) ;


signature:
public class WritableLoadCaster { /** * Deserializes raw bytes into a Writable instance , returning that instance . This implementation
 * relies on derived classes to initialize { @ link # writable } before this method is called .
 * @ param bytes serialized Writable data .
 * @ param writable Writable instance into which data should be stored .
 * @ return { @ code writable } .
 * @ throws IllegalArgumentException if either argument is { @ code null } .
 * @ throws IOException */
protected < T extends Writable > T readFields ( byte [ ] bytes , T writable ) throws IOException { } }

implementation: Preconditions . checkNotNull ( bytes , "Input bytes are null" ) ; Preconditions . checkNotNull ( writable , "Writable is null" ) ; buf . reset ( bytes , bytes . length ) ; writable . readFields ( buf ) ; return writable ;


signature:
public class TypeRef { /** * Use the typeRef ' s parameter to create a new instance of the TypeRef ' s template parameter .
 * @ return a new instance of type T
 * @ throws NoSuchMethodException
 * @ throws IllegalAccessException
 * @ throws InvocationTargetException
 * @ throws InstantiationException */
@ SuppressWarnings ( "unchecked" ) public T newInstance ( ) throws NoSuchMethodException , IllegalAccessException , InvocationTargetException , InstantiationException { } }

implementation: if ( constructor_ == null ) { constructor_ = getRawClass ( ) . getConstructor ( ) ; } return ( T ) constructor_ . newInstance ( ) ;


signature:
public class TypeRef { /** * The cheap , ugly version of the above , for when you don ' t want to catch 900 exceptions at
 * the calling site .
 * @ return a new instance of type T . */
public T safeNewInstance ( ) { } }

implementation: try { return newInstance ( ) ; } catch ( NoSuchMethodException e ) { throw new IllegalArgumentException ( e ) ; } catch ( IllegalAccessException e ) { throw new IllegalArgumentException ( e ) ; } catch ( InvocationTargetException e ) { throw new IllegalArgumentException ( e ) ; } catch ( InstantiationException e ) { throw new IllegalArgumentException ( e ) ; }


signature:
public class ProtobufPigLoader { /** * Return every non - null line as a single - element tuple to Pig .
 * A small fraction of bad records in input are tolerated .
 * See { @ link LzoRecordReader } for more information on error handling . */
@ Override public Tuple getNext ( ) throws IOException { } }

implementation: if ( tupleTemplate == null ) { tupleTemplate = new ProjectedProtobufTupleFactory < M > ( typeRef , requiredFieldList ) ; } M value = getNextBinaryValue ( typeRef ) ; return value != null ? tupleTemplate . newTuple ( value ) : null ;


signature:
public class ThriftToDynamicProto { /** * For the given thriftClass , return a Protobufs builder to build a similar protobuf class .
 * @ param thriftClass The thrift class for which the builder is desired .
 * @ return a protobuf message builder */
public Message . Builder getBuilder ( Class < ? extends TBase < ? , ? > > thriftClass ) { } }

implementation: return messageBuilderMap . get ( protoMessageType ( thriftClass ) ) . clone ( ) ;


signature:
public class ThriftToDynamicProto { /** * Return Protobufs builder for a Map field */
private Message . Builder mapEntryProtoBuilder ( TStructDescriptor descriptor , Field field ) { } }

implementation: return messageBuilderMap . get ( mapProtoMessageType ( descriptor , field ) ) . clone ( ) ;


signature:
public class ThriftToDynamicProto { /** * When dealing with a Set / List , we want to retrieve the relevant Field type */
private Field resolveField ( Field inputField ) { } }

implementation: if ( inputField . isList ( ) ) { return inputField . getListElemField ( ) ; } else if ( inputField . isSet ( ) ) { return inputField . getSetElemField ( ) ; } else { return inputField ; }


signature:
public class ThriftToDynamicProto { /** * Determines whether a field is considered to be a nested object based on :
 * - whether the field itself is a struct
 * - whether the field is a list / set of structs
 * - whether field is a Map */
private boolean hasNestedObject ( Field field ) { } }

implementation: return field . isStruct ( ) || ( field . isList ( ) && field . getListElemField ( ) . isStruct ( ) ) || ( field . isSet ( ) && field . getSetElemField ( ) . isStruct ( ) ) || field . isMap ( ) ;


signature:
public class ThriftToDynamicProto { /** * Generate a DescriptorProto . Builder for the Message type that will be used
 * to represent the entries of the input Map field .
 * @ param field a Map Field ( field . isMap ( ) = = true )
 * @ param typeName name of new message type */
private DescriptorProtos . DescriptorProto . Builder mapDescriptorProtoBuilder ( Field field , String typeName ) throws DescriptorValidationException { } }

implementation: DescriptorProtos . DescriptorProto . Builder mapBuilder = DescriptorProtos . DescriptorProto . newBuilder ( ) . setName ( typeName ) ; Field keyField = field . getMapKeyField ( ) ; Field valueField = field . getMapValueField ( ) ; DescriptorProtos . FieldDescriptorProto . Builder keyBuilder = mapKeyProtoBuilder ( ) ; DescriptorProtos . FieldDescriptorProto . Builder valueBuilder = mapValueProtoBuilder ( ) ; setBuilderTypeFromField ( keyField , keyBuilder ) ; setBuilderTypeFromField ( valueField , valueBuilder ) ; mapBuilder . addField ( keyBuilder . build ( ) ) ; mapBuilder . addField ( valueBuilder . build ( ) ) ; return mapBuilder ;


signature:
public class ThriftToDynamicProto { /** * For a TStructDescriptor , resolves the typeName and optionally converts and memoizes it ' s
 * schema . */
private String resolveMessageTypeName ( TStructDescriptor descriptor ) throws DescriptorValidationException { } }

implementation:
String typeName = protoMessageType ( descriptor . getThriftClass ( ) ) ; // Anytime we have a new message typeName , we make sure that we have a builder for it .
// If not , we create one .
DescriptorProtos . DescriptorProto . Builder builder = descriptorBuilderMap . get ( typeName ) ; if ( builder == null ) { builder = DescriptorProtos . DescriptorProto . newBuilder ( ) ; builder . setName ( typeName ) ; descriptorBuilderMap . put ( typeName , builder ) ; doSchemaMapping ( builder , descriptor ) ; } return typeName ;


signature:
public class ThriftToDynamicProto { /** * Convert a thrift object to a protobuf message .
 * @ param thriftObj thrift object
 * @ return protobuf protobuf message */
@ SuppressWarnings ( "unchecked" ) public Message convert ( T thriftObj ) { } }

implementation: return doConvert ( ( TBase < ? , ? > ) Preconditions . checkNotNull ( thriftObj , "Can not convert a null object" ) ) ;


signature:
public class ThriftToDynamicProto { /** * conver TBase object to Message object
 * @ param thriftObj */
@ SuppressWarnings ( "unchecked" ) public < F extends TFieldIdEnum > Message doConvert ( TBase < ? , F > thriftObj ) { } }

implementation:
if ( thriftObj == null ) { return null ; } Class < TBase < ? , F > > clazz = ( Class < TBase < ? , F > > ) thriftObj . getClass ( ) ; checkState ( clazz ) ; Message . Builder builder = getBuilder ( clazz ) ; TStructDescriptor fieldDesc = TStructDescriptor . getInstance ( clazz ) ; int fieldId = 0 ; for ( Field tField : fieldDesc . getFields ( ) ) { // don ' t want to carry over default values from unset fields
if ( ! thriftObj . isSet ( ( F ) tField . getFieldIdEnum ( ) ) || ( ! supportNestedObjects && hasNestedObject ( tField ) ) ) { fieldId ++ ; continue ; } // recurse into the object if it ' s a struct , otherwise just add the field
if ( supportNestedObjects && tField . getType ( ) == TType . STRUCT ) { TBase < ? , ? > fieldValue = ( TBase < ? , ? > ) fieldDesc . getFieldValue ( fieldId ++ , thriftObj ) ; Message message = doConvert ( fieldValue ) ; if ( message != null ) { FieldDescriptor protoFieldDesc = builder . getDescriptorForType ( ) . findFieldByName ( tField . getName ( ) ) ; builder . setField ( protoFieldDesc , message ) ; } } else { fieldId = convertField ( thriftObj , builder , fieldDesc , fieldId , tField ) ; } } return builder . build ( ) ;


signature:
public class ThriftToDynamicProto { /** * Determines if the field in question is a Set or List of a Struct type */
private boolean isStructContainer ( Field tField ) { } }

implementation: return ( tField . isList ( ) && tField . getListElemField ( ) . isStruct ( ) ) || ( tField . isSet ( ) && tField . getSetElemField ( ) . isStruct ( ) ) ;


signature:
public class ThriftToDynamicProto { /** * Builds a Message that contains the key value pair of a Map entry */
private Message buildMapEntryMessage ( Message . Builder mapBuilder , Field field , Object mapKey , Object mapValue ) { } }

implementation: FieldDescriptor keyFieldDescriptor = mapBuilder . getDescriptorForType ( ) . findFieldByName ( MAP_KEY_FIELD_NAME ) ; FieldDescriptor valueFieldDescriptor = mapBuilder . getDescriptorForType ( ) . findFieldByName ( MAP_VALUE_FIELD_NAME ) ; boolean isKeyStruct = field . getMapKeyField ( ) . isStruct ( ) ; boolean isValueStruct = field . getMapValueField ( ) . isStruct ( ) ; Object convertedKey ; if ( isKeyStruct ) { convertedKey = doConvert ( ( TBase < ? , ? > ) mapKey ) ; } else { convertedKey = sanitizeRawValue ( mapKey , field . getMapKeyField ( ) ) ; } Object convertedValue ; if ( isValueStruct ) { convertedValue = doConvert ( ( TBase < ? , ? > ) mapValue ) ; } else { convertedValue = sanitizeRawValue ( mapValue , field . getMapValueField ( ) ) ; } mapBuilder . setField ( keyFieldDescriptor , convertedKey ) ; mapBuilder . setField ( valueFieldDescriptor , convertedValue ) ; return mapBuilder . build ( ) ;


signature:
public class ThriftToDynamicProto { /** * name the proto message used for Map types after the thrift class name of the enclosing
 * struct and the field name */
private String mapProtoMessageType ( TStructDescriptor descriptor , Field field ) { } }

implementation: return String . format ( "%s_%s" , protoMessageType ( descriptor . getThriftClass ( ) ) , field . getName ( ) ) ;


signature:
public class ThriftToDynamicProto { /** * FieldDescriptor */
public FieldDescriptor getFieldDescriptor ( Class < ? extends TBase < ? , ? > > thriftClass , String fieldName ) { } }

implementation: checkState ( thriftClass ) ; Descriptors . Descriptor descriptor = getBuilder ( thriftClass ) . getDescriptorForType ( ) ; return descriptor . findFieldByName ( fieldName ) ;


signature:
public class LzoBaseLoadFunc { /** * A convenience function for working with Hadoop counter objects from load functions . The Hadoop
 * reporter object isn ' t always set up at first , so this class provides brief buffering to ensure
 * that counters are always recorded . */
protected void incrCounter ( String group , String counter , long incr ) { } }

implementation: counterHelper_ . incrCounter ( group , counter , incr ) ;


signature:
public class LzoBaseLoadFunc { /** * same as incrCounter ( pair . first , pair . second , incr ) . */
protected void incrCounter ( Pair < String , String > groupCounterPair , long incr ) { } }

implementation: counterHelper_ . incrCounter ( groupCounterPair . first , groupCounterPair . second , incr ) ;


signature:
public class LzoBaseLoadFunc { /** * A utility method for loaders that read { @ link BinaryWritable } values . */
protected < M > M getNextBinaryValue ( TypeRef < M > typeRef ) throws IOException { } }

implementation:
// typeRef is just to help compiler resolve the type at compile time .
try { if ( reader != null && reader . nextKeyValue ( ) ) { @ SuppressWarnings ( "unchecked" ) BinaryWritable < M > writable = ( BinaryWritable < M > ) reader . getCurrentValue ( ) ; return writable . get ( ) ; } } catch ( InterruptedException e ) { LOG . error ( "InterruptedException encountered, bailing." , e ) ; throw new IOException ( e ) ; } return null ;


signature:
public class LzoBaseLoadFunc { /** * A helper method for implementing
 * { @ link LoadPushDown # pushProjection ( RequiredFieldList ) } . < p >
 * Stores requiredFieldList in context . The requiredFields are read from
 * context on the backend ( in side { @ link # setLocation ( String , Job ) } ) . */
protected RequiredFieldResponse pushProjectionHelper ( RequiredFieldList requiredFieldList ) throws FrontendException { } }

implementation:
try { getUDFProperties ( ) . setProperty ( projectionKey , ObjectSerializer . serialize ( requiredFieldList ) ) ; } catch ( IOException e ) { // not expected
throw new FrontendException ( e ) ; } return new RequiredFieldResponse ( true ) ;


signature:
public class BaseStoreFunc { /** * A convenience function for working with Hadoop counter objects from load functions . The Hadoop
 * reporter object isn ' t always set up at first , so this class provides brief buffering to ensure
 * that counters are always recorded . */
protected void incrCounter ( String group , String counter , long incr ) { } }

implementation: counterHelper . incrCounter ( group , counter , incr ) ;


signature:
public class BaseStoreFunc { /** * same as incrCounter ( pair . first , pair . second , incr ) . */
protected void incrCounter ( Pair < String , String > groupCounterPair , long incr ) { } }

implementation: counterHelper . incrCounter ( groupCounterPair . first , groupCounterPair . second , incr ) ;


signature:
public class BaseStoreFunc { /** * Invokes writer . write ( key , value ) . < br >
 * The main purpose is to catch { @ link InterruptedException } and covert it to an
 * { @ link IOException } . Avoids unnecessary try - catch dance in putNext ( ) implementations . */
@ SuppressWarnings ( "unchecked" ) final protected void writeRecord ( Object key , Object value ) throws IOException { } }

implementation: try { writer . write ( key , value ) ; } catch ( InterruptedException e ) { throw new IOException ( e ) ; }


signature:
public class HdfsUtils { /** * Recursively walk a path applying visitor to each path accepted by
 * filter
 * @ param path root path to begin walking , will be visited if
 * it passes the filter and directory flag
 * @ param fs FileSystem for this path
 * @ param filter filter to determine which paths to accept
 * @ param visitor visitor to apply to each accepted path
 * @ throws IOException */
public static void walkPath ( Path path , FileSystem fs , PathFilter filter , PathVisitor visitor ) throws IOException { } }

implementation: FileStatus fileStatus = fs . getFileStatus ( path ) ; if ( filter . accept ( path ) ) { visitor . visit ( fileStatus ) ; } if ( fileStatus . isDir ( ) ) { FileStatus [ ] children = fs . listStatus ( path ) ; for ( FileStatus childStatus : children ) { walkPath ( childStatus . getPath ( ) , fs , filter , visitor ) ; } }


signature:
public class HdfsUtils { /** * Recursively walk a path , adding paths that are accepted by filter to accumulator
 * @ param path root path to begin walking , will be added to accumulator
 * @ param fs FileSystem for this path
 * @ param filter filter to determine which paths to accept
 * @ param accumulator all paths accepted will be added to accumulator
 * @ throws IOException */
public static void collectPaths ( Path path , FileSystem fs , PathFilter filter , final List < Path > accumulator ) throws IOException { } }

implementation: walkPath ( path , fs , filter , new PathVisitor ( ) { @ Override public void visit ( FileStatus fileStatus ) { accumulator . add ( fileStatus . getPath ( ) ) ; } } ) ;


signature:
public class HdfsUtils { /** * Calculates the total size of all the contents of a directory that are accepted
 * by filter . All subdirectories will be searched recursively and paths in subdirectories
 * that are accepted by filter will also be counted .
 * Does not include the size of directories themselves
 * ( which are 0 in HDFS but may not be 0 on local file systems )
 * To get the size of a directory without filtering , use
 * { @ link # getDirectorySize ( Path , FileSystem ) } which is much more efficient .
 * @ param path path to recursively walk
 * @ param fs FileSystem for this path
 * @ param filter path filter for which paths size ' s to include in the total
 * NOTE : you do * not * need to filter out directories , this will be done for you
 * @ return size of the directory in bytes
 * @ throws IOException */
public static long getDirectorySize ( Path path , FileSystem fs , PathFilter filter ) throws IOException { } }

implementation: PathSizeVisitor visitor = new PathSizeVisitor ( ) ; PathFilter composite = new PathFilters . CompositePathFilter ( PathFilters . newExcludeDirectoriesFilter ( fs . getConf ( ) ) , filter ) ; walkPath ( path , fs , composite , visitor ) ; return visitor . getSize ( ) ;


signature:
public class HdfsUtils { /** * Calculates the total size of all the contents of a directory ,
 * including the contents of all of its subdirectories .
 * Does not include the size of directories themselves
 * ( which are 0 in HDFS but may not be 0 on local file systems )
 * @ param path path to recursively walk
 * @ param fs FileSystem for this path
 * @ return size of the directory ' s contents in bytes
 * @ throws IOException */
public static long getDirectorySize ( Path path , FileSystem fs ) throws IOException { } }

implementation: ContentSummary cs = fs . getContentSummary ( path ) ; return cs . getLength ( ) ;


signature:
public class HdfsUtils { /** * Given a list of paths that ( potentially ) have glob syntax in them ,
 * return a list of paths with all the globs expanded .
 * @ param pathsWithGlobs a list of paths that may or may not have glob syntax in them
 * @ param conf job conf
 * @ return an equivalent list of paths with no glob syntax in them
 * @ throws IOException */
public static List < Path > expandGlobs ( List < String > pathsWithGlobs , Configuration conf ) throws IOException { } }

implementation:
List < Path > paths = Lists . newLinkedList ( ) ; for ( String pathStr : pathsWithGlobs ) { Path path = new Path ( pathStr ) ; FileSystem fs = path . getFileSystem ( conf ) ; FileStatus [ ] statuses = fs . globStatus ( path ) ; // some versions of hadoop return null for non - existent paths
if ( statuses != null ) { for ( FileStatus status : statuses ) { paths . add ( status . getPath ( ) ) ; } } } return paths ;


signature:
public class LzoW3CLogLoader { /** * Return every non - null line as a single - element tuple to Pig . */
@ Override public Tuple getNext ( ) throws IOException { } }

implementation: LzoW3CLogRecordReader w3CLogRecordReader = ( LzoW3CLogRecordReader ) reader ; if ( w3CLogRecordReader == null ) { return null ; } MapWritable value_ ; try { if ( w3CLogRecordReader . nextKeyValue ( ) && ( value_ = w3CLogRecordReader . getCurrentValue ( ) ) != null ) { Map < String , String > values = Maps . newHashMap ( ) ; for ( Writable key : value_ . keySet ( ) ) { Writable value = value_ . get ( key ) ; values . put ( key . toString ( ) , value != null ? value . toString ( ) : null ) ; } incrCounter ( LzoW3CLogLoaderCounters . LinesW3CDecoded , 1L ) ; incrCounter ( LzoW3CLogLoaderCounters . UnparseableLines , w3CLogRecordReader . getBadRecordsSkipped ( ) ) ; return tupleFactory_ . newTuple ( values ) ; } } catch ( InterruptedException e ) { int errCode = 6018 ; String errMsg = "Error while reading input" ; throw new ExecException ( errMsg , errCode , PigException . REMOTE_ENVIRONMENT , e ) ; } return null ;


signature:
public class LzoBufferedPositionedInputStream { /** * super . skip ( ) skips uncompressed bytes , so we need to override it .
 * This allows us to specify skip size in a compressed stream . */
@ Override public long skip ( long toSkip ) throws IOException { } }

implementation: long pos = getPosition ( ) ; while ( getPosition ( ) - pos < toSkip ) { if ( super . skip ( toSkip ) == 0 ) break ; } return getPosition ( ) - pos ;


signature:
public class RCFileProtobufOutputFormat { /** * Stores supplied class name in configuration . This configuration is
 * read on the remote tasks to initialize the output format correctly . */
public static void setClassConf ( Class < ? extends Message > protoClass , Configuration conf ) { } }

implementation: Protobufs . setClassConf ( conf , RCFileProtobufOutputFormat . class , protoClass ) ;


signature:
public class RCFileOutputFormat { /** * set number of columns into the given configuration .
 * @ param conf
 * configuration instance which need to set the column number
 * @ param columnNum
 * column number for RCFile ' s Writer */
public static void setColumnNumber ( Configuration conf , int columnNum ) { } }

implementation: assert columnNum > 0 ; conf . setInt ( RCFile . COLUMN_NUMBER_CONF_STR , columnNum ) ;


signature:
public class PigCounterHelper { /** * Mocks the Reporter . incrCounter , but adds buffering .
 * See org . apache . hadoop . mapred . Reporter ' s incrCounter . */
public void incrCounter ( String group , String counterName , long incr ) { } }

implementation:
PigStatusReporter reporter = PigStatusReporter . getInstance ( ) ; if ( reporter != null ) { // common case
Counter counter = reporter . getCounter ( group , counterName ) ; if ( counter != null ) { HadoopCompat . incrementCounter ( counter , incr ) ; if ( counterStringMap_ . size ( ) > 0 ) { for ( Map . Entry < Pair < String , String > , Long > entry : counterStringMap_ . entrySet ( ) ) { HadoopCompat . incrementCounter ( reporter . getCounter ( entry . getKey ( ) . first , entry . getKey ( ) . second ) , entry . getValue ( ) ) ; } counterStringMap_ . clear ( ) ; } return ; } } // In the case when reporter is not available , or we can ' t get the Counter ,
// store in the local map .
Pair < String , String > key = new Pair < String , String > ( group , counterName ) ; Long currentValue = counterStringMap_ . get ( key ) ; counterStringMap_ . put ( key , ( currentValue == null ? 0 : currentValue ) + incr ) ;


signature:
public class PigCounterHelper { /** * Mocks the Reporter . incrCounter , but adds buffering .
 * See org . apache . hadoop . mapred . Reporter ' s incrCounter . */
public void incrCounter ( Enum < ? > key , long incr ) { } }

implementation:
PigStatusReporter reporter = PigStatusReporter . getInstance ( ) ; if ( reporter != null && reporter . getCounter ( key ) != null ) { HadoopCompat . incrementCounter ( reporter . getCounter ( key ) , incr ) ; if ( counterEnumMap_ . size ( ) > 0 ) { for ( Map . Entry < Enum < ? > , Long > entry : counterEnumMap_ . entrySet ( ) ) { HadoopCompat . incrementCounter ( reporter . getCounter ( entry . getKey ( ) ) , entry . getValue ( ) ) ; } counterEnumMap_ . clear ( ) ; } } else { // buffer the increments
Long currentValue = counterEnumMap_ . get ( key ) ; counterEnumMap_ . put ( key , ( currentValue == null ? 0 : currentValue ) + incr ) ; }


signature:
public class CombinedSequenceFile { /** * We can allow overriding the compression settings for just this scheme here */
private static void updateJobConfForLocalSettings ( JobConf conf ) { } }

implementation: String localSetCompressionEnabled = conf . get ( COMPRESS_ENABLE ) ; if ( localSetCompressionEnabled != null ) { conf . set ( MR_COMPRESS_ENABLE , localSetCompressionEnabled ) ; } String localSetCompressionType = conf . get ( COMPRESS_TYPE ) ; if ( localSetCompressionType != null ) { conf . set ( MR_COMPRESS_TYPE , localSetCompressionType ) ; } String localSetCompressionCodec = conf . get ( COMPRESS_CODEC ) ; if ( localSetCompressionCodec != null ) { conf . set ( MR_COMPRESS_CODEC , localSetCompressionCodec ) ; }


signature:
public class ProtobufReflectionUtil { /** * Parse the method for a message
 * @ param klass the class containing the message
 * @ return the parsed method */
public static Method parseMethodFor ( Class < Message > klass ) { } }

implementation: try { return klass . getMethod ( "parseDelimitedFrom" , new Class [ ] { InputStream . class } ) ; } catch ( SecurityException e ) { throw new RuntimeException ( e ) ; } catch ( NoSuchMethodException e ) { throw new RuntimeException ( e ) ; }


signature:
public class ProtobufReflectionUtil { /** * Parse the message in a given InputStream using scpecified Method
 * @ param parseMethod the method used for parsing
 * @ param in the input stream
 * @ return the parsed message */
public static Message parseMessage ( Method parseMethod , InputStream in ) { } }

implementation: try { return ( Message ) parseMethod . invoke ( null , in ) ; } catch ( IllegalArgumentException e ) { throw new RuntimeException ( e ) ; } catch ( IllegalAccessException e ) { throw new RuntimeException ( e ) ; } catch ( InvocationTargetException e ) { throw new RuntimeException ( e ) ; }


signature:
public class ProtobufReflectionUtil { /** * Parse the message in a given Message container
 * @ param klass the class containing the message
 * @ param in the input stream
 * @ return the parsed Message */
public static Message parseMessage ( Class < Message > klass , InputStream in ) { } }

implementation: Method parseMethod = parseMethodFor ( klass ) ; return parseMessage ( parseMethod , in ) ;


signature:
public class RCFileThriftInputFormat { /** * Stores supplied class name in configuration . This configuration is
 * read on the remote tasks to initialize the input format correctly . */
public static void setClassConf ( Class < ? extends TBase < ? , ? > > thriftClass , Configuration conf ) { } }

implementation: ThriftUtils . setClassConf ( conf , RCFileThriftInputFormat . class , thriftClass ) ;


signature:
public class LzoJsonStorage { /** * The first element is expected to be a map , or null . Anything else causes an error .
 * @ param tuple the tuple to write . */
@ Override @ SuppressWarnings ( "unchecked" ) public void putNext ( Tuple tuple ) throws IOException { } }

implementation:
json . clear ( ) ; if ( tuple != null && tuple . size ( ) >= 1 ) { Map < String , Object > map = ( Map < String , Object > ) tuple . get ( 0 ) ; if ( keysToKeep_ == null ) { json . putAll ( map ) ; } else { for ( Map . Entry < String , Object > entry : map . entrySet ( ) ) { if ( keysToKeep_ . contains ( entry . getKey ( ) ) ) { json . put ( entry . getKey ( ) , entry . getValue ( ) ) ; } } } } try { writer . write ( null , new Text ( json . toString ( ) ) ) ; } catch ( InterruptedException e ) { // Under what circumstances does this happen ?
throw new IOException ( e ) ; }


signature:
public class LzoProtobufB64LineOutputFormat { /** * Sets an internal configuration in jobConf so that remote Tasks
 * instantiate appropriate object for this generic class based on protoClass */
public static < M extends Message > void setClassConf ( Class < M > protoClass , Configuration jobConf ) { } }

implementation: Protobufs . setClassConf ( jobConf , LzoProtobufB64LineOutputFormat . class , protoClass ) ;


signature:
public class MultiInputFormat { /** * Sets jobs input format to { @ link MultiInputFormat } and stores
 * supplied clazz ' s name in job configuration . This configuration is
 * read on the remote tasks to initialize the input format correctly . */
public static void setInputFormatClass ( Class < ? > clazz , Job job ) { } }

implementation: job . setInputFormatClass ( MultiInputFormat . class ) ; setClassConf ( clazz , HadoopCompat . getConfiguration ( job ) ) ;


signature:
public class MultiInputFormat { /** * Stores supplied class name in configuration . This configuration is
 * read on the remote tasks to initialize the input format correctly . */
public static void setClassConf ( Class < ? > clazz , Configuration conf ) { } }

implementation: HadoopUtils . setClassConf ( conf , CLASS_CONF_KEY , clazz ) ;


signature:
public class MultiInputFormat { /** * set typeRef from conf */
private void setTypeRef ( Configuration conf ) { } }

implementation: String className = conf . get ( CLASS_CONF_KEY ) ; if ( className == null ) { throw new RuntimeException ( CLASS_CONF_KEY + " is not set" ) ; } Class < ? > clazz = null ; try { clazz = conf . getClassByName ( className ) ; } catch ( ClassNotFoundException e ) { throw new RuntimeException ( "failed to instantiate class '" + className + "'" , e ) ; } typeRef = new TypeRef < M > ( clazz ) { } ;


signature:
public class MultiInputFormat { /** * Checks to see if the input records are stored as SerializedBlock .
 * The block format starts with { @ link Protobufs # KNOWN _ GOOD _ POSITION _ MARKER } .
 * Otherwise the input is assumed to be Base64 encoded lines . */
private static Format determineFileFormat ( InputSplit split , Configuration conf ) throws IOException { } }

implementation:
FileSplit fileSplit = ( FileSplit ) split ; Path file = fileSplit . getPath ( ) ; /* we could have a an optional configuration that maps a regex on a
 * file name to a format . E . g . " . * - block . lzo " to LZO _ BLOCK file . */
// most of the cost is opening the file and
// reading first lzo block ( about 256k of uncompressed data )
CompressionCodec codec = new CompressionCodecFactory ( conf ) . getCodec ( file ) ; if ( codec == null ) { throw new IOException ( "No codec for file " + file + " found" ) ; } InputStream in = file . getFileSystem ( conf ) . open ( file ) ; InputStream lzoIn = null ; // check if the file starts with magic bytes for Block storage format .
try { lzoIn = codec . createInputStream ( in ) ; for ( byte magic : Protobufs . KNOWN_GOOD_POSITION_MARKER ) { int b = lzoIn . read ( ) ; if ( b < 0 || ( byte ) b != magic ) { return Format . LZO_B64LINE ; } } } finally { IOUtils . closeStream ( lzoIn ) ; IOUtils . closeStream ( in ) ; } // the check passed
return Format . LZO_BLOCK ;


signature:
public class AbstractWritableConverter { /** * Default implementation tests that class of internal Writable instance , if defined , is
 * assignable from the given class . If so , the internal Writable instance is replaced with a new
 * instance of the given class . If the given class is { @ code null } , nothing is done .
 * @ see WritableConverter # initialize ( java . lang . Class ) */
@ Override public void initialize ( Class < ? extends W > writableClass ) throws IOException { } }

implementation: if ( writableClass == null ) { return ; } if ( writable != null ) { Class < ? > existingWritableClass = writable . getClass ( ) ; Preconditions . checkArgument ( existingWritableClass . isAssignableFrom ( writableClass ) , "Existing Writable implementation '%s' is not assignable from class '%s'" , existingWritableClass . getName ( ) , writableClass . getName ( ) ) ; } try { writable = writableClass . newInstance ( ) ; } catch ( Exception e ) { throw new IOException ( String . format ( "Failed to create instance of Writable type '%s'" , writableClass . getName ( ) ) , e ) ; }


signature:
public class AbstractWritableConverter { /** * Default implementation returns value of { @ link # writableClass } .
 * @ see com . twitter . elephantbird . pig . util . WritableConverter # getWritableClass ( ) */
@ Override @ SuppressWarnings ( "unchecked" ) public Class < W > getWritableClass ( ) throws IOException { } }

implementation: if ( writable == null ) { return null ; } return ( Class < W > ) writable . getClass ( ) ;


signature:
public class AbstractWritableConverter { /** * Default implementation routes to one of the other { @ code toWritable } methods based on the type
 * of the input Pig value .
 * @ see WritableConverter # toWritable ( java . lang . Object , org . apache . hadoop . io . Writable ) */
@ Override @ SuppressWarnings ( "unchecked" ) public W toWritable ( Object value ) throws IOException { } }

implementation:
if ( value == null ) { return null ; } // route to appropriate method using Pig data type
switch ( DataType . findType ( value ) ) { case DataType . BYTEARRAY : return toWritable ( ( DataByteArray ) value ) ; case DataType . CHARARRAY : return toWritable ( ( String ) value ) ; case DataType . INTEGER : return toWritable ( ( Integer ) value ) ; case DataType . LONG : return toWritable ( ( Long ) value ) ; case DataType . FLOAT : return toWritable ( ( Float ) value ) ; case DataType . DOUBLE : return toWritable ( ( Double ) value ) ; case DataType . MAP : return toWritable ( ( Map < String , Object > ) value ) ; case DataType . TUPLE : return toWritable ( ( Tuple ) value ) ; case DataType . BAG : return toWritable ( ( DataBag ) value ) ; default : throw new IOException ( "Pig value class '" + value . getClass ( ) . getName ( ) + "' is unsupported" ) ; }


signature:
public class AbstractThriftBinaryProtocol { /** * Check if the container size is valid .
 * NOTE : This assumes that the elements are one byte each . So this does not
 * catch all cases , but does increase the chances of handling malformed
 * lengths when the number of remaining bytes in the underlying Transport is
 * clearly less than the container size that the Transport provides . */
protected void checkContainerSize ( int size ) throws TProtocolException { } }

implementation: if ( size < 0 ) { throw new TProtocolException ( "Negative container size: " + size ) ; } if ( checkReadLength_ && ( readLength_ - size ) < 0 ) { throw new TProtocolException ( "Remaining message length is " + readLength_ + " but container size in underlying TTransport is set to at least: " + size ) ; }


signature:
public class StreamSearcher { /** * Sets a new pattern for this StreamSearcher to use .
 * @ param pattern
 * the pattern the StreamSearcher will look for in future calls to search ( . . . ) */
public void setPattern ( byte [ ] pattern ) { } }

implementation: pattern_ = Arrays . copyOf ( pattern , pattern . length ) ; borders_ = new int [ pattern_ . length + 1 ] ; preProcess ( ) ;


signature:
public class StreamSearcher { /** * Searches for the next occurrence of the pattern in the stream , starting from the current stream position . Note
 * that the position of the stream is changed . If a match is found , the stream points to the end of the match - - i . e . the
 * byte AFTER the pattern . Else , the stream is entirely consumed . The latter is because InputStream semantics make it difficult to have
 * another reasonable default , i . e . leave the stream unchanged .
 * @ return bytes consumed if found , - 1 otherwise .
 * @ throws IOException */
public long search ( InputStream stream ) throws IOException { } }

implementation:
long bytesRead = 0 ; int b ; int j = 0 ; while ( ( b = stream . read ( ) ) != - 1 ) { bytesRead ++ ; while ( j >= 0 && ( byte ) b != pattern_ [ j ] ) { j = borders_ [ j ] ; } // Move to the next character in the pattern .
++ j ; // If we ' ve matched up to the full pattern length , we found it . Return ,
// which will automatically save our position in the InputStream at the point immediately
// following the pattern match .
if ( j == pattern_ . length ) { return bytesRead ; } } // No dice , Note that the stream is now completely consumed .
return - 1 ;


signature:
public class StreamSearcher { /** * Builds up a table of longest " borders " for each prefix of the pattern to find . This table is stored internally
 * and aids in implementation of the Knuth - Moore - Pratt string search .
 * For more information , see : http : / / www . inf . fh - flensburg . de / lang / algorithmen / pattern / kmpen . htm . */
protected void preProcess ( ) { } }

implementation: int i = 0 ; int j = - 1 ; borders_ [ i ] = j ; while ( i < pattern_ . length ) { while ( j >= 0 && pattern_ [ i ] != pattern_ [ j ] ) { j = borders_ [ j ] ; } borders_ [ ++ i ] = ++ j ; }


signature:
public class Strings { /** * Underscore the given word .
 * @ param word The word
 * @ return The underscored version of the word */
public static String underscore ( String word ) { } }

implementation:
String firstPattern = "([A-Z]+)([A-Z][a-z])" ; String secondPattern = "([a-z\\d])([A-Z])" ; String replacementPattern = "$1_$2" ; // Replace package separator with slash .
word = word . replaceAll ( "\\." , "/" ) ; // Replace $ with two underscores for inner classes .
word = word . replaceAll ( "\\$" , "__" ) ; // Replace capital letter with _ plus lowercase letter .
word = word . replaceAll ( firstPattern , replacementPattern ) ; word = word . replaceAll ( secondPattern , replacementPattern ) ; word = word . replace ( '-' , '_' ) ; word = word . toLowerCase ( ) ; return word ;


signature:
public class Strings { /** * Return the ordinal for the given number .
 * @ param n The number
 * @ return The ordinal String */
public static String ordinalize ( int n ) { } }

implementation: int mod100 = n % 100 ; if ( mod100 == 11 || mod100 == 12 || mod100 == 13 ) { return String . valueOf ( n ) + "th" ; } switch ( n % 10 ) { case 1 : return String . valueOf ( n ) + "st" ; case 2 : return String . valueOf ( n ) + "nd" ; case 3 : return String . valueOf ( n ) + "rd" ; default : return String . valueOf ( n ) + "th" ; }


signature:
public class ProtobufWritable { /** * Returns a ProtobufWritable for a given Protobuf class . */
public static < M extends Message > ProtobufWritable < M > newInstance ( Class < M > tClass ) { } }

implementation: return new ProtobufWritable < M > ( new TypeRef < M > ( tClass ) { } ) ;


signature:
public class Utils { /** * returns Class . forName ( className , true , classLoader ) . < br >
 * Throws a RuntimeExcepiton if the class is not found .
 * @ see { @ link Class # forName ( String , boolean , ClassLoader ) */
public static Class < ? > classForName ( String className , ClassLoader classLoader ) { } }

implementation: try { return Class . forName ( className , true , classLoader ) ; } catch ( ClassNotFoundException e ) { throw new RuntimeException ( "failed to load class " + className , e ) ; }


signature:
public class Utils { /** * Ensures the < code > classLoader < / code > is ' consistent ' with the original
 * class loader that created < code > existingClass < / code > . Asserts < br >
 * < code > classLoader . loadClass ( existingClass . getName ( ) ) = = existingClass < / code > .
 * If classLoader fails to load the class , this returns silently . < br >
 * Throws a RuntimeException with detailed message if the consistency
 * check fails .
 * @ param existingClass
 * @ param classLoader */
public static void ensureClassLoaderConsistency ( Class < ? > existingClass , ClassLoader classLoader ) { } }

implementation:
Class < ? > loadedClass ; try { loadedClass = Class . forName ( existingClass . getName ( ) , true , classLoader ) ; } catch ( ClassNotFoundException e ) { return ; // let class loading fail some where else .
} if ( ! loadedClass . equals ( existingClass ) ) { throw new RuntimeException ( "The class loader is incosistent with the " + "class loader that initially loaded " + existingClass . getClass ( ) + ". This can lead to various unexpected side effects." ) ; }


signature:
public class LzoW3CLogInputFormat { /** * Use this method to create valid instances of LzoW3CLogInputFormat
 * @ param fieldDefinitionFile path to file in HDFS that contains the CRC hash to column mappings .
 * @ return */
public static LzoW3CLogInputFormat newInstance ( final String fieldDefinitionFile ) { } }

implementation: return new LzoW3CLogInputFormat ( ) { @ Override public RecordReader < LongWritable , MapWritable > createRecordReader ( InputSplit split , TaskAttemptContext context ) throws IOException , InterruptedException { RecordReader < LongWritable , MapWritable > reader = new LzoW3CLogRecordReader ( ) { @ Override protected String getFieldDefinitionFile ( ) { return fieldDefinitionFile ; } } ; reader . initialize ( split , context ) ; return reader ; } } ;


signature:
public class LzoProtobufSource { /** * Factory method for creating a new { @ code LzoProtobufSource } from a given path and protocol buffer
 * message class .
 * @ param path path to the data
 * @ param protoClass the Message class to read
 * @ return a new { @ code LzoProtobufSource } */
public static < S extends Message > LzoProtobufSource < S > at ( Path path , Class < S > protoClass ) { } }

implementation: return new LzoProtobufSource < S > ( path , EBTypes . protos ( protoClass ) ) ;


signature:
public class LzoProtobufSource { /** * Factory method for creating a new { @ code LzoProtobufSource } from the given paths and protocol buffer
 * message class .
 * @ param paths paths to the data
 * @ param protoClass the Message class to read
 * @ return a new { @ code LzoProtobufSource } */
public static < S extends Message > LzoProtobufSource < S > at ( List < Path > paths , Class < S > protoClass ) { } }

implementation: return new LzoProtobufSource < S > ( paths , EBTypes . protos ( protoClass ) ) ;


signature:
public class LzoBinaryBlockRecordReader { /** * Read the next key , value pair .
 * A small fraction of bad records in input are tolerated .
 * See { @ link LzoRecordReader } for more information on error handling .
 * @ return true if a key / value pair was read
 * @ throws IOException
 * @ throws InterruptedException */
@ Override public boolean nextKeyValue ( ) throws IOException , InterruptedException { } }

implementation:
// If we are past the end of the file split , tell the reader not to read any more new blocks .
// Then continue reading until the last of the reader ' s already - parsed values are used up .
while ( true ) { // loop to skip over bad records
if ( pos_ > end_ ) { reader_ . markNoMoreNewBlocks ( ) ; // Why not pos _ > = end _ , stop when we just reach the end ?
// we don ' t know if we have read all the bytes uncompressed in the current lzo block ,
// only way to make sure that we have read all of the split is to read till the
// first record that has at least one byte in the next split .
// As a consequence of this , next split reader skips at least one byte
// ( i . e . skips either partial or full record at the beginning ) .
} byte [ ] byteArray = reader_ . readNextProtoBytes ( ) ; if ( byteArray == null ) { return false ; } errorTracker . incRecords ( ) ; M decoded = null ; try { decoded = deserializer_ . fromBytes ( byteArray ) ; } catch ( DecodeException e ) { errorTracker . incErrors ( e ) ; HadoopCompat . incrementCounter ( recordErrorsCounter , 1 ) ; continue ; } if ( updatePosition ) { pos_ = getLzoFilePos ( ) ; updatePosition = false ; } if ( decoded != null ) { key_ . set ( pos_ ) ; value_ . set ( decoded ) ; pos_ = getLzoFilePos ( ) ; HadoopCompat . incrementCounter ( recordsReadCounter , 1 ) ; return true ; } else { HadoopCompat . incrementCounter ( recordsSkippedCounter , 1 ) ; } }


signature:
public class DeprecatedOutputFormatWrapper { /** * Sets jobs output format to { @ link DeprecatedOutputFormatWrapper } and stores
 * supplied real { @ link OutputFormat } class name in job configuration .
 * This configuration is read on the remote tasks to instantiate actual
 * OutputFormat correctly . */
public static void setOutputFormat ( Class < ? > realOutputFormatClass , JobConf jobConf ) { } }

implementation: jobConf . setOutputFormat ( DeprecatedOutputFormatWrapper . class ) ; HadoopUtils . setClassConf ( jobConf , CLASS_CONF_KEY , realOutputFormatClass ) ;


signature:
public class DeprecatedOutputFormatWrapper { /** * For cases where we need to set hadoop1 output format in a hadoop2 Configuration object . */
public static void setOutputFormat ( Class < ? > realOutputFormatClass , Configuration conf ) { } }

implementation: conf . setClass ( "mapred.output.format.class" , DeprecatedOutputFormatWrapper . class , org . apache . hadoop . mapred . OutputFormat . class ) ; HadoopUtils . setClassConf ( conf , CLASS_CONF_KEY , realOutputFormatClass ) ;


signature:
public class Inflection { /** * Does the given word match ?
 * @ param word The word
 * @ return True if it matches the inflection pattern */
public boolean match ( String word ) { } }

implementation: int flags = ignoreCase_ ? Pattern . CASE_INSENSITIVE : 0 ; return Pattern . compile ( pattern_ , flags ) . matcher ( word ) . find ( ) ;


signature:
public class Inflection { /** * Replace the word with its pattern .
 * @ param word The word
 * @ return The result */
public String replace ( String word ) { } }

implementation: int flags = ignoreCase_ ? Pattern . CASE_INSENSITIVE : 0 ; return Pattern . compile ( pattern_ , flags ) . matcher ( word ) . replaceAll ( replacement_ ) ;


signature:
public class Inflection { /** * Return the pluralized version of a word .
 * @ param word The word
 * @ return The pluralized word */
public static String pluralize ( String word ) { } }

implementation: if ( isUncountable ( word ) ) { return word ; } else { for ( Inflection inflection : plurals_ ) { if ( inflection . match ( word ) ) { return inflection . replace ( word ) ; } } return word ; }


signature:
public class Inflection { /** * Return true if the word is uncountable .
 * @ param word The word
 * @ return True if it is uncountable */
public static boolean isUncountable ( String word ) { } }

implementation: for ( String w : uncountables_ ) { if ( w . equalsIgnoreCase ( word ) ) { return true ; } } return false ;


signature:
public class LzoOutputFormat { /** * Helper method to create lzo output file needed to create RecordWriter */
protected DataOutputStream getOutputStream ( TaskAttemptContext job ) throws IOException , InterruptedException { } }

implementation: return LzoUtils . getIndexedLzoOutputStream ( HadoopCompat . getConfiguration ( job ) , getDefaultWorkFile ( job , LzopCodec . DEFAULT_LZO_EXTENSION ) ) ;


signature:
public class ThriftBinaryDeserializer { /** * Same as { @ link # deserialize ( TBase , byte [ ] ) } , but much more buffer copy friendly . */
public void deserialize ( TBase base , byte [ ] bytes , int offset , int len ) throws TException { } }

implementation:
// If incoming payload is larger than threshold then adjust the threshold by making it 10 % more
// than incoming payload - length and re - initialize object . This generally should not happen .
if ( len > lengthLimit ) { lengthLimit = len + Double . valueOf ( 0.1 * len ) . longValue ( ) ; protocol = new ThriftBinaryProtocol ( trans , lengthLimit , lengthLimit ) ; } resetAndInitialize ( protocol , len ) ; trans . reset ( bytes , offset , len ) ; base . read ( protocol ) ;


signature:
public class SequenceFileStorage { /** * Tests validity of Writable class , ensures consistent error message for both key and value
 * tests .
 * @ param writableClass class being tested .
 * @ param isKeyClass { @ code true } if testing keyClass , { @ code false } otherwise .
 * @ param writableConverter associated WritableConverter instance . */
private static < W extends Writable > void verifyWritableClass ( Class < W > writableClass , boolean isKeyClass , WritableConverter < W > writableConverter ) { } }

implementation: Preconditions . checkNotNull ( writableClass , "%s Writable class is undefined;" + " WritableConverter of type '%s' does not define default Writable type," + " and no type was specified by user" , isKeyClass ? "Key" : "Value" , writableConverter . getClass ( ) . getName ( ) ) ;


signature:
public class ThriftConverter { /** * Returns a ThriftConverter for a given Thrift class . */
public static < M extends TBase < ? , ? > > ThriftConverter < M > newInstance ( Class < M > tClass ) { } }

implementation: return new ThriftConverter < M > ( new TypeRef < M > ( tClass ) { } ) ;