signature:
public class ParquetMetadataConverter { /** * Visible for testing */
Type getType ( PrimitiveTypeName type ) { } }

implementation: switch ( type ) { case INT64 : return Type . INT64 ; case INT32 : return Type . INT32 ; case BOOLEAN : return Type . BOOLEAN ; case BINARY : return Type . BYTE_ARRAY ; case FLOAT : return Type . FLOAT ; case DOUBLE : return Type . DOUBLE ; case INT96 : return Type . INT96 ; case FIXED_LEN_BYTE_ARRAY : return Type . FIXED_LEN_BYTE_ARRAY ; default : throw new RuntimeException ( "Unknown primitive type " + type ) ; }


signature:
public class ParquetMetadataConverter { /** * Visible for testing */
LogicalTypeAnnotation getLogicalTypeAnnotation ( ConvertedType type , SchemaElement schemaElement ) { } }

implementation: switch ( type ) { case UTF8 : return LogicalTypeAnnotation . stringType ( ) ; case MAP : return LogicalTypeAnnotation . mapType ( ) ; case MAP_KEY_VALUE : return LogicalTypeAnnotation . MapKeyValueTypeAnnotation . getInstance ( ) ; case LIST : return LogicalTypeAnnotation . listType ( ) ; case ENUM : return LogicalTypeAnnotation . enumType ( ) ; case DECIMAL : int scale = ( schemaElement == null ? 0 : schemaElement . scale ) ; int precision = ( schemaElement == null ? 0 : schemaElement . precision ) ; return LogicalTypeAnnotation . decimalType ( scale , precision ) ; case DATE : return LogicalTypeAnnotation . dateType ( ) ; case TIME_MILLIS : return LogicalTypeAnnotation . timeType ( true , LogicalTypeAnnotation . TimeUnit . MILLIS ) ; case TIME_MICROS : return LogicalTypeAnnotation . timeType ( true , LogicalTypeAnnotation . TimeUnit . MICROS ) ; case TIMESTAMP_MILLIS : return LogicalTypeAnnotation . timestampType ( true , LogicalTypeAnnotation . TimeUnit . MILLIS ) ; case TIMESTAMP_MICROS : return LogicalTypeAnnotation . timestampType ( true , LogicalTypeAnnotation . TimeUnit . MICROS ) ; case INTERVAL : return LogicalTypeAnnotation . IntervalLogicalTypeAnnotation . getInstance ( ) ; case INT_8 : return LogicalTypeAnnotation . intType ( 8 , true ) ; case INT_16 : return LogicalTypeAnnotation . intType ( 16 , true ) ; case INT_32 : return LogicalTypeAnnotation . intType ( 32 , true ) ; case INT_64 : return LogicalTypeAnnotation . intType ( 64 , true ) ; case UINT_8 : return LogicalTypeAnnotation . intType ( 8 , false ) ; case UINT_16 : return LogicalTypeAnnotation . intType ( 16 , false ) ; case UINT_32 : return LogicalTypeAnnotation . intType ( 32 , false ) ; case UINT_64 : return LogicalTypeAnnotation . intType ( 64 , false ) ; case JSON : return LogicalTypeAnnotation . jsonType ( ) ; case BSON : return LogicalTypeAnnotation . bsonType ( ) ; default : throw new RuntimeException ( "Can't convert converted type to logical type, unknown converted type " + type ) ; }


signature:
public class ParquetMetadataConverter { /** * Visible for testing */
static FileMetaData filterFileMetaDataByMidpoint ( FileMetaData metaData , RangeMetadataFilter filter ) { } }

implementation: List < RowGroup > rowGroups = metaData . getRow_groups ( ) ; List < RowGroup > newRowGroups = new ArrayList < RowGroup > ( ) ; for ( RowGroup rowGroup : rowGroups ) { long totalSize = 0 ; long startIndex = getOffset ( rowGroup . getColumns ( ) . get ( 0 ) ) ; for ( ColumnChunk col : rowGroup . getColumns ( ) ) { totalSize += col . getMeta_data ( ) . getTotal_compressed_size ( ) ; } long midPoint = startIndex + totalSize / 2 ; if ( filter . contains ( midPoint ) ) { newRowGroups . add ( rowGroup ) ; } } metaData . setRow_groups ( newRowGroups ) ; return metaData ;


signature:
public class ParquetMetadataConverter { /** * Visible for testing */
static FileMetaData filterFileMetaDataByStart ( FileMetaData metaData , OffsetMetadataFilter filter ) { } }

implementation: List < RowGroup > rowGroups = metaData . getRow_groups ( ) ; List < RowGroup > newRowGroups = new ArrayList < RowGroup > ( ) ; for ( RowGroup rowGroup : rowGroups ) { long startIndex = getOffset ( rowGroup . getColumns ( ) . get ( 0 ) ) ; if ( filter . contains ( startIndex ) ) { newRowGroups . add ( rowGroup ) ; } } metaData . setRow_groups ( newRowGroups ) ; return metaData ;


signature:
public class ParquetMetadataConverter { /** * Visible for testing */
static long getOffset ( ColumnChunk columnChunk ) { } }

implementation: ColumnMetaData md = columnChunk . getMeta_data ( ) ; long offset = md . getData_page_offset ( ) ; if ( md . isSetDictionary_page_offset ( ) && offset > md . getDictionary_page_offset ( ) ) { offset = md . getDictionary_page_offset ( ) ; } return offset ;


signature:
public class ParquetMetadataConverter { /** * Visible for testing */
MessageType fromParquetSchema ( List < SchemaElement > schema , List < ColumnOrder > columnOrders ) { } }

implementation: Iterator < SchemaElement > iterator = schema . iterator ( ) ; SchemaElement root = iterator . next ( ) ; Types . MessageTypeBuilder builder = Types . buildMessage ( ) ; if ( root . isSetField_id ( ) ) { builder . id ( root . field_id ) ; } buildChildren ( builder , iterator , root . getNum_children ( ) , columnOrders , 0 ) ; return builder . named ( root . name ) ;


signature:
public class ParquetMetadataConverter { /** * Statistics are no longer saved in page headers */
@ Deprecated public void writeDataPageV2Header ( int uncompressedSize , int compressedSize , int valueCount , int nullCount , int rowCount , org . apache . parquet . column . statistics . Statistics statistics , org . apache . parquet . column . Encoding dataEncoding , int rlByteLength , int dlByteLength , OutputStream to ) throws IOException { } }

implementation: writePageHeader ( newDataPageV2Header ( uncompressedSize , compressedSize , valueCount , nullCount , rowCount , dataEncoding , rlByteLength , dlByteLength ) , to ) ;


signature:
public class LittleEndianDataOutputStream { /** * Writes a < code > long < / code > to the underlying output stream as eight
 * bytes , low byte first . In no exception is thrown , the counter
 * < code > written < / code > is incremented by < code > 8 < / code > .
 * @ param v a < code > long < / code > to be written .
 * @ exception IOException if an I / O error occurs .
 * @ see java . io . FilterOutputStream # out */
public final void writeLong ( long v ) throws IOException { } }

implementation: writeBuffer [ 7 ] = ( byte ) ( v >>> 56 ) ; writeBuffer [ 6 ] = ( byte ) ( v >>> 48 ) ; writeBuffer [ 5 ] = ( byte ) ( v >>> 40 ) ; writeBuffer [ 4 ] = ( byte ) ( v >>> 32 ) ; writeBuffer [ 3 ] = ( byte ) ( v >>> 24 ) ; writeBuffer [ 2 ] = ( byte ) ( v >>> 16 ) ; writeBuffer [ 1 ] = ( byte ) ( v >>> 8 ) ; writeBuffer [ 0 ] = ( byte ) ( v >>> 0 ) ; out . write ( writeBuffer , 0 , 8 ) ;


signature:
public class FilteredRecordReader { /** * Skips forwards until the filter finds the first match . Returns false
 * if none found . */
private void skipToMatch ( ) { } }

implementation:
while ( recordsRead < recordCount && ! recordFilter . isMatch ( ) ) { State currentState = getState ( 0 ) ; do { ColumnReader columnReader = currentState . column ; // currentLevel = depth + 1 at this point
// set the current value
if ( columnReader . getCurrentDefinitionLevel ( ) >= currentState . maxDefinitionLevel ) { columnReader . skip ( ) ; } columnReader . consume ( ) ; // Based on repetition level work out next state to go to
int nextR = currentState . maxRepetitionLevel == 0 ? 0 : columnReader . getCurrentRepetitionLevel ( ) ; currentState = currentState . getNextState ( nextR ) ; } while ( currentState != null ) ; ++ recordsRead ; }


signature:
public class SerializationUtil { /** * Writes an object to a configuration .
 * @ param key for the configuration
 * @ param obj the object to write
 * @ param conf to read from
 * @ throws IOException if there is an error while writing */
public static void writeObjectToConfAsBase64 ( String key , Object obj , Configuration conf ) throws IOException { } }

implementation: try ( ByteArrayOutputStream baos = new ByteArrayOutputStream ( ) ) { try ( GZIPOutputStream gos = new GZIPOutputStream ( baos ) ; ObjectOutputStream oos = new ObjectOutputStream ( gos ) ) { oos . writeObject ( obj ) ; } conf . set ( key , new String ( Base64 . encodeBase64 ( baos . toByteArray ( ) ) , StandardCharsets . UTF_8 ) ) ; }


signature:
public class SerializationUtil { /** * Reads an object ( that was written using
 * { @ link # writeObjectToConfAsBase64 } ) from a configuration
 * @ param key for the configuration
 * @ param conf to read from
 * @ param < T > the Java type of the deserialized object
 * @ return the read object , or null if key is not present in conf
 * @ throws IOException if there is an error while reading */
@ SuppressWarnings ( "unchecked" ) public static < T > T readObjectFromConfAsBase64 ( String key , Configuration conf ) throws IOException { } }

implementation: String b64 = conf . get ( key ) ; if ( b64 == null ) { return null ; } byte [ ] bytes = Base64 . decodeBase64 ( b64 . getBytes ( StandardCharsets . UTF_8 ) ) ; try ( ByteArrayInputStream bais = new ByteArrayInputStream ( bytes ) ; GZIPInputStream gis = new GZIPInputStream ( bais ) ; ObjectInputStream ois = new ObjectInputStream ( gis ) ) { return ( T ) ois . readObject ( ) ; } catch ( ClassNotFoundException e ) { throw new IOException ( "Could not read object from config with key " + key , e ) ; } catch ( ClassCastException e ) { throw new IOException ( "Couldn't cast object read from config with key " + key , e ) ; }


signature:
public class OffsetIndexBuilder { /** * Adds the specified parameters to this builder . Used by the writers to building up { @ link OffsetIndex } objects to be
 * written to the Parquet file .
 * @ param compressedPageSize
 * the size of the page ( including header )
 * @ param rowCount
 * the number of rows in the page */
public void add ( int compressedPageSize , long rowCount ) { } }

implementation: add ( previousOffset + previousPageSize , compressedPageSize , previousRowIndex + previousRowCount ) ; previousRowCount = rowCount ;


signature:
public class OffsetIndexBuilder { /** * Adds the specified parameters to this builder . Used by the metadata converter to building up { @ link OffsetIndex }
 * objects read from the Parquet file .
 * @ param offset
 * the offset of the page in the file
 * @ param compressedPageSize
 * the size of the page ( including header )
 * @ param firstRowIndex
 * the index of the first row in the page ( within the row group ) */
public void add ( long offset , int compressedPageSize , long firstRowIndex ) { } }

implementation: previousOffset = offset ; offsets . add ( offset ) ; previousPageSize = compressedPageSize ; compressedPageSizes . add ( compressedPageSize ) ; previousRowIndex = firstRowIndex ; firstRowIndexes . add ( firstRowIndex ) ;


signature:
public class OffsetIndexBuilder { /** * Builds the offset index . Used by the writers to building up { @ link OffsetIndex } objects to be
 * written to the Parquet file .
 * @ param firstPageOffset
 * the actual offset in the file to be used to translate all the collected offsets
 * @ return the newly created offset index or { @ code null } if the { @ link OffsetIndex } object would be empty */
public OffsetIndex build ( long firstPageOffset ) { } }

implementation: if ( compressedPageSizes . isEmpty ( ) ) { return null ; } long [ ] offsets = this . offsets . toLongArray ( ) ; if ( firstPageOffset != 0 ) { for ( int i = 0 , n = offsets . length ; i < n ; ++ i ) { offsets [ i ] += firstPageOffset ; } } OffsetIndexImpl offsetIndex = new OffsetIndexImpl ( ) ; offsetIndex . offsets = offsets ; offsetIndex . compressedPageSizes = compressedPageSizes . toIntArray ( ) ; offsetIndex . firstRowIndexes = firstRowIndexes . toLongArray ( ) ; return offsetIndex ;


signature:
public class LruCache { /** * Removes the mapping for the specified key from this cache if present .
 * @ param key key whose mapping is to be removed from the cache
 * @ return the previous value associated with key , or null if there was no
 * mapping for key . */
public V remove ( final K key ) { } }

implementation: V oldValue = cacheMap . remove ( key ) ; if ( oldValue != null ) { LOG . debug ( "Removed cache entry for '{}'" , key ) ; } return oldValue ;


signature:
public class LruCache { /** * Associates the specified value with the specified key in this cache . The
 * value is only inserted if it is not null and it is considered current . If
 * the cache previously contained a mapping for the key , the old value is
 * replaced only if the new value is " newer " than the old one .
 * @ param key key with which the specified value is to be associated
 * @ param newValue value to be associated with the specified key */
public void put ( final K key , final V newValue ) { } }

implementation:
if ( newValue == null || ! newValue . isCurrent ( key ) ) { if ( LOG . isWarnEnabled ( ) ) { LOG . warn ( "Ignoring new cache entry for '{}' because it is {}" , key , ( newValue == null ? "null" : "not current" ) ) ; } return ; } V oldValue = cacheMap . get ( key ) ; if ( oldValue != null && oldValue . isNewerThan ( newValue ) ) { if ( LOG . isWarnEnabled ( ) ) { LOG . warn ( "Ignoring new cache entry for '{}' because " + "existing cache entry is newer" , key ) ; } return ; } // no existing value or new value is newer than old value
oldValue = cacheMap . put ( key , newValue ) ; if ( LOG . isDebugEnabled ( ) ) { if ( oldValue == null ) { LOG . debug ( "Added new cache entry for '{}'" , key ) ; } else { LOG . debug ( "Overwrote existing cache entry for '{}'" , key ) ; } }


signature:
public class LruCache { /** * Returns the value to which the specified key is mapped , or null if 1 ) the
 * value is not current or 2 ) this cache contains no mapping for the key .
 * @ param key the key whose associated value is to be returned
 * @ return the value to which the specified key is mapped , or null if 1 ) the
 * value is not current or 2 ) this cache contains no mapping for the key */
public V getCurrentValue ( final K key ) { } }

implementation:
V value = cacheMap . get ( key ) ; LOG . debug ( "Value for '{}' {} in cache" , key , ( value == null ? "not " : "" ) ) ; if ( value != null && ! value . isCurrent ( key ) ) { // value is not current ; remove it and return null
remove ( key ) ; return null ; } return value ;


signature:
public class AvroWriteSupport { /** * Calls an appropriate write method based on the value .
 * Value MUST not be null .
 * @ param type the Parquet type
 * @ param avroSchema the Avro schema
 * @ param value a non - null value to write */
private void writeValue ( Type type , Schema avroSchema , Object value ) { } }

implementation: Schema nonNullAvroSchema = AvroSchemaConverter . getNonNull ( avroSchema ) ; LogicalType logicalType = nonNullAvroSchema . getLogicalType ( ) ; if ( logicalType != null ) { Conversion < ? > conversion = model . getConversionByClass ( value . getClass ( ) , logicalType ) ; writeValueWithoutConversion ( type , nonNullAvroSchema , convert ( nonNullAvroSchema , logicalType , conversion , value ) ) ; } else { writeValueWithoutConversion ( type , nonNullAvroSchema , value ) ; }


signature:
public class AvroWriteSupport { /** * Calls an appropriate write method based on the value .
 * Value must not be null and the schema must not be nullable .
 * @ param type a Parquet type
 * @ param avroSchema a non - nullable Avro schema
 * @ param value a non - null value to write */
@ SuppressWarnings ( "unchecked" ) private void writeValueWithoutConversion ( Type type , Schema avroSchema , Object value ) { } }

implementation: switch ( avroSchema . getType ( ) ) { case BOOLEAN : recordConsumer . addBoolean ( ( Boolean ) value ) ; break ; case INT : if ( value instanceof Character ) { recordConsumer . addInteger ( ( Character ) value ) ; } else { recordConsumer . addInteger ( ( ( Number ) value ) . intValue ( ) ) ; } break ; case LONG : recordConsumer . addLong ( ( ( Number ) value ) . longValue ( ) ) ; break ; case FLOAT : recordConsumer . addFloat ( ( ( Number ) value ) . floatValue ( ) ) ; break ; case DOUBLE : recordConsumer . addDouble ( ( ( Number ) value ) . doubleValue ( ) ) ; break ; case FIXED : recordConsumer . addBinary ( Binary . fromReusedByteArray ( ( ( GenericFixed ) value ) . bytes ( ) ) ) ; break ; case BYTES : if ( value instanceof byte [ ] ) { recordConsumer . addBinary ( Binary . fromReusedByteArray ( ( byte [ ] ) value ) ) ; } else { recordConsumer . addBinary ( Binary . fromReusedByteBuffer ( ( ByteBuffer ) value ) ) ; } break ; case STRING : recordConsumer . addBinary ( fromAvroString ( value ) ) ; break ; case RECORD : writeRecord ( type . asGroupType ( ) , avroSchema , value ) ; break ; case ENUM : recordConsumer . addBinary ( Binary . fromString ( value . toString ( ) ) ) ; break ; case ARRAY : listWriter . writeList ( type . asGroupType ( ) , avroSchema , value ) ; break ; case MAP : writeMap ( type . asGroupType ( ) , avroSchema , ( Map < CharSequence , ? > ) value ) ; break ; case UNION : writeUnion ( type . asGroupType ( ) , avroSchema , value ) ; break ;


signature:
public class PathGlobPattern { /** * Set and compile a glob pattern
 * @ param glob the glob pattern string */
public void set ( String glob ) { } }

implementation:
StringBuilder regex = new StringBuilder ( ) ; int setOpen = 0 ; int curlyOpen = 0 ; int len = glob . length ( ) ; hasWildcard = false ; for ( int i = 0 ; i < len ; i ++ ) { char c = glob . charAt ( i ) ; switch ( c ) { case BACKSLASH : if ( ++ i >= len ) { error ( "Missing escaped character" , glob , i ) ; } regex . append ( c ) . append ( glob . charAt ( i ) ) ; continue ; case '.' : case '$' : case '(' : case ')' : case '|' : case '+' : // escape regex special chars that are not glob special chars
regex . append ( BACKSLASH ) ; break ; case '*' : if ( i + 1 < len && glob . charAt ( i + 1 ) == '*' ) { regex . append ( '.' ) ; i ++ ; break ; } regex . append ( "[^" + PATH_SEPARATOR + "]" ) ; hasWildcard = true ; break ; case '?' : regex . append ( '.' ) ; hasWildcard = true ; continue ; case '{' : // start of a group
regex . append ( "(?:" ) ; // non - capturing
curlyOpen ++ ; hasWildcard = true ; continue ; case ',' : regex . append ( curlyOpen > 0 ? '|' : c ) ; continue ; case '}' : if ( curlyOpen > 0 ) { // end of a group
curlyOpen -- ; regex . append ( ")" ) ; continue ; } break ; case '[' : if ( setOpen > 0 ) { error ( "Unclosed character class" , glob , i ) ; } setOpen ++ ; hasWildcard = true ; break ; case '^' : // ^ inside [ . . . ] can be unescaped
if ( setOpen == 0 ) { regex . append ( BACKSLASH ) ; } break ; case '!' : // [ ! needs to be translated to [ ^
regex . append ( setOpen > 0 && '[' == glob . charAt ( i - 1 ) ? '^' : '!' ) ; continue ; case ']' : // Many set errors like [ ] [ ] could not be easily detected here ,
// as [ ] ] , [ ] - ] and [ - ] are all valid POSIX glob and java regex .
// We ' ll just let the regex compiler do the real work .
setOpen = 0 ; break ; default : } regex . append ( c ) ; } if ( setOpen > 0 ) { error ( "Unclosed character class" , glob , len ) ; } if ( curlyOpen > 0 ) { error ( "Unclosed group" , glob , len ) ; } compiled = Pattern . compile ( regex . toString ( ) ) ;


signature:
public class BaseCommand { /** * Output content to the console or a file .
 * This will not produce checksum files .
 * @ param content String content to write
 * @ param console A { @ link Logger } for writing to the console
 * @ param filename The destination { @ link Path } as a String
 * @ throws IOException if there is an error while writing */
public void output ( String content , Logger console , String filename ) throws IOException { } }

implementation: if ( filename == null || "-" . equals ( filename ) ) { console . info ( content ) ; } else { FSDataOutputStream outgoing = create ( filename ) ; try { outgoing . write ( content . getBytes ( StandardCharsets . UTF_8 ) ) ; } finally { outgoing . close ( ) ; } }


signature:
public class BaseCommand { /** * Returns a qualified { @ link Path } for the { @ code filename } .
 * If the file does not have a file system scheme , this uses the default FS .
 * @ param filename The filename to qualify
 * @ return A qualified Path for the filename
 * @ throws IOException if there is an error creating a qualified path */
public Path qualifiedPath ( String filename ) throws IOException { } }

implementation: Path cwd = defaultFS ( ) . makeQualified ( new Path ( "." ) ) ; return new Path ( filename ) . makeQualified ( defaultFS ( ) . getUri ( ) , cwd ) ;


signature:
public class BaseCommand { /** * Returns a { @ link URI } for the { @ code filename } that is a qualified Path or
 * a resource URI .
 * If the file does not have a file system scheme , this uses the default FS .
 * @ param filename The filename to qualify
 * @ return A qualified URI for the filename
 * @ throws IOException if there is an error creating a qualified URI */
public URI qualifiedURI ( String filename ) throws IOException { } }

implementation: URI fileURI = URI . create ( filename ) ; if ( RESOURCE_URI_SCHEME . equals ( fileURI . getScheme ( ) ) ) { return fileURI ; } else { return qualifiedPath ( filename ) . toUri ( ) ; }


signature:
public class BaseCommand { /** * Opens an existing file or resource .
 * If the file does not have a file system scheme , this uses the default FS .
 * @ param filename The filename to open .
 * @ return An open InputStream with the file contents
 * @ throws IOException if there is an error opening the file
 * @ throws IllegalArgumentException If the file does not exist */
public InputStream open ( String filename ) throws IOException { } }

implementation:
if ( STDIN_AS_SOURCE . equals ( filename ) ) { return System . in ; } URI uri = qualifiedURI ( filename ) ; if ( RESOURCE_URI_SCHEME . equals ( uri . getScheme ( ) ) ) { return Resources . getResource ( uri . getRawSchemeSpecificPart ( ) ) . openStream ( ) ; } else { Path filePath = new Path ( uri ) ; // even though it was qualified using the default FS , it may not be in it
FileSystem fs = filePath . getFileSystem ( getConf ( ) ) ; return fs . open ( filePath ) ; }


signature:
public class BaseCommand { /** * Returns a { @ link ClassLoader } for a set of jars and directories .
 * @ param jars A list of jar paths
 * @ param paths A list of directories containing . class files
 * @ return a classloader for the jars and paths
 * @ throws MalformedURLException if a jar or path is invalid */
protected static ClassLoader loaderFor ( List < String > jars , List < String > paths ) throws MalformedURLException { } }

implementation: return AccessController . doPrivileged ( new GetClassLoader ( urls ( jars , paths ) ) ) ;


signature:
public class BaseCommand { /** * Returns a { @ link ClassLoader } for a set of jars .
 * @ param jars A list of jar paths
 * @ return a classloader for the jars
 * @ throws MalformedURLException if a URL is invalid */
protected static ClassLoader loaderForJars ( List < String > jars ) throws MalformedURLException { } }

implementation: return AccessController . doPrivileged ( new GetClassLoader ( urls ( jars , null ) ) ) ;


signature:
public class BaseCommand { /** * Returns a { @ link ClassLoader } for a set of directories .
 * @ param paths A list of directories containing . class files
 * @ return a classloader for the paths
 * @ throws MalformedURLException if a path is invalid */
protected static ClassLoader loaderForPaths ( List < String > paths ) throws MalformedURLException { } }

implementation: return AccessController . doPrivileged ( new GetClassLoader ( urls ( null , paths ) ) ) ;


signature:
public class BitPackingValuesWriter { /** * { @ inheritDoc }
 * @ see org . apache . parquet . column . values . ValuesWriter # writeInteger ( int ) */
@ Override public void writeInteger ( int v ) { } }

implementation: try { bitPackingWriter . write ( v ) ; } catch ( IOException e ) { throw new ParquetEncodingException ( e ) ; }


signature:
public class BitPackingValuesWriter { /** * { @ inheritDoc }
 * @ see org . apache . parquet . column . values . ValuesWriter # getBytes ( ) */
@ Override public BytesInput getBytes ( ) { } }

implementation: try { this . bitPackingWriter . finish ( ) ; return BytesInput . from ( out ) ; } catch ( IOException e ) { throw new ParquetEncodingException ( e ) ; }


signature:
public class ColumnRecordFilter { /** * Factory method for record filter which applies the supplied predicate to the specified column .
 * Note that if searching for a repeated sub - attribute it will only ever match against the
 * first instance of it in the object .
 * @ param columnPath Dot separated path specifier , e . g . " engine . capacity "
 * @ param predicate Should call getBinary etc . and check the value
 * @ return a column filter */
public static final UnboundRecordFilter column ( final String columnPath , final ColumnPredicates . Predicate predicate ) { } }

implementation: checkNotNull ( columnPath , "columnPath" ) ; checkNotNull ( predicate , "predicate" ) ; return new UnboundRecordFilter ( ) { final String [ ] filterPath = columnPath . split ( "\\." ) ; @ Override public RecordFilter bind ( Iterable < ColumnReader > readers ) { for ( ColumnReader reader : readers ) { if ( Arrays . equals ( reader . getDescriptor ( ) . getPath ( ) , filterPath ) ) { return new ColumnRecordFilter ( reader , predicate ) ; } } throw new IllegalArgumentException ( "Column " + columnPath + " does not exist." ) ; } } ;


signature:
public class ColumnIndexFilterUtils { /** * Returns the filtered offset index containing only the pages which are overlapping with rowRanges . */
static OffsetIndex filterOffsetIndex ( OffsetIndex offsetIndex , RowRanges rowRanges , long totalRowCount ) { } }

implementation: IntList indexMap = new IntArrayList ( ) ; for ( int i = 0 , n = offsetIndex . getPageCount ( ) ; i < n ; ++ i ) { long from = offsetIndex . getFirstRowIndex ( i ) ; if ( rowRanges . isOverlapping ( from , offsetIndex . getLastRowIndex ( i , totalRowCount ) ) ) { indexMap . add ( i ) ; } } return new FilteredOffsetIndex ( offsetIndex , indexMap . toIntArray ( ) ) ;


signature:
public class ThriftSchemaConverter { /** * struct is assumed to contain valid structOrUnionType metadata when used with this method .
 * This method may throw if structOrUnionType is unknown .
 * Use convertWithoutProjection below to convert a StructType to MessageType
 * @ param struct the thrift type descriptor
 * @ return the struct as a Parquet message type */
public MessageType convert ( StructType struct ) { } }

implementation: MessageType messageType = ThriftSchemaConvertVisitor . convert ( struct , fieldProjectionFilter , true ) ; fieldProjectionFilter . assertNoUnmatchedPatterns ( ) ; return messageType ;


signature:
public class ThriftSchemaConverter { /** * struct is not required to have known structOrUnionType , which is useful
 * for converting a StructType from an ( older ) file schema to a MessageType
 * @ param struct the thrift type descriptor
 * @ return the struct as a Parquet message type */
public static MessageType convertWithoutProjection ( StructType struct ) { } }

implementation: return ThriftSchemaConvertVisitor . convert ( struct , FieldProjectionFilter . ALL_COLUMNS , false ) ;


signature:
public class ThriftSchemaConverter { /** * Returns whether the given type is the element type of a list or is a
 * synthetic group with one field that is the element type . This is
 * determined by checking whether the type can be a synthetic group and by
 * checking whether a potential synthetic group matches the expected
 * ThriftField .
 * This method never guesses because the expected ThriftField is known .
 * @ param repeatedType a type that may be the element type
 * @ param thriftElement the expected Schema for list elements
 * @ return { @ code true } if the repeatedType is the element schema */
static boolean isListElementType ( Type repeatedType , ThriftField thriftElement ) { } }

implementation:
if ( repeatedType . isPrimitive ( ) || ( repeatedType . asGroupType ( ) . getFieldCount ( ) != 1 ) || ( repeatedType . asGroupType ( ) . getType ( 0 ) . isRepetition ( REPEATED ) ) ) { // The repeated type must be the element type because it is an invalid
// synthetic wrapper . Must be a group with one optional or required field
return true ; } else if ( thriftElement != null && thriftElement . getType ( ) instanceof StructType ) { Set < String > fieldNames = new HashSet < String > ( ) ; for ( ThriftField field : ( ( StructType ) thriftElement . getType ( ) ) . getChildren ( ) ) { fieldNames . add ( field . getName ( ) ) ; } // If the repeated type is a subset of the structure of the ThriftField ,
// then it must be the element type .
return fieldNames . contains ( repeatedType . asGroupType ( ) . getFieldName ( 0 ) ) ; } return false ;


signature:
public class ConversionPatterns { /** * to preserve the difference between empty list and null when optional
 * @ param repetition repetition for the list or map
 * @ param alias name of the field
 * @ param logicalTypeAnnotation logical type for the list or map
 * @ param nested the nested repeated field
 * @ return a group type */
private static GroupType listWrapper ( Repetition repetition , String alias , LogicalTypeAnnotation logicalTypeAnnotation , Type nested ) { } }

implementation: if ( ! nested . isRepetition ( Repetition . REPEATED ) ) { throw new IllegalArgumentException ( "Nested type should be repeated: " + nested ) ; } return new GroupType ( repetition , alias , logicalTypeAnnotation , nested ) ;


signature:
public class ConversionPatterns { /** * Creates a 3 - level list structure annotated with LIST with elements of the
 * given elementType . The repeated level is inserted automatically and the
 * elementType ' s repetition should be the correct repetition of the elements ,
 * required for non - null and optional for nullable .
 * @ param listRepetition the repetition of the entire list structure
 * @ param name the name of the list structure type
 * @ param elementType the type of elements contained by the list
 * @ return a GroupType that represents the list */
public static GroupType listOfElements ( Repetition listRepetition , String name , Type elementType ) { } }

implementation: Preconditions . checkArgument ( elementType . getName ( ) . equals ( ELEMENT_NAME ) , "List element type must be named 'element'" ) ; return listWrapper ( listRepetition , name , LogicalTypeAnnotation . listType ( ) , new GroupType ( Repetition . REPEATED , "list" , elementType ) ) ;


signature:
public class ParquetStorer { /** * { @ inheritDoc } */
@ Override public void checkSchema ( ResourceSchema s ) throws IOException { } }

implementation: getProperties ( ) . setProperty ( SCHEMA , s . toString ( ) ) ;


signature:
public class ParquetStorer { /** * { @ inheritDoc } */
@ Override public OutputFormat < Void , Tuple > getOutputFormat ( ) throws IOException { } }

implementation: Schema pigSchema = getSchema ( ) ; return new ParquetOutputFormat < Tuple > ( new TupleWriteSupport ( pigSchema ) ) ;


signature:
public class ParquetStorer { /** * { @ inheritDoc } */
@ Override public void putNext ( Tuple tuple ) throws IOException { } }

implementation: try { this . recordWriter . write ( null , tuple ) ; } catch ( InterruptedException e ) { Thread . interrupted ( ) ; throw new ParquetEncodingException ( "Interrupted while writing" , e ) ; }


signature:
public class InitContext { /** * If there is a conflicting value when reading from multiple files ,
 * an exception will be thrown
 * @ return the merged key values metadata form the file footers */
@ Deprecated public Map < String , String > getMergedKeyValueMetaData ( ) { } }

implementation: if ( mergedKeyValueMetadata == null ) { Map < String , String > mergedKeyValues = new HashMap < String , String > ( ) ; for ( Entry < String , Set < String > > entry : keyValueMetadata . entrySet ( ) ) { if ( entry . getValue ( ) . size ( ) > 1 ) { throw new RuntimeException ( "could not merge metadata: key " + entry . getKey ( ) + " has conflicting values: " + entry . getValue ( ) ) ; } mergedKeyValues . put ( entry . getKey ( ) , entry . getValue ( ) . iterator ( ) . next ( ) ) ; } mergedKeyValueMetadata = mergedKeyValues ; } return mergedKeyValueMetadata ;


signature:
public class ParquetRecordReaderWrapper { /** * gets a ParquetInputSplit corresponding to a split given by Hive
 * @ param oldSplit The split given by Hive
 * @ param conf The JobConf of the Hive job
 * @ return a ParquetInputSplit corresponding to the oldSplit
 * @ throws IOException if the config cannot be enhanced or if the footer cannot be read from the file */
protected ParquetInputSplit getSplit ( final InputSplit oldSplit , final JobConf conf ) throws IOException { } }

implementation: if ( oldSplit instanceof FileSplit ) { FileSplit fileSplit = ( FileSplit ) oldSplit ; final long splitStart = fileSplit . getStart ( ) ; final long splitLength = fileSplit . getLength ( ) ; final Path finalPath = fileSplit . getPath ( ) ; final JobConf cloneJob = hiveBinding . pushProjectionsAndFilters ( conf , finalPath . getParent ( ) ) ; final ParquetMetadata parquetMetadata = ParquetFileReader . readFooter ( cloneJob , finalPath , SKIP_ROW_GROUPS ) ; final FileMetaData fileMetaData = parquetMetadata . getFileMetaData ( ) ; final ReadContext readContext = new DataWritableReadSupport ( ) . init ( cloneJob , fileMetaData . getKeyValueMetaData ( ) , fileMetaData . getSchema ( ) ) ; schemaSize = MessageTypeParser . parseMessageType ( readContext . getReadSupportMetadata ( ) . get ( DataWritableReadSupport . HIVE_SCHEMA_KEY ) ) . getFieldCount ( ) ; return new ParquetInputSplit ( finalPath , splitStart , splitStart + splitLength , splitLength , fileSplit . getLocations ( ) , null ) ; } else { throw new IllegalArgumentException ( "Unknown split type: " + oldSplit ) ; }


signature:
public class ParquetOutputFormat { /** * { @ inheritDoc } */
@ Override public RecordWriter < Void , T > getRecordWriter ( TaskAttemptContext taskAttemptContext ) throws IOException , InterruptedException { } }

implementation: final Configuration conf = getConfiguration ( taskAttemptContext ) ; CompressionCodecName codec = getCodec ( taskAttemptContext ) ; String extension = codec . getExtension ( ) + ".parquet" ; Path file = getDefaultWorkFile ( taskAttemptContext , extension ) ; return getRecordWriter ( conf , file , codec ) ;


signature:
public class LittleEndianDataInputStream { /** * See the general contract of the < code > skipBytes < / code >
 * method of < code > DataInput < / code > .
 * Bytes for this operation are read from the contained
 * input stream .
 * @ param n the number of bytes to be skipped .
 * @ return the actual number of bytes skipped .
 * @ exception IOException if the contained input stream does not support
 * seek , or the stream has been closed and
 * the contained input stream does not support
 * reading after close , or another I / O error occurs . */
public final int skipBytes ( int n ) throws IOException { } }

implementation: int total = 0 ; int cur = 0 ; while ( ( total < n ) && ( ( cur = ( int ) in . skip ( n - total ) ) > 0 ) ) { total += cur ; } return total ;


signature:
public class LittleEndianDataInputStream { /** * Bytes
 * for this operation are read from the contained
 * input stream .
 * @ return the next two bytes of this input stream , interpreted as an
 * unsigned 16 - bit integer .
 * @ exception EOFException if this input stream reaches the end before
 * reading two bytes .
 * @ exception IOException the stream has been closed and the contained
 * input stream does not support reading after close , or
 * another I / O error occurs .
 * @ see java . io . FilterInputStream # in */
public final int readUnsignedShort ( ) throws IOException { } }

implementation: int ch2 = in . read ( ) ; int ch1 = in . read ( ) ; if ( ( ch1 | ch2 ) < 0 ) throw new EOFException ( ) ; return ( ch1 << 8 ) + ( ch2 << 0 ) ;


signature:
public class LittleEndianDataInputStream { /** * Bytes
 * for this operation are read from the contained
 * input stream .
 * @ return the next eight bytes of this input stream , interpreted as a
 * < code > long < / code > .
 * @ exception EOFException if this input stream reaches the end before
 * reading eight bytes .
 * @ exception IOException the stream has been closed and the contained
 * input stream does not support reading after close , or
 * another I / O error occurs .
 * @ see java . io . FilterInputStream # in */
public final long readLong ( ) throws IOException { } }

implementation:
// TODO : see perf question above in readInt
readFully ( readBuffer , 0 , 8 ) ; return ( ( ( long ) readBuffer [ 7 ] << 56 ) + ( ( long ) ( readBuffer [ 6 ] & 255 ) << 48 ) + ( ( long ) ( readBuffer [ 5 ] & 255 ) << 40 ) + ( ( long ) ( readBuffer [ 4 ] & 255 ) << 32 ) + ( ( long ) ( readBuffer [ 3 ] & 255 ) << 24 ) + ( ( readBuffer [ 2 ] & 255 ) << 16 ) + ( ( readBuffer [ 1 ] & 255 ) << 8 ) + ( ( readBuffer [ 0 ] & 255 ) << 0 ) ) ;


signature:
public class ParquetInputSplit { /** * Builds a { @ code ParquetInputSplit } from a mapreduce { @ link FileSplit } .
 * @ param split a mapreduce FileSplit
 * @ return a ParquetInputSplit
 * @ throws IOException if there is an error while creating the Parquet split */
static ParquetInputSplit from ( FileSplit split ) throws IOException { } }

implementation: return new ParquetInputSplit ( split . getPath ( ) , split . getStart ( ) , split . getStart ( ) + split . getLength ( ) , split . getLength ( ) , split . getLocations ( ) , null ) ;


signature:
public class ParquetInputSplit { /** * Builds a { @ code ParquetInputSplit } from a mapred
 * { @ link org . apache . hadoop . mapred . FileSplit } .
 * @ param split a mapred FileSplit
 * @ return a ParquetInputSplit
 * @ throws IOException if there is an error while creating the Parquet split */
static ParquetInputSplit from ( org . apache . hadoop . mapred . FileSplit split ) throws IOException { } }


signature:
public class ParquetInputSplit { /** * { @ inheritDoc } */
@ Override public void readFields ( DataInput hin ) throws IOException { } }

implementation: byte [ ] bytes = readArray ( hin ) ; DataInputStream in = new DataInputStream ( new GZIPInputStream ( new ByteArrayInputStream ( bytes ) ) ) ; super . readFields ( in ) ; this . end = in . readLong ( ) ; if ( in . readBoolean ( ) ) { this . rowGroupOffsets = new long [ in . readInt ( ) ] ; for ( int i = 0 ; i < rowGroupOffsets . length ; i ++ ) { rowGroupOffsets [ i ] = in . readLong ( ) ; } } in . close ( ) ;


signature:
public class ParquetInputSplit { /** * { @ inheritDoc } */
@ Override public void write ( DataOutput hout ) throws IOException { } }

implementation: ByteArrayOutputStream baos = new ByteArrayOutputStream ( ) ; DataOutputStream out = new DataOutputStream ( new GZIPOutputStream ( baos ) ) ; super . write ( out ) ; out . writeLong ( end ) ; out . writeBoolean ( rowGroupOffsets != null ) ; if ( rowGroupOffsets != null ) { out . writeInt ( rowGroupOffsets . length ) ; for ( long o : rowGroupOffsets ) { out . writeLong ( o ) ; } } out . close ( ) ; writeArray ( hout , baos . toByteArray ( ) ) ;


signature:
public class Types { /** * Returns a builder to construct a required { @ link PrimitiveType } .
 * @ param type a { @ link PrimitiveTypeName } for the constructed type
 * @ return a { @ link PrimitiveBuilder } */
public static PrimitiveBuilder < PrimitiveType > required ( PrimitiveTypeName type ) { } }

implementation: return new PrimitiveBuilder < PrimitiveType > ( PrimitiveType . class , type ) . repetition ( Type . Repetition . REQUIRED ) ;


signature:
public class Types { /** * Returns a builder to construct an optional { @ link PrimitiveType } .
 * @ param type a { @ link PrimitiveTypeName } for the constructed type
 * @ return a { @ link PrimitiveBuilder } */
public static PrimitiveBuilder < PrimitiveType > optional ( PrimitiveTypeName type ) { } }

implementation: return new PrimitiveBuilder < PrimitiveType > ( PrimitiveType . class , type ) . repetition ( Type . Repetition . OPTIONAL ) ;


signature:
public class Types { /** * Returns a builder to construct a repeated { @ link PrimitiveType } .
 * @ param type a { @ link PrimitiveTypeName } for the constructed type
 * @ return a { @ link PrimitiveBuilder } */
public static PrimitiveBuilder < PrimitiveType > repeated ( PrimitiveTypeName type ) { } }

implementation: return new PrimitiveBuilder < PrimitiveType > ( PrimitiveType . class , type ) . repetition ( Type . Repetition . REPEATED ) ;


signature:
public class Types { /** * Returns a builder to construct a required { @ link GroupType } .
 * @ return a { @ link GroupBuilder } */
public static GroupBuilder < GroupType > requiredGroup ( ) { } }

implementation: return new GroupBuilder < GroupType > ( GroupType . class ) . repetition ( Type . Repetition . REQUIRED ) ;


signature:
public class Types { /** * Returns a builder to construct an optional { @ link GroupType } .
 * @ return a { @ link GroupBuilder } */
public static GroupBuilder < GroupType > optionalGroup ( ) { } }

implementation: return new GroupBuilder < GroupType > ( GroupType . class ) . repetition ( Type . Repetition . OPTIONAL ) ;


signature:
public class Types { /** * Returns a builder to construct a repeated { @ link GroupType } .
 * @ return a { @ link GroupBuilder } */
public static GroupBuilder < GroupType > repeatedGroup ( ) { } }

implementation: return new GroupBuilder < GroupType > ( GroupType . class ) . repetition ( Type . Repetition . REPEATED ) ;


signature:
public class IntList { /** * ( not an actual Iterable )
 * @ return an IntIterator on the content */
public IntIterator iterator ( ) { } }

implementation: if ( currentSlab == null ) { allocateSlab ( ) ; } int [ ] [ ] itSlabs = slabs . toArray ( new int [ slabs . size ( ) + 1 ] [ ] ) ; itSlabs [ slabs . size ( ) ] = currentSlab ; return new IntIterator ( itSlabs , size ( ) ) ;


signature:
public class CorruptStatistics { /** * Decides if the statistics from a file created by createdBy ( the created _ by field from parquet format )
 * should be ignored because they are potentially corrupt .
 * @ param createdBy the created - by string from a file footer
 * @ param columnType the type of the column that this is checking
 * @ return true if the statistics may be invalid and should be ignored , false otherwise */
public static boolean shouldIgnoreStatistics ( String createdBy , PrimitiveTypeName columnType ) { } }

implementation:
if ( columnType != PrimitiveTypeName . BINARY && columnType != PrimitiveTypeName . FIXED_LEN_BYTE_ARRAY ) { // the bug only applies to binary columns
return false ; } if ( Strings . isNullOrEmpty ( createdBy ) ) { // created _ by is not populated , which could have been caused by
// parquet - mr during the same time as PARQUET - 251 , see PARQUET - 297
warnOnce ( "Ignoring statistics because created_by is null or empty! See PARQUET-251 and PARQUET-297" ) ; return true ; } try { ParsedVersion version = VersionParser . parse ( createdBy ) ; if ( ! "parquet-mr" . equals ( version . application ) ) { // assume other applications don ' t have this bug
return false ; } if ( Strings . isNullOrEmpty ( version . version ) ) { warnOnce ( "Ignoring statistics because created_by did not contain a semver (see PARQUET-251): " + createdBy ) ; return true ; } SemanticVersion semver = SemanticVersion . parse ( version . version ) ; if ( semver . compareTo ( PARQUET_251_FIXED_VERSION ) < 0 && ! ( semver . compareTo ( CDH_5_PARQUET_251_FIXED_START ) >= 0 && semver . compareTo ( CDH_5_PARQUET_251_FIXED_END ) < 0 ) ) { warnOnce ( "Ignoring statistics because this file was created prior to " + PARQUET_251_FIXED_VERSION + ", see PARQUET-251" ) ; return true ; } // this file was created after the fix
return false ; } catch ( RuntimeException e ) { // couldn ' t parse the created _ by field , log what went wrong , don ' t trust the stats ,
// but don ' t make this fatal .
warnParseErrorOnce ( createdBy , e ) ; return true ; } catch ( SemanticVersionParseException e ) { // couldn ' t parse the created _ by field , log what went wrong , don ' t trust the stats ,
// but don ' t make this fatal .
warnParseErrorOnce ( createdBy , e ) ; return true ; } catch ( VersionParseException e ) { // couldn ' t parse the created _ by field , log what went wrong , don ' t trust the stats ,
// but don ' t make this fatal .
warnParseErrorOnce ( createdBy , e ) ; return true ; }


signature:
public class RecordReaderImplementation { /** * TODO : have those wrappers for a converter */
private RecordConsumer validator ( RecordConsumer recordConsumer , boolean validating , MessageType schema ) { } }

implementation: return validating ? new ValidatingRecordConsumer ( recordConsumer , schema ) : recordConsumer ;


signature:
public class AvroRecordConverter { /** * this was taken from Avro ' s ReflectData */
private static Map < String , Class < ? > > getFieldsByName ( Class < ? > recordClass , boolean excludeJava ) { } }

implementation:
Map < String , Class < ? > > fields = new LinkedHashMap < String , Class < ? > > ( ) ; if ( recordClass != null ) { Class < ? > current = recordClass ; do { if ( excludeJava && current . getPackage ( ) != null && current . getPackage ( ) . getName ( ) . startsWith ( "java." ) ) { break ; // skip java built - in classes
} for ( Field field : current . getDeclaredFields ( ) ) { if ( field . isAnnotationPresent ( AvroIgnore . class ) || isTransientOrStatic ( field ) ) { continue ; } AvroName altName = field . getAnnotation ( AvroName . class ) ; Class < ? > existing = fields . put ( altName != null ? altName . value ( ) : field . getName ( ) , field . getType ( ) ) ; if ( existing != null ) { throw new AvroTypeException ( current + " contains two fields named: " + field . getName ( ) ) ; } } current = current . getSuperclass ( ) ; } while ( current != null ) ; } return fields ;


signature:
public class AvroRecordConverter { /** * Returns whether the given type is the element type of a list or is a
 * synthetic group with one field that is the element type . This is
 * determined by checking whether the type can be a synthetic group and by
 * checking whether a potential synthetic group matches the expected schema .
 * Unlike { @ link AvroSchemaConverter # isElementType ( Type , String ) } , this
 * method never guesses because the expected schema is known .
 * @ param repeatedType a type that may be the element type
 * @ param elementSchema the expected Schema for list elements
 * @ return { @ code true } if the repeatedType is the element schema */
static boolean isElementType ( Type repeatedType , Schema elementSchema ) { } }

implementation:
if ( repeatedType . isPrimitive ( ) || repeatedType . asGroupType ( ) . getFieldCount ( ) > 1 || repeatedType . asGroupType ( ) . getType ( 0 ) . isRepetition ( REPEATED ) ) { // The repeated type must be the element type because it is an invalid
// synthetic wrapper . Must be a group with one optional or required field
return true ; } else if ( elementSchema != null && elementSchema . getType ( ) == Schema . Type . RECORD ) { Schema schemaFromRepeated = CONVERTER . convert ( repeatedType . asGroupType ( ) ) ; if ( checkReaderWriterCompatibility ( elementSchema , schemaFromRepeated ) . getType ( ) == COMPATIBLE ) { return true ; } } return false ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeMessageBegin ( org . apache . thrift . protocol . TMessage ) */
@ Override public void writeMessageBegin ( TMessage message ) throws TException { } }

implementation: LOG . debug ( "writeMessageBegin({})" , message ) ; currentProtocol . writeMessageBegin ( message ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeStructBegin ( org . apache . thrift . protocol . TStruct ) */
@ Override public void writeStructBegin ( TStruct struct ) throws TException { } }

implementation: if ( LOG . isDebugEnabled ( ) ) LOG . debug ( "writeStructBegin(" + toString ( struct ) + ")" ) ; currentProtocol . writeStructBegin ( struct ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeFieldBegin ( org . apache . thrift . protocol . TField ) */
@ Override public void writeFieldBegin ( TField field ) throws TException { } }

implementation: LOG . debug ( "writeFieldBegin({})" , field ) ; currentProtocol . writeFieldBegin ( field ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeMapBegin ( org . apache . thrift . protocol . TMap ) */
@ Override public void writeMapBegin ( TMap map ) throws TException { } }

implementation: if ( LOG . isDebugEnabled ( ) ) LOG . debug ( "writeMapBegin(" + toString ( map ) + ")" ) ; currentProtocol . writeMapBegin ( map ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeListBegin ( org . apache . thrift . protocol . TList ) */
@ Override public void writeListBegin ( TList list ) throws TException { } }

implementation: if ( LOG . isDebugEnabled ( ) ) LOG . debug ( "writeListBegin(" + toString ( list ) + ")" ) ; currentProtocol . writeListBegin ( list ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeSetBegin ( org . apache . thrift . protocol . TSet ) */
@ Override public void writeSetBegin ( TSet set ) throws TException { } }

implementation: LOG . debug ( "writeSetBegin({})" , set ) ; currentProtocol . writeSetBegin ( set ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeBool ( boolean ) */
@ Override public void writeBool ( boolean b ) throws TException { } }

implementation: LOG . debug ( "writeBool({})" , b ) ; currentProtocol . writeBool ( b ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeByte ( byte ) */
@ Override public void writeByte ( byte b ) throws TException { } }

implementation: LOG . debug ( "writeByte({})" , b ) ; currentProtocol . writeByte ( b ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeI16 ( short ) */
@ Override public void writeI16 ( short i16 ) throws TException { } }

implementation: LOG . debug ( "writeI16({})" , i16 ) ; currentProtocol . writeI16 ( i16 ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeI32 ( int ) */
@ Override public void writeI32 ( int i32 ) throws TException { } }

implementation: LOG . debug ( "writeI32({})" , i32 ) ; currentProtocol . writeI32 ( i32 ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeI64 ( long ) */
@ Override public void writeI64 ( long i64 ) throws TException { } }

implementation: LOG . debug ( "writeI64({})" , i64 ) ; currentProtocol . writeI64 ( i64 ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeDouble ( double ) */
@ Override public void writeDouble ( double dub ) throws TException { } }

implementation: LOG . debug ( "writeDouble({})" , dub ) ; currentProtocol . writeDouble ( dub ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeString ( java . lang . String ) */
@ Override public void writeString ( String str ) throws TException { } }

implementation: LOG . debug ( "writeString({})" , str ) ; currentProtocol . writeString ( str ) ;


signature:
public class ParquetWriteProtocol { /** * { @ inheritDoc }
 * @ see org . apache . parquet . thrift . ParquetProtocol # writeBinary ( java . nio . ByteBuffer ) */
@ Override public void writeBinary ( ByteBuffer buf ) throws TException { } }

implementation: LOG . debug ( "writeBinary({})" , buf ) ; currentProtocol . writeBinary ( buf ) ;


signature:
public class DataWritableReadSupport { /** * Determine the file column names based on the position within the requested columns and
 * use that as the requested schema . */
private MessageType resolveSchemaAccess ( MessageType requestedSchema , MessageType fileSchema , Configuration configuration ) { } }

implementation: if ( configuration . getBoolean ( PARQUET_COLUMN_INDEX_ACCESS , false ) ) { final List < String > listColumns = getColumns ( configuration . get ( IOConstants . COLUMNS ) ) ; List < Type > requestedTypes = new ArrayList < Type > ( ) ; for ( Type t : requestedSchema . getFields ( ) ) { int index = listColumns . indexOf ( t . getName ( ) ) ; requestedTypes . add ( fileSchema . getType ( index ) ) ; } requestedSchema = new MessageType ( requestedSchema . getName ( ) , requestedTypes ) ; } return requestedSchema ;


signature:
public class RecordConsumerLoggingWrapper { /** * { @ inheritDoc } */
@ Override public void startField ( String field , int index ) { } }

implementation: logOpen ( field ) ; delegate . startField ( field , index ) ;


signature:
public class RecordConsumerLoggingWrapper { /** * { @ inheritDoc } */
@ Override public void addBinary ( Binary value ) { } }

implementation: if ( LOG . isDebugEnabled ( ) ) log ( Arrays . toString ( value . getBytesUnsafe ( ) ) ) ; delegate . addBinary ( value ) ;


signature:
public class RecordConsumerLoggingWrapper { /** * { @ inheritDoc } */
@ Override public void endField ( String field , int index ) { } }

implementation: logClose ( field ) ; delegate . endField ( field , index ) ;


signature:
public class MergeCommand { /** * Get all input files .
 * @ param input input files or directory .
 * @ return ordered input files . */
private List < Path > getInputFiles ( List < String > input ) throws IOException { } }

implementation: List < Path > inputFiles = null ; if ( input . size ( ) == 1 ) { Path p = new Path ( input . get ( 0 ) ) ; FileSystem fs = p . getFileSystem ( conf ) ; FileStatus status = fs . getFileStatus ( p ) ; if ( status . isDir ( ) ) { inputFiles = getInputFilesFromDirectory ( status ) ; } } else { inputFiles = parseInputFiles ( input ) ; } checkParquetFiles ( inputFiles ) ; return inputFiles ;


signature:
public class MergeCommand { /** * Check input files basically .
 * ParquetFileReader will throw exception when reading an illegal parquet file .
 * @ param inputFiles files to be merged .
 * @ throws IOException */
private void checkParquetFiles ( List < Path > inputFiles ) throws IOException { } }

implementation: if ( inputFiles == null || inputFiles . size ( ) <= 1 ) { throw new IllegalArgumentException ( "Not enough files to merge" ) ; } for ( Path inputFile : inputFiles ) { FileSystem fs = inputFile . getFileSystem ( conf ) ; FileStatus status = fs . getFileStatus ( inputFile ) ; if ( status . isDir ( ) ) { throw new IllegalArgumentException ( "Illegal parquet file: " + inputFile . toUri ( ) ) ; } }


signature:
public class MergeCommand { /** * Get all parquet files under partition directory .
 * @ param partitionDir partition directory .
 * @ return parquet files to be merged . */
private List < Path > getInputFilesFromDirectory ( FileStatus partitionDir ) throws IOException { } }

implementation: FileSystem fs = partitionDir . getPath ( ) . getFileSystem ( conf ) ; FileStatus [ ] inputFiles = fs . listStatus ( partitionDir . getPath ( ) , HiddenFileFilter . INSTANCE ) ; List < Path > input = new ArrayList < Path > ( ) ; for ( FileStatus f : inputFiles ) { input . add ( f . getPath ( ) ) ; } return input ;


signature:
public class BitPackingValuesReader { /** * { @ inheritDoc }
 * @ see org . apache . parquet . column . values . ValuesReader # initFromPage ( int , ByteBufferInputStream ) */
@ Override public void initFromPage ( int valueCount , ByteBufferInputStream stream ) throws IOException { } }

implementation: int effectiveBitLength = valueCount * bitsPerValue ; int length = BytesUtils . paddedByteCountFromBits ( effectiveBitLength ) ; LOG . debug ( "reading {} bytes for {} values of size {} bits." , length , valueCount , bitsPerValue ) ; this . in = stream . sliceStream ( length ) ; this . bitPackingReader = createBitPackingReader ( bitsPerValue , this . in , valueCount ) ; updateNextOffset ( length ) ;


signature:
public class PagedRecordFilter { /** * Returns builder for creating a paged query .
 * @ param startPos The record to start from , numbering starts at 1.
 * @ param pageSize The size of the page .
 * @ return a paged record filter */
public static final UnboundRecordFilter page ( final long startPos , final long pageSize ) { } }

implementation: return new UnboundRecordFilter ( ) { @ Override public RecordFilter bind ( Iterable < ColumnReader > readers ) { return new PagedRecordFilter ( startPos , pageSize ) ; } } ;


signature:
public class DelegatingSeekableInputStream { /** * Visible for testing */
static void readFully ( InputStream f , byte [ ] bytes , int start , int len ) throws IOException { } }

implementation: int offset = start ; int remaining = len ; while ( remaining > 0 ) { int bytesRead = f . read ( bytes , offset , remaining ) ; if ( bytesRead < 0 ) { throw new EOFException ( "Reached the end of stream with " + remaining + " bytes left to read" ) ; } remaining -= bytesRead ; offset += bytesRead ; }


signature:
public class DelegatingSeekableInputStream { /** * Visible for testing */
static int readHeapBuffer ( InputStream f , ByteBuffer buf ) throws IOException { } }

implementation:
int bytesRead = f . read ( buf . array ( ) , buf . arrayOffset ( ) + buf . position ( ) , buf . remaining ( ) ) ; if ( bytesRead < 0 ) { // if this resulted in EOF , don ' t update position
return bytesRead ; } else { buf . position ( buf . position ( ) + bytesRead ) ; return bytesRead ; }


signature:
public class DelegatingSeekableInputStream { /** * Visible for testing */
static void readFullyHeapBuffer ( InputStream f , ByteBuffer buf ) throws IOException { } }

implementation: readFully ( f , buf . array ( ) , buf . arrayOffset ( ) + buf . position ( ) , buf . remaining ( ) ) ; buf . position ( buf . limit ( ) ) ;


signature:
public class DelegatingSeekableInputStream { /** * Visible for testing */
static int readDirectBuffer ( InputStream f , ByteBuffer buf , byte [ ] temp ) throws IOException { } }

implementation:
// copy all the bytes that return immediately , stopping at the first
// read that doesn ' t return a full buffer .
int nextReadLength = Math . min ( buf . remaining ( ) , temp . length ) ; int totalBytesRead = 0 ; int bytesRead ; while ( ( bytesRead = f . read ( temp , 0 , nextReadLength ) ) == temp . length ) { buf . put ( temp ) ; totalBytesRead += bytesRead ; nextReadLength = Math . min ( buf . remaining ( ) , temp . length ) ; } if ( bytesRead < 0 ) { // return - 1 if nothing was read
return totalBytesRead == 0 ? - 1 : totalBytesRead ; } else { // copy the last partial buffer
buf . put ( temp , 0 , bytesRead ) ; totalBytesRead += bytesRead ; return totalBytesRead ; }


signature:
public class DelegatingSeekableInputStream { /** * Visible for testing */
static void readFullyDirectBuffer ( InputStream f , ByteBuffer buf , byte [ ] temp ) throws IOException { } }

implementation: int nextReadLength = Math . min ( buf . remaining ( ) , temp . length ) ; int bytesRead = 0 ; while ( nextReadLength > 0 && ( bytesRead = f . read ( temp , 0 , nextReadLength ) ) >= 0 ) { buf . put ( temp , 0 , bytesRead ) ; nextReadLength = Math . min ( buf . remaining ( ) , temp . length ) ; } if ( bytesRead < 0 && buf . remaining ( ) > 0 ) { throw new EOFException ( "Reached the end of stream with " + buf . remaining ( ) + " bytes left to read" ) ; }


signature:
public class Consumers { /** * To consume a list of elements
 * @ param c the class of the list content
 * @ param consumer the consumer that will receive the list
 * @ param < T > the type of the list content
 * @ return a ListConsumer that can be passed to the DelegatingFieldConsumer */
public static < T extends TBase < T , ? extends TFieldIdEnum > > ListConsumer listOf ( Class < T > c , final Consumer < List < T > > consumer ) { } }

implementation: class ListConsumer implements Consumer < T > { List < T > list ; @ Override public void consume ( T t ) { list . add ( t ) ; } } final ListConsumer co = new ListConsumer ( ) ; return new DelegatingListElementsConsumer ( struct ( c , co ) ) { @ Override public void consumeList ( TProtocol protocol , EventBasedThriftReader reader , TList tList ) throws TException { co . list = new ArrayList < T > ( ) ; super . consumeList ( protocol , reader , tList ) ; consumer . consume ( co . list ) ; } } ;


signature:
public class Hive010Binding { /** * Initialize the mrwork variable in order to get all the partition and start to update the jobconf
 * @ param job */
private void init ( final JobConf job ) { } }

implementation: final String plan = HiveConf . getVar ( job , HiveConf . ConfVars . PLAN ) ; if ( mrwork == null && plan != null && plan . length ( ) > 0 ) { mrwork = Utilities . getMapRedWork ( job ) ; pathToPartitionInfo . clear ( ) ; for ( final Map . Entry < String , PartitionDesc > entry : mrwork . getPathToPartitionInfo ( ) . entrySet ( ) ) { pathToPartitionInfo . put ( new Path ( entry . getKey ( ) ) . toUri ( ) . getPath ( ) . toString ( ) , entry . getValue ( ) ) ; } }


signature:
public class Hive010Binding { /** * { @ inheritDoc } */
@ Override public JobConf pushProjectionsAndFilters ( JobConf jobConf , Path path ) throws IOException { } }

implementation: init ( jobConf ) ; final JobConf cloneJobConf = new JobConf ( jobConf ) ; final PartitionDesc part = pathToPartitionInfo . get ( path . toString ( ) ) ; if ( ( part != null ) && ( part . getTableDesc ( ) != null ) ) { Utilities . copyTableJobPropertiesToConf ( part . getTableDesc ( ) , cloneJobConf ) ; } pushProjectionsAndFilters ( cloneJobConf , path . toString ( ) , path . toUri ( ) . toString ( ) ) ; return cloneJobConf ;


signature:
public class ProtoSchemaConverter { /** * / * Iterates over list of fields . * */
private < T > GroupBuilder < T > convertFields ( GroupBuilder < T > groupBuilder , List < FieldDescriptor > fieldDescriptors ) { } }

implementation: for ( FieldDescriptor fieldDescriptor : fieldDescriptors ) { groupBuilder = addField ( fieldDescriptor , groupBuilder ) . id ( fieldDescriptor . getNumber ( ) ) . named ( fieldDescriptor . getName ( ) ) ; } return groupBuilder ;


signature:
public class PigSchemaConverter { /** * filters a Parquet schema based on a pig schema for projection
 * @ param schemaToFilter the schema to be filter
 * @ param requestedPigSchema the pig schema to filter it with
 * @ return the resulting filtered schema */
public MessageType filter ( MessageType schemaToFilter , Schema requestedPigSchema ) { } }

implementation: return filter ( schemaToFilter , requestedPigSchema , null ) ;


signature:
public class PigSchemaConverter { /** * filters a Parquet schema based on a pig schema for projection
 * @ param schemaToFilter the schema to be filter
 * @ param requestedPigSchema the pig schema to filter it with
 * @ param requiredFieldList projected required fields
 * @ return the resulting filtered schema */
public MessageType filter ( MessageType schemaToFilter , Schema requestedPigSchema , RequiredFieldList requiredFieldList ) { } }

implementation: try { if ( LOG . isDebugEnabled ( ) ) LOG . debug ( "filtering schema:\n" + schemaToFilter + "\nwith requested pig schema:\n " + requestedPigSchema ) ; List < Type > result = columnAccess . filterTupleSchema ( schemaToFilter , requestedPigSchema , requiredFieldList ) ; if ( LOG . isDebugEnabled ( ) ) LOG . debug ( "schema:\n" + schemaToFilter + "\nfiltered to:\n" + result ) ; return new MessageType ( schemaToFilter . getName ( ) , result ) ; } catch ( RuntimeException e ) { throw new RuntimeException ( "can't filter " + schemaToFilter + " with " + requestedPigSchema , e ) ; }


signature:
public class AvroCSV { /** * Create a { @ link Schema } for the given type . If the type is null ,
 * the schema will be a nullable String . If isNullable is true , the returned
 * schema will be nullable .
 * @ param type a { @ link Schema . Type } compatible with { @ code Schema . create }
 * @ param makeNullable If { @ code true } , the return type will be nullable
 * @ return a { @ code Schema } for the given { @ code Schema . Type }
 * @ see Schema # create ( org . apache . avro . Schema . Type ) */
private static Schema schema ( Schema . Type type , boolean makeNullable ) { } }

implementation: Schema schema = Schema . create ( type == null ? Schema . Type . STRING : type ) ; if ( makeNullable || type == null ) { schema = Schema . createUnion ( Lists . newArrayList ( Schema . create ( Schema . Type . NULL ) , schema ) ) ; } return schema ;


signature:
public class ThriftReadSupport { /** * A { @ link ThriftRecordConverter } builds an object by working with { @ link TProtocol } . The default
 * implementation creates standard Apache Thrift { @ link TBase } objects ; to support alternatives , such
 * as < a href = " http : / / github . com / twitter / scrooge " > Twiter ' s Scrooge < / a > , a custom converter can be specified
 * ( for example , ScroogeRecordConverter from parquet - scrooge ) .
 * @ param conf a mapred jobconf
 * @ param klass a thrift class
 * @ deprecated use { @ link # setRecordConverterClass ( Configuration , Class ) } below */
@ Deprecated public static void setRecordConverterClass ( JobConf conf , Class < ? > klass ) { } }

implementation: setRecordConverterClass ( ( Configuration ) conf , klass ) ;


signature:
public class ThriftReadSupport { /** * A { @ link ThriftRecordConverter } builds an object by working with { @ link TProtocol } . The default
 * implementation creates standard Apache Thrift { @ link TBase } objects ; to support alternatives , such
 * as < a href = " http : / / github . com / twitter / scrooge " > Twiter ' s Scrooge < / a > , a custom converter can be specified
 * ( for example , ScroogeRecordConverter from parquet - scrooge ) .
 * @ param conf a configuration
 * @ param klass a thrift class */
public static void setRecordConverterClass ( Configuration conf , Class < ? > klass ) { } }

implementation: conf . set ( RECORD_CONVERTER_CLASS_KEY , klass . getName ( ) ) ;


signature:
public class Summary { /** * the input tuple contains a bag of string representations of TupleSummaryData
 * @ param t
 * @ return
 * @ throws ExecException
 * @ throws JsonParseException
 * @ throws JsonMappingException
 * @ throws IOException */
private static TupleSummaryData merge ( Tuple t ) throws IOException { } }

implementation: TupleSummaryData summaryData = new TupleSummaryData ( ) ; DataBag bag = ( DataBag ) t . get ( 0 ) ; for ( Tuple tuple : bag ) { summaryData . merge ( getData ( tuple ) ) ; } return summaryData ;


signature:
public class Summary { /** * The input tuple contains a bag of Tuples to sum up
 * @ param t
 * @ return
 * @ throws ExecException */
private static TupleSummaryData sumUp ( Schema schema , Tuple t ) throws ExecException { } }

implementation: TupleSummaryData summaryData = new TupleSummaryData ( ) ; DataBag bag = ( DataBag ) t . get ( 0 ) ; for ( Tuple tuple : bag ) { summaryData . addTuple ( schema , tuple ) ; } return summaryData ;


signature:
public class ParquetRecordReader { /** * { @ inheritDoc } */
@ Override public void initialize ( InputSplit inputSplit , TaskAttemptContext context ) throws IOException , InterruptedException { } }

implementation: if ( ContextUtil . hasCounterMethod ( context ) ) { BenchmarkCounter . initCounterFromContext ( context ) ; } else { LOG . error ( String . format ( "Can not initialize counter because the class '%s' does not have a '.getCounterMethod'" , context . getClass ( ) . getCanonicalName ( ) ) ) ; } initializeInternalReader ( toParquetSplit ( inputSplit ) , ContextUtil . getConfiguration ( context ) ) ;


signature: public class WriteBenchmarks { @ Benchmark public void write1MRowsDefaultBlockAndPageSizeSNAPPY ( ) throws IOException { } }

implementation: dataGenerator . generateData ( file_1M_SNAPPY , configuration , PARQUET_2_0 , BLOCK_SIZE_DEFAULT , PAGE_SIZE_DEFAULT , FIXED_LEN_BYTEARRAY_SIZE , SNAPPY , ONE_MILLION ) ;


signature:
public class EventBasedThriftReader { /** * reads a Struct from the underlying protocol and passes the field events to the FieldConsumer
 * @ param c the field consumer
 * @ throws TException if any thrift related error occurs during the reading */
public void readStruct ( FieldConsumer c ) throws TException { } }

implementation: protocol . readStructBegin ( ) ; readStructContent ( c ) ; protocol . readStructEnd ( ) ;


signature:
public class EventBasedThriftReader { /** * reads the content of a struct ( fields ) from the underlying protocol and passes the events to c
 * @ param c the field consumer
 * @ throws TException if any thrift related error occurs during the reading */
public void readStructContent ( FieldConsumer c ) throws TException { } }

implementation: TField field ; while ( true ) { field = protocol . readFieldBegin ( ) ; if ( field . type == TType . STOP ) { break ; } c . consumeField ( protocol , this , field . id , field . type ) ; }


signature:
public class EventBasedThriftReader { /** * reads the set content ( elements ) from the underlying protocol and passes the events to the set event consumer
 * @ param eventConsumer the consumer
 * @ param tSet the set descriptor
 * @ throws TException if any thrift related error occurs during the reading */
public void readSetContent ( SetConsumer eventConsumer , TSet tSet ) throws TException { } }

implementation: for ( int i = 0 ; i < tSet . size ; i ++ ) { eventConsumer . consumeElement ( protocol , this , tSet . elemType ) ; }


signature:
public class EventBasedThriftReader { /** * reads the map content ( key values ) from the underlying protocol and passes the events to the map event consumer
 * @ param eventConsumer the consumer
 * @ param tMap the map descriptor
 * @ throws TException if any thrift related error occurs during the reading */
public void readMapContent ( MapConsumer eventConsumer , TMap tMap ) throws TException { } }

implementation: for ( int i = 0 ; i < tMap . size ; i ++ ) { eventConsumer . consumeEntry ( protocol , this , tMap . keyType , tMap . valueType ) ; }


signature:
public class EventBasedThriftReader { /** * reads a key - value pair
 * @ param keyType the type of the key
 * @ param keyConsumer the consumer for the key
 * @ param valueType the type of the value
 * @ param valueConsumer the consumer for the value
 * @ throws TException if any thrift related error occurs during the reading */
public void readMapEntry ( byte keyType , TypedConsumer keyConsumer , byte valueType , TypedConsumer valueConsumer ) throws TException { } }

implementation: keyConsumer . read ( protocol , this , keyType ) ; valueConsumer . read ( protocol , this , valueType ) ;


signature:
public class EventBasedThriftReader { /** * reads the list content ( elements ) from the underlying protocol and passes the events to the list event consumer
 * @ param eventConsumer the consumer
 * @ param tList the list descriptor
 * @ throws TException if any thrift related error occurs during the reading */
public void readListContent ( ListConsumer eventConsumer , TList tList ) throws TException { } }

implementation: for ( int i = 0 ; i < tList . size ; i ++ ) { eventConsumer . consumeElement ( protocol , this , tList . elemType ) ; }


signature:
public class ByteBasedBitPackingEncoder { /** * writes an int using the requested number of bits .
 * accepts only values less than 2 ^ bitWidth
 * @ param value the value to write
 * @ throws IOException if there is an exception while writing */
public void writeInt ( int value ) throws IOException { } }

implementation: input [ inputSize ] = value ; ++ inputSize ; if ( inputSize == VALUES_WRITTEN_AT_A_TIME ) { pack ( ) ; if ( packedPosition == slabSize ) { slabs . add ( BytesInput . from ( packed ) ) ; totalFullSlabSize += slabSize ; if ( slabSize < bitWidth * MAX_SLAB_SIZE_MULT ) { slabSize *= 2 ; } initPackedSlab ( ) ; } }


signature:
public class BytesUtils { /** * reads an int in little endian at the given position
 * @ param in a byte buffer
 * @ param offset an offset into the byte buffer
 * @ return the integer at position offset read using little endian byte order
 * @ throws IOException if there is an exception reading from the byte buffer */
public static int readIntLittleEndian ( ByteBuffer in , int offset ) throws IOException { } }

implementation: int ch4 = in . get ( offset ) & 0xff ; int ch3 = in . get ( offset + 1 ) & 0xff ; int ch2 = in . get ( offset + 2 ) & 0xff ; int ch1 = in . get ( offset + 3 ) & 0xff ; return ( ( ch1 << 24 ) + ( ch2 << 16 ) + ( ch3 << 8 ) + ( ch4 << 0 ) ) ;


signature:
public class BytesUtils { /** * Write a little endian int to out , using the the number of bytes required by
 * bit width
 * @ param out an output stream
 * @ param v an int value
 * @ param bitWidth bit width for padding
 * @ throws IOException if there is an exception while writing */
public static void writeIntLittleEndianPaddedOnBitWidth ( OutputStream out , int v , int bitWidth ) throws IOException { } }

implementation: int bytesWidth = paddedByteCountFromBits ( bitWidth ) ; switch ( bytesWidth ) { case 0 : break ; case 1 : writeIntLittleEndianOnOneByte ( out , v ) ; break ; case 2 : writeIntLittleEndianOnTwoBytes ( out , v ) ; break ; case 3 : writeIntLittleEndianOnThreeBytes ( out , v ) ; break ; case 4 : writeIntLittleEndian ( out , v ) ; break ; default : throw new IOException ( String . format ( "Encountered value (%d) that requires more than 4 bytes" , v ) ) ; }


signature:
public class BytesUtils { /** * uses a trick mentioned in https : / / developers . google . com / protocol - buffers / docs / encoding to read zigZag encoded data
 * @ param in an input stream
 * @ return the value of a zig - zag varint read from the current position in the stream
 * @ throws IOException if there is an exception while reading */
public static int readZigZagVarInt ( InputStream in ) throws IOException { } }

implementation: int raw = readUnsignedVarInt ( in ) ; int temp = ( ( ( raw << 31 ) >> 31 ) ^ raw ) >> 1 ; return temp ^ ( raw & ( 1 << 31 ) ) ;


signature:
public class BytesUtils { /** * uses a trick mentioned in https : / / developers . google . com / protocol - buffers / docs / encoding to read zigZag encoded data
 * TODO : the implementation is compatible with readZigZagVarInt . Is there a need for different functions ?
 * @ param in an input stream
 * @ return the value of a zig - zag var - long read from the current position in the stream
 * @ throws IOException if there is an exception while reading */
public static long readZigZagVarLong ( InputStream in ) throws IOException { } }

implementation: long raw = readUnsignedVarLong ( in ) ; long temp = ( ( ( raw << 63 ) >> 63 ) ^ raw ) >> 1 ; return temp ^ ( raw & ( 1L << 63 ) ) ;


signature:
public class SnappyDecompressor { /** * Fills specified buffer with uncompressed data . Returns actual number
 * of bytes of uncompressed data . A return value of 0 indicates that
 * { @ link # needsInput ( ) } should be called in order to determine if more
 * input data is required .
 * @ param buffer Buffer for the compressed data
 * @ param off Start offset of the data
 * @ param len Size of the buffer
 * @ return The actual number of bytes of uncompressed data .
 * @ throws IOException if reading or decompression fails */
@ Override public synchronized int decompress ( byte [ ] buffer , int off , int len ) throws IOException { } }

implementation:
SnappyUtil . validateBuffer ( buffer , off , len ) ; if ( inputBuffer . position ( ) == 0 && ! outputBuffer . hasRemaining ( ) ) { return 0 ; } if ( ! outputBuffer . hasRemaining ( ) ) { inputBuffer . rewind ( ) ; Preconditions . checkArgument ( inputBuffer . position ( ) == 0 , "Invalid position of 0." ) ; Preconditions . checkArgument ( outputBuffer . position ( ) == 0 , "Invalid position of 0." ) ; // There is compressed input , decompress it now .
int decompressedSize = Snappy . uncompressedLength ( inputBuffer ) ; if ( decompressedSize > outputBuffer . capacity ( ) ) { ByteBuffer oldBuffer = outputBuffer ; outputBuffer = ByteBuffer . allocateDirect ( decompressedSize ) ; CleanUtil . clean ( oldBuffer ) ; } // Reset the previous outputBuffer ( i . e . set position to 0)
outputBuffer . clear ( ) ; int size = Snappy . uncompress ( inputBuffer , outputBuffer ) ; outputBuffer . limit ( size ) ; // We ' ve decompressed the entire input , reset the input now
inputBuffer . clear ( ) ; inputBuffer . limit ( 0 ) ; finished = true ; } // Return compressed output up to ' len '
int numBytes = Math . min ( len , outputBuffer . remaining ( ) ) ; outputBuffer . get ( buffer , off , numBytes ) ; return numBytes ;


signature:
public class SnappyDecompressor { /** * Sets input data for decompression .
 * This should be called if and only if { @ link # needsInput ( ) } returns
 * < code > true < / code > indicating that more input data is required .
 * ( Both native and non - native versions of various Decompressors require
 * that the data passed in via < code > b [ ] < / code > remain unmodified until
 * the caller is explicitly notified - - via { @ link # needsInput ( ) } - - that the
 * buffer may be safely modified . With this requirement , an extra
 * buffer - copy can be avoided . )
 * @ param buffer Input data
 * @ param off Start offset
 * @ param len Length */
@ Override public synchronized void setInput ( byte [ ] buffer , int off , int len ) { } }

implementation: SnappyUtil . validateBuffer ( buffer , off , len ) ; if ( inputBuffer . capacity ( ) - inputBuffer . position ( ) < len ) { ByteBuffer newBuffer = ByteBuffer . allocateDirect ( inputBuffer . position ( ) + len ) ; inputBuffer . rewind ( ) ; newBuffer . put ( inputBuffer ) ; ByteBuffer oldBuffer = inputBuffer ; inputBuffer = newBuffer ; CleanUtil . clean ( oldBuffer ) ; } else { inputBuffer . limit ( inputBuffer . position ( ) + len ) ; } inputBuffer . put ( buffer , off , len ) ;


signature:
public class AvroParquetOutputFormat { /** * Set the Avro schema to use for writing . The schema is translated into a Parquet
 * schema so that the records can be written in Parquet format . It is also
 * stored in the Parquet metadata so that records can be reconstructed as Avro
 * objects at read time without specifying a read schema .
 * @ param job a job
 * @ param schema a schema for the data that will be written
 * @ see org . apache . parquet . avro . AvroParquetInputFormat # setAvroReadSchema ( org . apache . hadoop . mapreduce . Job , org . apache . avro . Schema ) */
public static void setSchema ( Job job , Schema schema ) { } }

implementation: AvroWriteSupport . setSchema ( ContextUtil . getConfiguration ( job ) , schema ) ;


signature:
public class AbstractHiveBinding { /** * { @ inheritDoc } */
@ Override public List < String > getColumns ( final String columns ) { } }

implementation: final List < String > result = ( List < String > ) StringUtils . getStringCollection ( columns ) ; result . removeAll ( virtualColumns ) ; return result ;


signature:
public class HadoopStreams { /** * Wraps a { @ link FSDataInputStream } in a { @ link SeekableInputStream }
 * implementation for Parquet readers .
 * @ param stream a Hadoop FSDataInputStream
 * @ return a SeekableInputStream */
public static SeekableInputStream wrap ( FSDataInputStream stream ) { } }

implementation: Preconditions . checkNotNull ( stream , "Cannot wrap a null input stream" ) ; if ( byteBufferReadableClass != null && h2SeekableConstructor != null && byteBufferReadableClass . isInstance ( stream . getWrappedStream ( ) ) ) { try { return h2SeekableConstructor . newInstance ( stream ) ; } catch ( InstantiationException e ) { LOG . warn ( "Could not instantiate H2SeekableInputStream, falling back to byte array reads" , e ) ; return new H1SeekableInputStream ( stream ) ; } catch ( IllegalAccessException e ) { LOG . warn ( "Could not instantiate H2SeekableInputStream, falling back to byte array reads" , e ) ; return new H1SeekableInputStream ( stream ) ; } catch ( InvocationTargetException e ) { throw new ParquetDecodingException ( "Could not instantiate H2SeekableInputStream" , e . getTargetException ( ) ) ; } } else { return new H1SeekableInputStream ( stream ) ; }


signature:
public class MultiKafkaRawSourcePreviewer { /** * theoretically they could just copy / paste the output into the Topics bulk edit field in the Kafka config */
private < T > String convertToRawListString ( Collection < T > collection ) { } }

implementation: return "[ " + collection . stream ( ) . map ( x -> '"' + x . toString ( ) + '"' ) . collect ( Collectors . joining ( ", " ) ) + " ]" ;


signature:
public class MapR52StreamsValidationUtil09 { /** * Should be called only by MapR Streams Producer . It creates a topic using KafkaProducer .
 * @ param topic
 * @ param kafkaClientConfigs
 * @ param metadataBrokerList
 * @ throws StageException */
@ Override public void createTopicIfNotExists ( String topic , Map < String , Object > kafkaClientConfigs , String metadataBrokerList ) throws StageException { } }

implementation:
// check stream path and topic
if ( topic . startsWith ( "/" ) && topic . contains ( ":" ) ) { String [ ] path = topic . split ( ":" ) ; if ( path . length != 2 ) { // Stream topic has invalid format . Record will be sent to error
throw new StageException ( MapRStreamsErrors . MAPRSTREAMS_21 , topic ) ; } String streamPath = path [ 0 ] ; if ( ! streamCache . contains ( streamPath ) ) { // This pipeline sees this stream path for the 1st time
Configuration conf = new Configuration ( ) ; kafkaClientConfigs . forEach ( ( k , v ) -> { conf . set ( k , v . toString ( ) ) ; } ) ; Admin streamAdmin = null ; try { streamAdmin = Streams . newAdmin ( conf ) ; // Check if the stream path exists already
streamAdmin . countTopics ( streamPath ) ; streamCache . add ( streamPath ) ; } catch ( TableNotFoundException e ) { LOG . debug ( "Stream not found. Creating a new stream: " + streamPath ) ; try { streamAdmin . createStream ( streamPath , Streams . newStreamDescriptor ( ) ) ; streamCache . add ( streamPath ) ; } catch ( IOException ioex ) { throw new StageException ( MapRStreamsErrors . MAPRSTREAMS_22 , streamPath , e . getMessage ( ) , e ) ; } } catch ( IOException | IllegalArgumentException e ) { throw new StageException ( MapRStreamsErrors . MAPRSTREAMS_23 , e . getMessage ( ) , e ) ; } finally { if ( streamAdmin != null ) { streamAdmin . close ( ) ; } } } } // Stream topic can be created through KafkaProducer if Stream Path exists already
KafkaProducer < String , String > kafkaProducer = createProducerTopicMetadataClient ( kafkaClientConfigs ) ; kafkaProducer . partitionsFor ( topic ) ;


signature:
public class StringRedactor { /** * Create a StringRedactor based on the JSON found in a file . The file
 * format looks like this :
 * " version " : 1,
 * " rules " : [
 * { " description " : " This is the first rule " ,
 * " trigger " : " triggerstring 1 " ,
 * " search " : " regex 1 " ,
 * " replace " : " replace 1"
 * { " description " : " This is the second rule " ,
 * " trigger " : " triggerstring 2 " ,
 * " search " : " regex 2 " ,
 * " replace " : " replace 2"
 * @ param fileName The name of the file to read
 * @ return A freshly allocated StringRedactor
 * @ throws JsonParseException , JsonMappingException , IOException */
public static StringRedactor createFromJsonFile ( String fileName ) throws IOException { } }

implementation:
StringRedactor sr = new StringRedactor ( ) ; if ( fileName == null ) { sr . policy = RedactionPolicy . emptyRedactionPolicy ( ) ; return sr ; } File file = new File ( fileName ) ; // An empty file is explicitly allowed as " no rules "
if ( file . exists ( ) && file . length ( ) == 0 ) { sr . policy = RedactionPolicy . emptyRedactionPolicy ( ) ; return sr ; } ObjectMapper mapper = new ObjectMapper ( ) ; RedactionPolicy policy = mapper . readValue ( file , RedactionPolicy . class ) ; policy . postProcess ( ) ; sr . policy = policy ; return sr ;


signature:
public class StringRedactor { /** * Create a StringRedactor based on the JSON found in the given String .
 * The format is identical to that described in createFromJsonFile ( ) .
 * @ param json String containing json formatted rules .
 * @ return A freshly allocated StringRedactor
 * @ throws JsonParseException , JsonMappingException , IOException */
public static StringRedactor createFromJsonString ( String json ) throws IOException { } }

implementation: StringRedactor sr = new StringRedactor ( ) ; if ( ( json == null ) || json . isEmpty ( ) ) { sr . policy = RedactionPolicy . emptyRedactionPolicy ( ) ; return sr ; } ObjectMapper mapper = new ObjectMapper ( ) ; RedactionPolicy policy = mapper . readValue ( json , RedactionPolicy . class ) ; policy . postProcess ( ) ; sr . policy = policy ; return sr ;


signature:
public class PipelineBeanCreator { /** * Create PipelineBean which means instantiating all stages for the pipeline .
 * For multi - threaded pipelines this method will * NOT * create all required instances since the number of required
 * instances is not known at the time of creation - for that the origin has to be initialized which is not at this
 * point . Hence this method will create only one instance of the whole pipeline and it ' s up to the caller to call
 * createPipelineStageBeans to instantiate remaining source - less pipelines later in the execution . */
public PipelineBean create ( boolean forExecution , StageLibraryTask library , PipelineConfiguration pipelineConf , InterceptorCreatorContextBuilder interceptorContextBuilder , List < Issue > errors , Map < String , Object > runtimeParameters ) { } }

implementation:
int priorErrors = errors . size ( ) ; PipelineConfigBean pipelineConfigBean = create ( pipelineConf , errors , runtimeParameters ) ; StageBean errorStageBean = null ; StageBean statsStageBean = null ; StageBean origin = null ; PipelineStageBeans stages = null ; PipelineStageBeans startEventBeans = null ; PipelineStageBeans stopEventBeans = null ; if ( pipelineConfigBean != null && pipelineConfigBean . constants != null ) { Map < String , Object > resolvedConstants = pipelineConfigBean . constants ; if ( interceptorContextBuilder != null ) { interceptorContextBuilder . withExecutionMode ( pipelineConfigBean . executionMode ) . withDeliveryGuarantee ( pipelineConfigBean . deliveryGuarantee ) ; } // Instantiate usual stages
if ( ! pipelineConf . getStages ( ) . isEmpty ( ) ) { origin = createStageBean ( forExecution , library , pipelineConf . getStages ( ) . get ( 0 ) , true , false , false , resolvedConstants , interceptorContextBuilder , errors ) ; stages = createPipelineStageBeans ( forExecution , library , pipelineConf . getStages ( ) . subList ( 1 , pipelineConf . getStages ( ) . size ( ) ) , interceptorContextBuilder , resolvedConstants , errors ) ; } // It is not mandatory to have a stats aggregating target configured
StageConfiguration statsStageConf = pipelineConf . getStatsAggregatorStage ( ) ; if ( statsStageConf != null ) { statsStageBean = createStageBean ( forExecution , library , statsStageConf , true , false , false , resolvedConstants , interceptorContextBuilder , errors ) ; } // Error stage is mandatory
StageConfiguration errorStageConf = pipelineConf . getErrorStage ( ) ; if ( errorStageConf != null ) { errorStageBean = createStageBean ( forExecution , library , errorStageConf , true , true , false , resolvedConstants , interceptorContextBuilder , errors ) ; } else if ( ! ( pipelineConfigBean . executionMode . equals ( ExecutionMode . BATCH ) || pipelineConfigBean . executionMode . equals ( ExecutionMode . STREAMING ) ) ) { errors . add ( IssueCreator . getPipeline ( ) . create ( PipelineGroups . BAD_RECORDS . name ( ) , "badRecordsHandling" , CreationError . CREATION_009 ) ) ; } // Pipeline Lifecycle event handlers
StageBean startBean = null ; if ( CollectionUtils . isNotEmpty ( pipelineConf . getStartEventStages ( ) ) ) { startBean = createStageBean ( forExecution , library , pipelineConf . getStartEventStages ( ) . get ( 0 ) , true , false , true , resolvedConstants , interceptorContextBuilder , errors ) ; } startEventBeans = new PipelineStageBeans ( startBean == null ? Collections . emptyList ( ) : ImmutableList . of ( startBean ) ) ; StageBean stopBean = null ; if ( CollectionUtils . isNotEmpty ( pipelineConf . getStopEventStages ( ) ) ) { stopBean = createStageBean ( forExecution , library , pipelineConf . getStopEventStages ( ) . get ( 0 ) , true , false , true , resolvedConstants , interceptorContextBuilder , errors ) ; } stopEventBeans = new PipelineStageBeans ( stopBean == null ? Collections . emptyList ( ) : ImmutableList . of ( stopBean ) ) ; // Validate Webhook Configs
if ( pipelineConfigBean . webhookConfigs != null && ! pipelineConfigBean . webhookConfigs . isEmpty ( ) ) { int index = 0 ; for ( PipelineWebhookConfig webhookConfig : pipelineConfigBean . webhookConfigs ) { if ( StringUtils . isEmpty ( webhookConfig . webhookUrl ) ) { Issue issue = IssueCreator . getPipeline ( ) . create ( PipelineGroups . NOTIFICATIONS . name ( ) , "webhookUrl" , CreationError . CREATION_080 ) ; issue . setAdditionalInfo ( "index" , index ) ; errors . add ( issue ) ; break ; } index ++ ; } } } // Something went wrong
if ( errors . size ( ) != priorErrors ) { return null ; } return new PipelineBean ( pipelineConfigBean , origin , stages , errorStageBean , statsStageBean , startEventBeans , stopEventBeans ) ;


signature:
public class PipelineBeanCreator { /** * Creates additional PipelineStageBeans for additional runners . Stages will share stage definition and thus
 * class loader with the first given runner . That includes stages with private class loader as well .
 * @ param pipelineStageBeans First runner that should be duplicated .
 * @ param interceptorCreatorContextBuilder Builder for interceptor context
 * @ param constants Pipeline constants
 * @ param errors Any generated errors will be stored in this list
 * @ return PipelineStageBeans with new instances of the given stages */
public PipelineStageBeans duplicatePipelineStageBeans ( StageLibraryTask stageLib , PipelineStageBeans pipelineStageBeans , InterceptorCreatorContextBuilder interceptorCreatorContextBuilder , Map < String , Object > constants , List < Issue > errors ) { } }

implementation:
List < StageBean > stageBeans = new ArrayList < > ( pipelineStageBeans . size ( ) ) ; for ( StageBean original : pipelineStageBeans . getStages ( ) ) { // Create StageDefinition map for this stage
Map < Class , ServiceDefinition > services = original . getServices ( ) . stream ( ) . collect ( Collectors . toMap ( c -> c . getDefinition ( ) . getProvides ( ) , ServiceBean :: getDefinition ) ) ; StageBean stageBean = createStage ( stageLib , original . getDefinition ( ) , ClassLoaderReleaser . NOOP_RELEASER , original . getConfiguration ( ) , services :: get , interceptorCreatorContextBuilder , constants , errors ) ; if ( stageBean != null ) { stageBeans . add ( stageBean ) ; } } return new PipelineStageBeans ( stageBeans ) ;


signature:
public class PipelineBeanCreator { /** * Create new instance of StageBean .
 * This method can be used outside of pipeline context ( for example for detached stage ) .
 * @ param forExecution If the instance is going to be used for execution ( and for example private classloaders must be respected )
 * @ param library Stage Library from which a definition will be loaded
 * @ param stageConf Stage configuration
 * @ param errorStage True if the stage needs to be declared as error stage
 * @ param pipelineLifecycleStage True if the stage needs to be declared as pipeline lifecycle stage
 * @ param constants Pipeline constants ( runtime parameters )
 * @ param errors List where all errors will be persisted
 * @ return New StageBean instance or null on any error */
public StageBean createStageBean ( boolean forExecution , StageLibraryTask library , StageConfiguration stageConf , boolean validateAnnotations , boolean errorStage , boolean pipelineLifecycleStage , Map < String , Object > constants , InterceptorCreatorContextBuilder interceptorContextBuilder , List < Issue > errors ) { } }

implementation:
IssueCreator issueCreator = IssueCreator . getStage ( stageConf . getInstanceName ( ) ) ; StageBean bean = null ; StageDefinition stageDef = library . getStage ( stageConf . getLibrary ( ) , stageConf . getStageName ( ) , forExecution ) ; if ( stageDef != null ) { // Pipeline lifecycle events validation must match , whether it ' s also marked as error stage does not matter
if ( validateAnnotations ) { if ( pipelineLifecycleStage ) { if ( ! stageDef . isPipelineLifecycleStage ( ) ) { errors . add ( issueCreator . create ( CreationError . CREATION_018 , stageDef . getLibraryLabel ( ) , stageDef . getLabel ( ) , stageConf . getStageVersion ( ) ) ) ; } // For non pipeline lifecycle stages , the error stage annotation must match
} else if ( stageDef . isErrorStage ( ) != errorStage ) { if ( stageDef . isErrorStage ( ) ) { errors . add ( issueCreator . create ( CreationError . CREATION_007 , stageDef . getLibraryLabel ( ) , stageDef . getLabel ( ) , stageConf . getStageVersion ( ) ) ) ; } else { errors . add ( issueCreator . create ( CreationError . CREATION_008 , stageDef . getLibraryLabel ( ) , stageDef . getLabel ( ) , stageConf . getStageVersion ( ) ) ) ; } } } bean = createStage ( library , stageDef , library , stageConf , serviceClass -> library . getServiceDefinition ( serviceClass , true ) , interceptorContextBuilder , constants , errors ) ; } else { errors . add ( issueCreator . create ( CreationError . CREATION_006 , stageConf . getLibrary ( ) , stageConf . getStageName ( ) , stageConf . getStageVersion ( ) ) ) ; } return bean ;


signature:
public class PipelineBeanCreator { /** * Create interceptors for given stage . */
public List < InterceptorBean > createInterceptors ( StageLibraryTask stageLib , StageConfiguration stageConfiguration , StageDefinition stageDefinition , InterceptorCreatorContextBuilder contextBuilder , InterceptorCreator . InterceptorType interceptorType , List < Issue > issues ) { } }

implementation: List < InterceptorBean > beans = new ArrayList < > ( ) ; if ( contextBuilder == null ) { return beans ; } for ( InterceptorDefinition definition : stageLib . getInterceptorDefinitions ( ) ) { InterceptorBean bean = createInterceptor ( stageLib , definition , stageConfiguration , stageDefinition , contextBuilder , interceptorType , issues ) ; if ( bean != null ) { beans . add ( bean ) ; } } return beans ;


signature:
public class PipelineBeanCreator { /** * Create a default interceptor for given InterceptorDefinition . This method might
 * return null as the underlying interface for default creation allows it as well -
 * in such case no interceptor is needed . */
public InterceptorBean createInterceptor ( StageLibraryTask stageLib , InterceptorDefinition definition , StageConfiguration stageConfiguration , StageDefinition stageDefinition , InterceptorCreatorContextBuilder contextBuilder , InterceptorCreator . InterceptorType interceptorType , List < Issue > issues ) { } }

implementation: ClassLoader classLoader = Thread . currentThread ( ) . getContextClassLoader ( ) ; InterceptorCreator . Context context = contextBuilder . buildFor ( definition . getLibraryDefinition ( ) . getName ( ) , definition . getKlass ( ) . getName ( ) , stageConfiguration , stageDefinition , interceptorType ) ; try { Thread . currentThread ( ) . setContextClassLoader ( definition . getStageClassLoader ( ) ) ; InterceptorCreator creator = definition . getDefaultCreator ( ) . newInstance ( ) ; Interceptor interceptor = creator . create ( context ) ; if ( interceptor == null ) { return null ; } return new InterceptorBean ( definition , interceptor , stageLib ) ; } catch ( IllegalAccessException | InstantiationException e ) { LOG . debug ( "Can't instantiate interceptor: {}" , e . toString ( ) , e ) ; IssueCreator issueCreator = IssueCreator . getStage ( stageDefinition . getName ( ) ) ; issues . add ( issueCreator . create ( CreationError . CREATION_000 , "interceptor" , definition . getKlass ( ) . getName ( ) , e . toString ( ) ) ) ; } finally { Thread . currentThread ( ) . setContextClassLoader ( classLoader ) ; } return null ;


signature:
public class AvroTypeUtil { /** * Parse JSON representation of Avro schema to Avro ' s Schema JAVA object */
public static Schema parseSchema ( String schema ) { } }

implementation:
Schema . Parser parser = new Schema . Parser ( ) ; parser . setValidate ( true ) ; // We sadly can ' t use this method directly because it was added after 1.7.3 and we have to stay
// compatible with 1.7.3 as this version ships with mapr ( and thus we end up using it ) . This code is
// however compiled against 1.7.7 and hence we don ' t have to do reflection here .
try { parser . setValidateDefaults ( true ) ; } catch ( NoSuchMethodError e ) { LOG . debug ( "Running old Avro version that doesn't have 'setValidateDefaults' method" , e ) ; } return parser . parse ( schema ) ;


signature:
public class AvroTypeUtil { /** * Return Date in milliseconds
 * @ param days Number of days since unix epoch
 * @ return Milliseconds representation for date
 * This function has been copied from Apache Hive project . */
public static long daysToMillis ( int days ) { } }

implementation: long millisUtc = ( long ) days * MILLIS_PER_DAY ; long tmp = millisUtc - ( long ) ( localTimeZone . getOffset ( millisUtc ) ) ; return millisUtc - ( long ) ( localTimeZone . getOffset ( tmp ) ) ;


signature:
public class AvroTypeUtil { /** * Return number of days since the unix epoch .
 * This function has been copied from Apache Hive project . */
private static int millisToDays ( long millisLocal ) { } }

implementation:
// We assume millisLocal is midnight of some date . What we are basically trying to do
// here is go from local - midnight to UTC - midnight ( or whatever time that happens to be ) .
long millisUtc = millisLocal + localTimeZone . getOffset ( millisLocal ) ; int days ; if ( millisUtc >= 0L ) { days = ( int ) ( millisUtc / MILLIS_PER_DAY ) ; } else { days = ( int ) ( ( millisUtc - 86399999 /* ( MILLIS _ PER _ DAY - 1) */
) / MILLIS_PER_DAY ) ; } return days ;


signature:
public class AvroTypeUtil { /** * Retrieves avro schema from given header . Throws an exception if the header is missing or is empty . */
public static String getAvroSchemaFromHeader ( Record record , String headerName ) throws DataGeneratorException { } }

implementation: String jsonSchema = record . getHeader ( ) . getAttribute ( headerName ) ; if ( jsonSchema == null || jsonSchema . isEmpty ( ) ) { throw new DataGeneratorException ( Errors . AVRO_GENERATOR_03 , record . getHeader ( ) . getSourceId ( ) ) ; } return jsonSchema ;