signature:
public class ParquetFileWriter { /** * Writes a column chunk at once
 * @ param descriptor the descriptor of the column
 * @ param valueCount the value count in this column
 * @ param compressionCodecName the name of the compression codec used for compressing the pages
 * @ param dictionaryPage the dictionary page for this column chunk ( might be null )
 * @ param bytes the encoded pages including page headers to be written as is
 * @ param uncompressedTotalPageSize total uncompressed size ( without page headers )
 * @ param compressedTotalPageSize total compressed size ( without page headers )
 * @ param totalStats accumulated statistics for the column chunk
 * @ param columnIndexBuilder the builder object for the column index
 * @ param offsetIndexBuilder the builder object for the offset index
 * @ param rlEncodings the RL encodings used in this column chunk
 * @ param dlEncodings the DL encodings used in this column chunk
 * @ param dataEncodings the data encodings used in this column chunk
 * @ throws IOException if there is an error while writing */
void writeColumnChunk ( ColumnDescriptor descriptor , long valueCount , CompressionCodecName compressionCodecName , DictionaryPage dictionaryPage , BytesInput bytes , long uncompressedTotalPageSize , long compressedTotalPageSize , Statistics < ? > totalStats , ColumnIndexBuilder columnIndexBuilder , OffsetIndexBuilder offsetIndexBuilder , Set < Encoding > rlEncodings , Set < Encoding > dlEncodings , List < Encoding > dataEncodings ) throws IOException { } }

implementation: startColumn ( descriptor , valueCount , compressionCodecName ) ; state = state . write ( ) ; if ( dictionaryPage != null ) { writeDictionaryPage ( dictionaryPage ) ; } LOG . debug ( "{}: write data pages" , out . getPos ( ) ) ; long headersSize = bytes . size ( ) - compressedTotalPageSize ; this . uncompressedLength += uncompressedTotalPageSize + headersSize ; this . compressedLength += compressedTotalPageSize + headersSize ; LOG . debug ( "{}: write data pages content" , out . getPos ( ) ) ; firstPageOffset = out . getPos ( ) ; bytes . writeAllTo ( out ) ; encodingStatsBuilder . addDataEncodings ( dataEncodings ) ; if ( rlEncodings . isEmpty ( ) ) { encodingStatsBuilder . withV2Pages ( ) ; } currentEncodings . addAll ( rlEncodings ) ; currentEncodings . addAll ( dlEncodings ) ; currentEncodings . addAll ( dataEncodings ) ; currentStatistics = totalStats ; this . columnIndexBuilder = columnIndexBuilder ; this . offsetIndexBuilder = offsetIndexBuilder ; endColumn ( ) ;


signature:
public class ParquetFileWriter { /** * end a column ( once all rep , def and data have been written )
 * @ throws IOException if there is an error while writing */
public void endColumn ( ) throws IOException { } }

implementation: state = state . endColumn ( ) ; LOG . debug ( "{}: end column" , out . getPos ( ) ) ; if ( columnIndexBuilder . getMinMaxSize ( ) > columnIndexBuilder . getPageCount ( ) * MAX_STATS_SIZE ) { currentColumnIndexes . add ( null ) ; } else { currentColumnIndexes . add ( columnIndexBuilder . build ( ) ) ; } currentOffsetIndexes . add ( offsetIndexBuilder . build ( firstPageOffset ) ) ; currentBlock . addColumn ( ColumnChunkMetaData . get ( currentChunkPath , currentChunkType , currentChunkCodec , encodingStatsBuilder . build ( ) , currentEncodings , currentStatistics , currentChunkFirstDataPage , currentChunkDictionaryPageOffset , currentChunkValueCount , compressedLength , uncompressedLength ) ) ; this . currentBlock . setTotalByteSize ( currentBlock . getTotalByteSize ( ) + uncompressedLength ) ; this . uncompressedLength = 0 ; this . compressedLength = 0 ; columnIndexBuilder = null ; offsetIndexBuilder = null ;


signature:
public class ParquetFileWriter { /** * ends a block once all column chunks have been written
 * @ throws IOException if there is an error while writing */
public void endBlock ( ) throws IOException { } }

implementation: state = state . endBlock ( ) ; LOG . debug ( "{}: end block" , out . getPos ( ) ) ; currentBlock . setRowCount ( currentRecordCount ) ; blocks . add ( currentBlock ) ; columnIndexes . add ( currentColumnIndexes ) ; offsetIndexes . add ( currentOffsetIndexes ) ; currentColumnIndexes = null ; currentOffsetIndexes = null ; currentBlock = null ;


signature:
public class ParquetFileWriter { /** * Copy from a FS input stream to an output stream . Thread - safe
 * @ param from a { @ link SeekableInputStream }
 * @ param to any { @ link PositionOutputStream }
 * @ param start where in the from stream to start copying
 * @ param length the number of bytes to copy
 * @ throws IOException if there is an error while reading or writing */
private static void copy ( SeekableInputStream from , PositionOutputStream to , long start , long length ) throws IOException { } }

implementation: LOG . debug ( "Copying {} bytes at {} to {}" , length , start , to . getPos ( ) ) ; from . seek ( start ) ; long bytesCopied = 0 ; byte [ ] buffer = COPY_BUFFER . get ( ) ; while ( bytesCopied < length ) { long bytesLeft = length - bytesCopied ; int bytesRead = from . read ( buffer , 0 , ( buffer . length < bytesLeft ? buffer . length : ( int ) bytesLeft ) ) ; if ( bytesRead < 0 ) { throw new IllegalArgumentException ( "Unexpected end of input file at " + start + bytesCopied ) ; } to . write ( buffer , 0 , bytesRead ) ; bytesCopied += bytesRead ; }


signature:
public class ParquetFileWriter { /** * ends a file once all blocks have been written .
 * closes the file .
 * @ param extraMetaData the extra meta data to write in the footer
 * @ throws IOException if there is an error while writing */
public void end ( Map < String , String > extraMetaData ) throws IOException { } }

implementation: state = state . end ( ) ; serializeColumnIndexes ( columnIndexes , blocks , out ) ; serializeOffsetIndexes ( offsetIndexes , blocks , out ) ; LOG . debug ( "{}: end" , out . getPos ( ) ) ; this . footer = new ParquetMetadata ( new FileMetaData ( schema , extraMetaData , Version . FULL_VERSION ) , blocks ) ; serializeFooter ( footer , out ) ; out . close ( ) ;


signature:
public class ParquetFileWriter { /** * Given a list of metadata files , merge them into a single ParquetMetadata
 * Requires that the schemas be compatible , and the extraMetadata be exactly equal .
 * @ param files a list of files to merge metadata from
 * @ param conf a configuration
 * @ return merged parquet metadata for the files
 * @ throws IOException if there is an error while writing
 * @ deprecated metadata files are not recommended and will be removed in 2.0.0 */
@ Deprecated public static ParquetMetadata mergeMetadataFiles ( List < Path > files , Configuration conf ) throws IOException { } }

implementation:
Preconditions . checkArgument ( ! files . isEmpty ( ) , "Cannot merge an empty list of metadata" ) ; GlobalMetaData globalMetaData = null ; List < BlockMetaData > blocks = new ArrayList < BlockMetaData > ( ) ; for ( Path p : files ) { ParquetMetadata pmd = ParquetFileReader . readFooter ( conf , p , ParquetMetadataConverter . NO_FILTER ) ; FileMetaData fmd = pmd . getFileMetaData ( ) ; globalMetaData = mergeInto ( fmd , globalMetaData , true ) ; blocks . addAll ( pmd . getBlocks ( ) ) ; } // collapse GlobalMetaData into a single FileMetaData , which will throw if they are not compatible
return new ParquetMetadata ( globalMetaData . merge ( ) , blocks ) ;


signature:
public class ParquetFileWriter { /** * Given a list of metadata files , merge them into a single metadata file .
 * Requires that the schemas be compatible , and the extraMetaData be exactly equal .
 * This is useful when merging 2 directories of parquet files into a single directory , as long
 * as both directories were written with compatible schemas and equal extraMetaData .
 * @ param files a list of files to merge metadata from
 * @ param outputPath path to write merged metadata to
 * @ param conf a configuration
 * @ throws IOException if there is an error while reading or writing
 * @ deprecated metadata files are not recommended and will be removed in 2.0.0 */
@ Deprecated public static void writeMergedMetadataFile ( List < Path > files , Path outputPath , Configuration conf ) throws IOException { } }

implementation: ParquetMetadata merged = mergeMetadataFiles ( files , conf ) ; writeMetadataFile ( outputPath , merged , outputPath . getFileSystem ( conf ) ) ;


signature:
public class ParquetFileWriter { /** * writes a _ metadata and _ common _ metadata file
 * @ param configuration the configuration to use to get the FileSystem
 * @ param outputPath the directory to write the _ metadata file to
 * @ param footers the list of footers to merge
 * @ throws IOException if there is an error while writing
 * @ deprecated metadata files are not recommended and will be removed in 2.0.0 */
@ Deprecated public static void writeMetadataFile ( Configuration configuration , Path outputPath , List < Footer > footers ) throws IOException { } }

implementation: writeMetadataFile ( configuration , outputPath , footers , JobSummaryLevel . ALL ) ;


signature:
public class ParquetFileWriter { /** * writes _ common _ metadata file , and optionally a _ metadata file depending on the { @ link JobSummaryLevel } provided
 * @ param configuration the configuration to use to get the FileSystem
 * @ param outputPath the directory to write the _ metadata file to
 * @ param footers the list of footers to merge
 * @ param level level of summary to write
 * @ throws IOException if there is an error while writing
 * @ deprecated metadata files are not recommended and will be removed in 2.0.0 */
@ Deprecated public static void writeMetadataFile ( Configuration configuration , Path outputPath , List < Footer > footers , JobSummaryLevel level ) throws IOException { } }

implementation: Preconditions . checkArgument ( level == JobSummaryLevel . ALL || level == JobSummaryLevel . COMMON_ONLY , "Unsupported level: " + level ) ; FileSystem fs = outputPath . getFileSystem ( configuration ) ; outputPath = outputPath . makeQualified ( fs ) ; ParquetMetadata metadataFooter = mergeFooters ( outputPath , footers ) ; if ( level == JobSummaryLevel . ALL ) { writeMetadataFile ( outputPath , metadataFooter , fs , PARQUET_METADATA_FILE ) ; } metadataFooter . getBlocks ( ) . clear ( ) ; writeMetadataFile ( outputPath , metadataFooter , fs , PARQUET_COMMON_METADATA_FILE ) ;


signature:
public class ParquetFileWriter { /** * Will return the result of merging toMerge into mergedMetadata
 * @ param toMerge the metadata toMerge
 * @ param mergedMetadata the reference metadata to merge into
 * @ return the result of the merge */
static GlobalMetaData mergeInto ( FileMetaData toMerge , GlobalMetaData mergedMetadata ) { } }

implementation: return mergeInto ( toMerge , mergedMetadata , true ) ;


signature:
public class ParquetFileWriter { /** * will return the result of merging toMerge into mergedSchema
 * @ param toMerge the schema to merge into mergedSchema
 * @ param mergedSchema the schema to append the fields to
 * @ return the resulting schema */
static MessageType mergeInto ( MessageType toMerge , MessageType mergedSchema ) { } }

implementation: return mergeInto ( toMerge , mergedSchema , true ) ;


signature:
public class ParquetFileWriter { /** * will return the result of merging toMerge into mergedSchema
 * @ param toMerge the schema to merge into mergedSchema
 * @ param mergedSchema the schema to append the fields to
 * @ param strict should schema primitive types match
 * @ return the resulting schema */
static MessageType mergeInto ( MessageType toMerge , MessageType mergedSchema , boolean strict ) { } }

implementation: if ( mergedSchema == null ) { return toMerge ; } return mergedSchema . union ( toMerge , strict ) ;


signature:
public class ColumnReaderBase { /** * Reads the value into the binding . */
public void readValue ( ) { } }

implementation:
try { if ( ! valueRead ) { binding . read ( ) ; valueRead = true ; } } catch ( RuntimeException e ) { if ( CorruptDeltaByteArrays . requiresSequentialReads ( writerVersion , currentEncoding ) && e instanceof ArrayIndexOutOfBoundsException ) { // this is probably PARQUET - 246 , which may happen if reading data with
// MR because this can ' t be detected without reading all footers
throw new ParquetDecodingException ( "Read failure possibly due to " + "PARQUET-246: try setting parquet.split.files to false" , new ParquetDecodingException ( format ( "Can't read value in column %s at value %d out of %d, " + "%d out of %d in currentPage. repetition level: " + "%d, definition level: %d" , path , readValues , totalValueCount , readValues - ( endOfPageValueCount - pageValueCount ) , pageValueCount , repetitionLevel , definitionLevel ) , e ) ) ; } throw new ParquetDecodingException ( format ( "Can't read value in column %s at value %d out of %d, " + "%d out of %d in currentPage. repetition level: " + "%d, definition level: %d" , path , readValues , totalValueCount , readValues - ( endOfPageValueCount - pageValueCount ) , pageValueCount , repetitionLevel , definitionLevel ) , e ) ; }


signature:
public class Schemas { /** * Returns whether null is allowed by the schema .
 * @ param schema a Schema
 * @ return true if schema allows the value to be null */
public static boolean nullOk ( Schema schema ) { } }

implementation: if ( Schema . Type . NULL == schema . getType ( ) ) { return true ; } else if ( Schema . Type . UNION == schema . getType ( ) ) { for ( Schema possible : schema . getTypes ( ) ) { if ( nullOk ( possible ) ) { return true ; } } } return false ;


signature:
public class Schemas { /** * Merges { @ link Schema } instances if they are compatible .
 * Schemas are incompatible if :
 * < ul >
 * < li > The { @ link Schema . Type } does not match . < / li >
 * < li > For record schemas , the record name does not match < / li >
 * < li > For enum schemas , the enum name does not match < / li >
 * < / ul >
 * Map value , array element , and record field types types will use unions if
 * necessary , and union schemas are merged recursively .
 * @ param schemas a set of { @ code Schema } instances to merge
 * @ return a merged { @ code Schema }
 * @ throws IllegalStateException if the schemas are not compatible */
public static Schema merge ( Iterable < Schema > schemas ) { } }

implementation: Iterator < Schema > iter = schemas . iterator ( ) ; if ( ! iter . hasNext ( ) ) { return null ; } Schema result = iter . next ( ) ; while ( iter . hasNext ( ) ) { result = merge ( result , iter . next ( ) ) ; } return result ;


signature:
public class Schemas { /** * Merges two { @ link Schema } instances if they are compatible .
 * Two schemas are incompatible if :
 * < ul >
 * < li > The { @ link Schema . Type } does not match . < / li >
 * < li > For record schemas , the record name does not match < / li >
 * < li > For enum schemas , the enum name does not match < / li >
 * < / ul >
 * Map value and array element types will use unions if necessary , and union
 * schemas are merged recursively .
 * @ param left a { @ code Schema }
 * @ param right a { @ code Schema }
 * @ return a merged { @ code Schema }
 * @ throws IllegalStateException if the schemas are not compatible */
public static Schema merge ( Schema left , Schema right ) { } }

implementation: Schema merged = mergeOnly ( left , right ) ; Preconditions . checkState ( merged != null , "Cannot merge %s and %s" , left , right ) ; return merged ;


signature:
public class Schemas { /** * Merges two { @ link Schema } instances or returns { @ code null } .
 * The two schemas are merged if they are the same type . Records are merged
 * if the two records have the same name or have no names but have a
 * significant number of shared fields .
 * @ see { @ link # mergeOrUnion } to return a union when a merge is not possible .
 * @ param left a { @ code Schema }
 * @ param right a { @ code Schema }
 * @ return a { @ code Schema } for both types */
private static Schema mergeOrUnion ( Schema left , Schema right ) { } }

implementation: Schema merged = mergeOnly ( left , right ) ; if ( merged != null ) { return merged ; } return union ( left , right ) ;


signature:
public class Schemas { /** * Creates a union of two { @ link Schema } instances .
 * If either { @ code Schema } is a union , this will attempt to merge the other
 * schema with the types contained in that union before adding more types to
 * the union that is produced .
 * If both schemas are not unions , no merge is attempted .
 * @ param left a { @ code Schema }
 * @ param right a { @ code Schema }
 * @ return a UNION schema of the to { @ code Schema } instances */
private static Schema union ( Schema left , Schema right ) { } }

implementation:
if ( left . getType ( ) == Schema . Type . UNION ) { if ( right . getType ( ) == Schema . Type . UNION ) { // combine the unions by adding each type in right individually
Schema combined = left ; for ( Schema type : right . getTypes ( ) ) { combined = union ( combined , type ) ; } return combined ; } else { boolean notMerged = true ; // combine a union with a non - union by checking if each type will merge
List < Schema > types = Lists . newArrayList ( ) ; Iterator < Schema > schemas = left . getTypes ( ) . iterator ( ) ; // try to merge each type and stop when one succeeds
while ( schemas . hasNext ( ) ) { Schema next = schemas . next ( ) ; Schema merged = mergeOnly ( next , right ) ; if ( merged != null ) { types . add ( merged ) ; notMerged = false ; break ; } else { // merge didn ' t work , add the type
types . add ( next ) ; } } // add the remaining types from the left union
while ( schemas . hasNext ( ) ) { types . add ( schemas . next ( ) ) ; } if ( notMerged ) { types . add ( right ) ; } return Schema . createUnion ( types ) ; } } else if ( right . getType ( ) == Schema . Type . UNION ) { return union ( right , left ) ; } return Schema . createUnion ( ImmutableList . of ( left , right ) ) ;


signature:
public class Schemas { /** * Merges two { @ link Schema } instances or returns { @ code null } .
 * The two schemas are merged if they are the same type . Records are merged
 * if the two records have the same name or have no names but have a
 * significant number of shared fields .
 * @ see { @ link # mergeOrUnion } to return a union when a merge is not possible .
 * @ param left a { @ code Schema }
 * @ param right a { @ code Schema }
 * @ return a merged { @ code Schema } or { @ code null } if merging is not possible */
private static Schema mergeOnly ( Schema left , Schema right ) { } }

implementation:
if ( Objects . equal ( left , right ) ) { return left ; } // handle primitive type promotion ; doesn ' t promote integers to floats
switch ( left . getType ( ) ) { case INT : if ( right . getType ( ) == Schema . Type . LONG ) { return right ; } break ; case LONG : if ( right . getType ( ) == Schema . Type . INT ) { return left ; } break ; case FLOAT : if ( right . getType ( ) == Schema . Type . DOUBLE ) { return right ; } break ; case DOUBLE : if ( right . getType ( ) == Schema . Type . FLOAT ) { return left ; } } // any other cases where the types don ' t match must be combined by a union
if ( left . getType ( ) != right . getType ( ) ) { return null ; } switch ( left . getType ( ) ) { case UNION : return union ( left , right ) ; case RECORD : if ( left . getName ( ) == null && right . getName ( ) == null && fieldSimilarity ( left , right ) < SIMILARITY_THRESH ) { return null ; } else if ( ! Objects . equal ( left . getName ( ) , right . getName ( ) ) ) { return null ; } Schema combinedRecord = Schema . createRecord ( coalesce ( left . getName ( ) , right . getName ( ) ) , coalesce ( left . getDoc ( ) , right . getDoc ( ) ) , coalesce ( left . getNamespace ( ) , right . getNamespace ( ) ) , false ) ; combinedRecord . setFields ( mergeFields ( left , right ) ) ; return combinedRecord ; case MAP : return Schema . createMap ( mergeOrUnion ( left . getValueType ( ) , right . getValueType ( ) ) ) ; case ARRAY : return Schema . createArray ( mergeOrUnion ( left . getElementType ( ) , right . getElementType ( ) ) ) ; case ENUM : if ( ! Objects . equal ( left . getName ( ) , right . getName ( ) ) ) { return null ; } Set < String > symbols = Sets . newLinkedHashSet ( ) ; symbols . addAll ( left . getEnumSymbols ( ) ) ; symbols . addAll ( right . getEnumSymbols ( ) ) ; return Schema . createEnum ( left . getName ( ) , coalesce ( left . getDoc ( ) , right . getDoc ( ) ) , coalesce ( left . getNamespace ( ) , right . getNamespace ( ) ) , ImmutableList . copyOf ( symbols ) ) ; default : // all primitives are handled before the switch by the equality check .
// schemas that reach this point are not primitives and also not any of
// the above known types .
throw new UnsupportedOperationException ( "Unknown schema type: " + left . getType ( ) ) ; }


signature:
public class Schemas { /** * Returns a union { @ link Schema } of NULL and the given { @ code schema } .
 * A NULL schema is always the first type in the union so that a null default
 * value can be set .
 * @ param schema a { @ code Schema }
 * @ return a union of null and the given schema */
private static Schema nullableForDefault ( Schema schema ) { } }

implementation: if ( schema . getType ( ) == Schema . Type . NULL ) { return schema ; } if ( schema . getType ( ) != Schema . Type . UNION ) { return Schema . createUnion ( ImmutableList . of ( NULL , schema ) ) ; } if ( schema . getTypes ( ) . get ( 0 ) . getType ( ) == Schema . Type . NULL ) { return schema ; } List < Schema > types = Lists . newArrayList ( ) ; types . add ( NULL ) ; for ( Schema type : schema . getTypes ( ) ) { if ( type . getType ( ) != Schema . Type . NULL ) { types . add ( type ) ; } } return Schema . createUnion ( types ) ;


signature:
public class Schemas { /** * Creates a new field with the same name , schema , doc , and default value as
 * the incoming schema .
 * Fields cannot be used in more than one record ( not Immutable ? ) .
 * @ param field an Avro schema field
 * @ return a copy of the field */
public static Schema . Field copy ( Schema . Field field ) { } }

implementation: return new Schema . Field ( field . name ( ) , field . schema ( ) , field . doc ( ) , field . defaultValue ( ) ) ;


signature:
public class Schemas { /** * Returns the first non - null object that is passed in . */
@ SafeVarargs private static < E > E coalesce ( E ... objects ) { } }

implementation: for ( E object : objects ) { if ( object != null ) { return object ; } } return null ;


signature:
public class ParquetRecordWriter { /** * { @ inheritDoc } */
@ Override public void close ( TaskAttemptContext context ) throws IOException , InterruptedException { } }

implementation:
try { internalWriter . close ( ) ; // release after the writer closes in case it is used for a last flush
} finally { if ( codecFactory != null ) { codecFactory . release ( ) ; } if ( memoryManager != null ) { memoryManager . removeWriter ( internalWriter ) ; } }


signature:
public class ParquetRecordWriter { /** * { @ inheritDoc } */
@ Override public void write ( Void key , T value ) throws IOException , InterruptedException { } }

implementation: internalWriter . write ( value ) ;


signature:
public class RecordBuilder { /** * Returns a the value as the first matching schema type or null .
 * Note that if the value may be null even if the schema does not allow the
 * value to be null .
 * @ param string a String representation of the value
 * @ param schema a Schema
 * @ return the string coerced to the correct type from the schema or null */
private static Object makeValue ( String string , Schema schema ) { } }

implementation:
if ( string == null ) { return null ; } try { switch ( schema . getType ( ) ) { case BOOLEAN : return Boolean . valueOf ( string ) ; case STRING : return string ; case FLOAT : return Float . valueOf ( string ) ; case DOUBLE : return Double . valueOf ( string ) ; case INT : return Integer . valueOf ( string ) ; case LONG : return Long . valueOf ( string ) ; case ENUM : // TODO : translate to enum class
if ( schema . hasEnumSymbol ( string ) ) { return string ; } else { try { return schema . getEnumSymbols ( ) . get ( Integer . parseInt ( string ) ) ; } catch ( IndexOutOfBoundsException ex ) { return null ; } } case UNION : Object value = null ; for ( Schema possible : schema . getTypes ( ) ) { value = makeValue ( string , possible ) ; if ( value != null ) { return value ; } } return null ; case NULL : return null ; default : // FIXED , BYTES , MAP , ARRAY , RECORD are not supported
throw new RecordException ( "Unsupported field type:" + schema . getType ( ) ) ; } } catch ( NumberFormatException e ) { // empty string is considered null for numeric types
if ( string . isEmpty ( ) ) { return null ; } else { throw e ; } }


signature:
public class ThriftMetaData { /** * Reads ThriftMetadata from the parquet file footer .
 * @ param extraMetaData extraMetaData field of the parquet footer
 * @ return the ThriftMetaData used to write a data file */
public static ThriftMetaData fromExtraMetaData ( Map < String , String > extraMetaData ) { } }

implementation: final String thriftClassName = extraMetaData . get ( THRIFT_CLASS ) ; final String thriftDescriptorString = extraMetaData . get ( THRIFT_DESCRIPTOR ) ; if ( thriftClassName == null || thriftDescriptorString == null ) { return null ; } final StructType descriptor = parseDescriptor ( thriftDescriptorString ) ; return new ThriftMetaData ( thriftClassName , descriptor ) ;


signature:
public class ThriftMetaData { /** * Creates ThriftMetaData from a Thrift - generated class .
 * @ param thriftClass a Thrift - generated class
 * @ return ThriftMetaData for the given class */
@ SuppressWarnings ( "unchecked" ) public static ThriftMetaData fromThriftClass ( Class < ? > thriftClass ) { } }

implementation: if ( thriftClass != null && TBase . class . isAssignableFrom ( thriftClass ) ) { Class < ? extends TBase < ? , ? > > tClass = ( Class < ? extends TBase < ? , ? > > ) thriftClass ; StructType descriptor = new ThriftSchemaConverter ( ) . toStructType ( tClass ) ; return new ThriftMetaData ( thriftClass . getName ( ) , descriptor ) ; } return null ;


signature:
public class ThriftMetaData { /** * generates a map of key values to store in the footer
 * @ return the key values */
public Map < String , String > toExtraMetaData ( ) { } }

implementation: final Map < String , String > map = new HashMap < String , String > ( ) ; map . put ( THRIFT_CLASS , getThriftClass ( ) . getName ( ) ) ; map . put ( THRIFT_DESCRIPTOR , descriptor . toJSON ( ) ) ; return map ;


signature:
public class ColumnWriterBase { /** * Writes the current null value
 * @ param repetitionLevel
 * @ param definitionLevel */
@ Override public void writeNull ( int repetitionLevel , int definitionLevel ) { } }

implementation: if ( DEBUG ) log ( null , repetitionLevel , definitionLevel ) ; repetitionLevel ( repetitionLevel ) ; definitionLevel ( definitionLevel ) ; statistics . incrementNumNulls ( ) ; ++ valueCount ;


signature:
public class ColumnWriterBase { /** * Writes the current value
 * @ param value
 * @ param repetitionLevel
 * @ param definitionLevel */
@ Override public void write ( double value , int repetitionLevel , int definitionLevel ) { } }

implementation: if ( DEBUG ) log ( value , repetitionLevel , definitionLevel ) ; repetitionLevel ( repetitionLevel ) ; definitionLevel ( definitionLevel ) ; dataColumn . writeDouble ( value ) ; statistics . updateStats ( value ) ; ++ valueCount ;


signature:
public class ColumnWriterBase { /** * Writes the current value
 * @ param value
 * @ param repetitionLevel
 * @ param definitionLevel */
@ Override public void write ( Binary value , int repetitionLevel , int definitionLevel ) { } }

implementation: if ( DEBUG ) log ( value , repetitionLevel , definitionLevel ) ; repetitionLevel ( repetitionLevel ) ; definitionLevel ( definitionLevel ) ; dataColumn . writeBytes ( value ) ; statistics . updateStats ( value ) ; ++ valueCount ;


signature:
public class ColumnWriterBase { /** * Finalizes the Column chunk . Possibly adding extra pages if needed ( dictionary , . . . )
 * Is called right after writePage */
void finalizeColumnChunk ( ) { } }

implementation: final DictionaryPage dictionaryPage = dataColumn . toDictPageAndClose ( ) ; if ( dictionaryPage != null ) { if ( DEBUG ) LOG . debug ( "write dictionary" ) ; try { pageWriter . writeDictionaryPage ( dictionaryPage ) ; } catch ( IOException e ) { throw new ParquetEncodingException ( "could not write dictionary page for " + path , e ) ; } dataColumn . resetDictionary ( ) ; }


signature:
public class ColumnWriterBase { /** * Writes the current data to a new page in the page store */
void writePage ( ) { } }

implementation: if ( valueCount == 0 ) { throw new ParquetEncodingException ( "writing empty page" ) ; } this . rowsWrittenSoFar += pageRowCount ; if ( DEBUG ) LOG . debug ( "write page" ) ; try { writePage ( pageRowCount , valueCount , statistics , repetitionLevelColumn , definitionLevelColumn , dataColumn ) ; } catch ( IOException e ) { throw new ParquetEncodingException ( "could not write page for " + path , e ) ; } repetitionLevelColumn . reset ( ) ; definitionLevelColumn . reset ( ) ; dataColumn . reset ( ) ; valueCount = 0 ; resetStatistics ( ) ; pageRowCount = 0 ;


signature:
public class StrictFieldProjectionFilter { /** * visible for testing */
static List < String > parseSemicolonDelimitedString ( String columnsToKeepGlobs ) { } }

implementation: String [ ] splits = columnsToKeepGlobs . split ( GLOB_SEPARATOR ) ; List < String > globs = new ArrayList < String > ( ) ; for ( String s : splits ) { if ( ! s . isEmpty ( ) ) { globs . add ( s ) ; } } if ( globs . isEmpty ( ) ) { throw new ThriftProjectionException ( String . format ( "Semicolon delimited string '%s' contains 0 glob strings" , columnsToKeepGlobs ) ) ; } return globs ;


signature:
public class StrictFieldProjectionFilter { /** * visible for testing */
boolean keep ( String path ) { } }

implementation:
WildcardPath match = null ; // since we have a rule of every path must match at least one column ,
// we visit every single wildcard path , instead of short circuiting ,
// for the case where more than one pattern matches a column . Otherwise
// we ' d get a misleading exception saying a path didn ' t match a column ,
// even though it looks like it should have ( but didn ' t because of short circuiting ) .
// This also allows us log a warning when more than one glob path matches .
for ( WildcardPathStatus wp : columnsToKeep ) { if ( wp . matches ( path ) ) { if ( match != null && ! match . getParentGlobPath ( ) . equals ( wp . getWildcardPath ( ) . getParentGlobPath ( ) ) ) { String message = "Field path: '%s' matched more than one glob path pattern. First match: " + "'%s' (when expanded to '%s') second match:'%s' (when expanded to '%s')" ; warn ( String . format ( message , path , match . getParentGlobPath ( ) , match . getOriginalPattern ( ) , wp . getWildcardPath ( ) . getParentGlobPath ( ) , wp . getWildcardPath ( ) . getOriginalPattern ( ) ) ) ; } else { match = wp . getWildcardPath ( ) ; } } } return match != null ;


signature:
public class ValidatingRecordConsumer { /** * { @ inheritDoc } */
public void endMessage ( ) { } }

implementation: delegate . endMessage ( ) ; validateMissingFields ( types . peek ( ) . asGroupType ( ) . getFieldCount ( ) ) ; previousField . pop ( ) ;


signature:
public class ValidatingRecordConsumer { /** * { @ inheritDoc } */
public void startField ( String field , int index ) { } }

implementation: if ( index <= previousField . peek ( ) ) { throw new InvalidRecordException ( "fields must be added in order " + field + " index " + index + " is before previous field " + previousField . peek ( ) ) ; } validateMissingFields ( index ) ; fields . push ( index ) ; fieldValueCount . push ( 0 ) ; delegate . startField ( field , index ) ;


signature:
public class ValidatingRecordConsumer { /** * { @ inheritDoc } */
public void endField ( String field , int index ) { } }

implementation: delegate . endField ( field , index ) ; fieldValueCount . pop ( ) ; previousField . push ( fields . pop ( ) ) ;


signature:
public class ValidatingRecordConsumer { /** * { @ inheritDoc } */
public void startGroup ( ) { } }

implementation: previousField . push ( - 1 ) ; types . push ( types . peek ( ) . asGroupType ( ) . getType ( fields . peek ( ) ) ) ; delegate . startGroup ( ) ;


signature:
public class ValidatingRecordConsumer { /** * { @ inheritDoc } */
public void endGroup ( ) { } }

implementation: delegate . endGroup ( ) ; validateMissingFields ( types . peek ( ) . asGroupType ( ) . getFieldCount ( ) ) ; types . pop ( ) ; previousField . pop ( ) ;


signature:
public class ValidatingRecordConsumer { /** * { @ inheritDoc } */
public void addBinary ( Binary value ) { } }

implementation: validate ( BINARY , INT96 , FIXED_LEN_BYTE_ARRAY ) ; delegate . addBinary ( value ) ;


signature:
public class PrimitiveType { /** * { @ inheritDoc } */
@ Override public void writeToStringBuilder ( StringBuilder sb , String indent ) { } }

implementation:
sb . append ( indent ) . append ( getRepetition ( ) . name ( ) . toLowerCase ( Locale . ENGLISH ) ) . append ( " " ) . append ( primitive . name ( ) . toLowerCase ( ) ) ; if ( primitive == PrimitiveTypeName . FIXED_LEN_BYTE_ARRAY ) { sb . append ( "(" + length + ")" ) ; } sb . append ( " " ) . append ( getName ( ) ) ; if ( getLogicalTypeAnnotation ( ) != null ) { // TODO : should we print decimal metadata too ?
sb . append ( " (" ) . append ( getLogicalTypeAnnotation ( ) . toString ( ) ) . append ( ")" ) ; } if ( getId ( ) != null ) { sb . append ( " = " ) . append ( getId ( ) ) ; }


signature:
public class PrimitiveType { /** * Returns the { @ link Type } specific comparator for properly comparing values . The natural ordering of the values
 * might not proper in certain cases ( e . g . { @ code UINT _ 32 } requires unsigned comparison of { @ code int } values while
 * the natural ordering is signed . )
 * @ param < T > the type of values compared by the returned PrimitiveComparator
 * @ return a PrimitiveComparator for values of this type */
@ SuppressWarnings ( "unchecked" ) public < T > PrimitiveComparator < T > comparator ( ) { } }

implementation: return ( PrimitiveComparator < T > ) getPrimitiveTypeName ( ) . comparator ( getLogicalTypeAnnotation ( ) ) ;


signature:
public class DeltaBinaryPackingValuesReader { /** * eagerly loads all the data into memory */
@ Override public void initFromPage ( int valueCount , ByteBufferInputStream stream ) throws IOException { } }

implementation:
this . in = stream ; long startPos = in . position ( ) ; this . config = DeltaBinaryPackingConfig . readConfig ( in ) ; this . totalValueCount = BytesUtils . readUnsignedVarInt ( in ) ; allocateValuesBuffer ( ) ; bitWidths = new int [ config . miniBlockNumInABlock ] ; // read first value from header
valuesBuffer [ valuesBuffered ++ ] = BytesUtils . readZigZagVarLong ( in ) ; while ( valuesBuffered < totalValueCount ) { // values Buffered could be more than totalValueCount , since we flush on a mini block basis
loadNewBlockToBuffer ( ) ; } updateNextOffset ( ( int ) ( in . position ( ) - startPos ) ) ;


signature:
public class DeltaBinaryPackingValuesReader { /** * the value buffer is allocated so that the size of it is multiple of mini block
 * because when writing , data is flushed on a mini block basis */
private void allocateValuesBuffer ( ) { } }

implementation:
int totalMiniBlockCount = ( int ) Math . ceil ( ( double ) totalValueCount / config . miniBlockSizeInValues ) ; // + 1 because first value written to header is also stored in values buffer
valuesBuffer = new long [ totalMiniBlockCount * config . miniBlockSizeInValues + 1 ] ;


signature:
public class DeltaBinaryPackingValuesReader { /** * mini block has a size of 8 * n , unpack 8 value each time
 * @ param packer the packer created from bitwidth of current mini block */
private void unpackMiniBlock ( BytePackerForLong packer ) throws IOException { } }

implementation: for ( int j = 0 ; j < config . miniBlockSizeInValues ; j += 8 ) { unpack8Values ( packer ) ; }


signature:
public class GlobalMetaData { /** * Will merge the metadata as if it was coming from a single file .
 * ( for all part files written together this will always work )
 * If there are conflicting values an exception will be thrown
 * @ return the merged version of this */
public FileMetaData merge ( ) { } }

implementation: String createdByString = createdBy . size ( ) == 1 ? createdBy . iterator ( ) . next ( ) : createdBy . toString ( ) ; Map < String , String > mergedKeyValues = new HashMap < String , String > ( ) ; for ( Entry < String , Set < String > > entry : keyValueMetaData . entrySet ( ) ) { if ( entry . getValue ( ) . size ( ) > 1 ) { throw new RuntimeException ( "could not merge metadata: key " + entry . getKey ( ) + " has conflicting values: " + entry . getValue ( ) ) ; } mergedKeyValues . put ( entry . getKey ( ) , entry . getValue ( ) . iterator ( ) . next ( ) ) ; } return new FileMetaData ( schema , mergedKeyValues , createdByString ) ;


signature:
public class LogicalTypeAnnotation { /** * Helper method to convert the old representation of logical types ( OriginalType ) to new logical type . */
@ InterfaceAudience . Private public static LogicalTypeAnnotation fromOriginalType ( OriginalType originalType , DecimalMetadata decimalMetadata ) { } }

implementation: if ( originalType == null ) { return null ; } switch ( originalType ) { case UTF8 : return stringType ( ) ; case MAP : return mapType ( ) ; case DECIMAL : int scale = ( decimalMetadata == null ? 0 : decimalMetadata . getScale ( ) ) ; int precision = ( decimalMetadata == null ? 0 : decimalMetadata . getPrecision ( ) ) ; return decimalType ( scale , precision ) ; case LIST : return listType ( ) ; case DATE : return dateType ( ) ; case INTERVAL : return IntervalLogicalTypeAnnotation . getInstance ( ) ; case TIMESTAMP_MILLIS : return timestampType ( true , LogicalTypeAnnotation . TimeUnit . MILLIS ) ; case TIMESTAMP_MICROS : return timestampType ( true , LogicalTypeAnnotation . TimeUnit . MICROS ) ; case TIME_MILLIS : return timeType ( true , LogicalTypeAnnotation . TimeUnit . MILLIS ) ; case TIME_MICROS : return timeType ( true , LogicalTypeAnnotation . TimeUnit . MICROS ) ; case UINT_8 : return intType ( 8 , false ) ; case UINT_16 : return intType ( 16 , false ) ; case UINT_32 : return intType ( 32 , false ) ; case UINT_64 : return intType ( 64 , false ) ; case INT_8 : return intType ( 8 , true ) ; case INT_16 : return intType ( 16 , true ) ; case INT_32 : return intType ( 32 , true ) ; case INT_64 : return intType ( 64 , true ) ; case ENUM : return enumType ( ) ; case JSON : return jsonType ( ) ; case BSON : return bsonType ( ) ; case MAP_KEY_VALUE : return MapKeyValueTypeAnnotation . getInstance ( ) ; default : throw new RuntimeException ( "Can't convert original type to logical type, unknown original type " + originalType ) ; }


signature:
public class ParquetThriftStorer { /** * { @ inheritDoc } */
@ Override public OutputFormat < Void , Tuple > getOutputFormat ( ) throws IOException { } }

implementation: return new ParquetOutputFormat < Tuple > ( new TupleToThriftWriteSupport ( className ) ) ;


signature:
public class DeltaBinaryPackingValuesWriterForLong { /** * getBytes will trigger flushing block buffer , DO NOT write after getBytes ( ) is called without calling reset ( )
 * @ return a BytesInput that contains the encoded page data */
@ Override public BytesInput getBytes ( ) { } }

implementation:
// The Page Header should include : blockSizeInValues , numberOfMiniBlocks , totalValueCount
if ( deltaValuesToFlush != 0 ) { flushBlockBuffer ( ) ; } return BytesInput . concat ( config . toBytesInput ( ) , BytesInput . fromUnsignedVarInt ( totalValueCount ) , BytesInput . fromZigZagVarLong ( firstValue ) , BytesInput . from ( baos ) ) ;


signature:
public class Preconditions { /** * Precondition - style validation that throws { @ link IllegalArgumentException } .
 * @ param isValid
 * { @ code true } if valid , { @ code false } if an exception should be
 * thrown
 * @ param message
 * A String message for the exception .
 * @ param args
 * Objects used to fill in { @ code % s } placeholders in the message
 * @ throws IllegalArgumentException if { @ code isValid } is false */
public static void checkArgument ( boolean isValid , String message , Object ... args ) throws IllegalArgumentException { } }

implementation: if ( ! isValid ) { throw new IllegalArgumentException ( String . format ( String . valueOf ( message ) , strings ( args ) ) ) ; }


signature:
public class Preconditions { /** * Precondition - style validation that throws { @ link IllegalStateException } .
 * @ param isValid
 * { @ code true } if valid , { @ code false } if an exception should be
 * thrown
 * @ param message
 * A String message for the exception .
 * @ param args
 * Objects used to fill in { @ code % s } placeholders in the message
 * @ throws IllegalStateException if { @ code isValid } is false */
public static void checkState ( boolean isValid , String message , Object ... args ) throws IllegalStateException { } }

implementation: if ( ! isValid ) { throw new IllegalStateException ( String . format ( String . valueOf ( message ) , strings ( args ) ) ) ; }


signature:
public class BufferedProtocolReadToWrite { /** * Reads one record from in and writes it to out .
 * Exceptions encountered during reading are treated as skippable exceptions ,
 * { @ link FieldIgnoredHandler } will be notified when registered .
 * @ param in input protocol
 * @ param out output protocol
 * @ throws org . apache . thrift . TException when an error happened while writing . Those are usually not recoverable */
@ Override public void readOne ( TProtocol in , TProtocol out ) throws TException { } }

implementation: List < Action > buffer = new LinkedList < Action > ( ) ; try { boolean hasFieldsIgnored = readOneStruct ( in , buffer , thriftType ) ; if ( hasFieldsIgnored ) { notifyRecordHasFieldIgnored ( ) ; } } catch ( Exception e ) { throw new SkippableException ( error ( "Error while reading" , buffer ) , e ) ; } try { for ( Action a : buffer ) { a . write ( out ) ; } } catch ( Exception e ) { throw new TException ( error ( "Can not write record" , buffer ) , e ) ; }


signature:
public class BufferedProtocolReadToWrite { /** * In thrift enum values are written as ints , this method checks if the enum index is defined .
 * @ param expectedType
 * @ param i */
private void checkEnum ( ThriftType expectedType , int i ) { } }

implementation: if ( expectedType . getType ( ) == ThriftTypeID . ENUM ) { ThriftType . EnumType expectedEnumType = ( ThriftType . EnumType ) expectedType ; if ( expectedEnumType . getEnumValueById ( i ) == null ) { throw new DecodingSchemaMismatchException ( "can not find index " + i + " in enum " + expectedType ) ; } }


signature:
public class ReadSupport { /** * attempts to validate and construct a { @ link MessageType } from a read projection schema
 * @ param fileMessageType the typed schema of the source
 * @ param partialReadSchemaString the requested projection schema
 * @ return the typed schema that should be used to read */
public static MessageType getSchemaForRead ( MessageType fileMessageType , String partialReadSchemaString ) { } }

implementation: if ( partialReadSchemaString == null ) return fileMessageType ; MessageType requestedMessageType = MessageTypeParser . parseMessageType ( partialReadSchemaString ) ; return getSchemaForRead ( fileMessageType , requestedMessageType ) ;


signature:
public class ReadSupport { /** * called in { @ link org . apache . hadoop . mapreduce . InputFormat # getSplits ( org . apache . hadoop . mapreduce . JobContext ) } in the front end
 * @ param configuration the job configuration
 * @ param keyValueMetaData the app specific metadata from the file
 * @ param fileSchema the schema of the file
 * @ return the readContext that defines how to read the file
 * @ deprecated override { @ link ReadSupport # init ( InitContext ) } instead */
@ Deprecated public ReadContext init ( Configuration configuration , Map < String , String > keyValueMetaData , MessageType fileSchema ) { } }

implementation: throw new UnsupportedOperationException ( "Override init(InitContext)" ) ;


signature:
public class ReadSupport { /** * called in { @ link org . apache . hadoop . mapreduce . InputFormat # getSplits ( org . apache . hadoop . mapreduce . JobContext ) } in the front end
 * @ param context the initialisation context
 * @ return the readContext that defines how to read the file */
public ReadContext init ( InitContext context ) { } }

implementation: return init ( context . getConfiguration ( ) , context . getMergedKeyValueMetaData ( ) , context . getFileSchema ( ) ) ;


signature:
public class BinaryStatistics { /** * Sets min and max values , re - uses the byte [ ] passed in .
 * Any changes made to byte [ ] will be reflected in min and max values as well .
 * @ param minBytes byte array to set the min value to
 * @ param maxBytes byte array to set the max value to */
@ Override public void setMinMaxFromBytes ( byte [ ] minBytes , byte [ ] maxBytes ) { } }

implementation: max = Binary . fromReusedByteArray ( maxBytes ) ; min = Binary . fromReusedByteArray ( minBytes ) ; this . markAsNotEmpty ( ) ;


signature:
public class DeltaBinaryPackingValuesWriterForInteger { /** * iterate through values in each mini block and calculate the bitWidths of max values .
 * @ param miniBlocksToFlush number of miniblocks */
private void calculateBitWidthsForDeltaBlockBuffer ( int miniBlocksToFlush ) { } }

implementation:
for ( int miniBlockIndex = 0 ; miniBlockIndex < miniBlocksToFlush ; miniBlockIndex ++ ) { int mask = 0 ; int miniStart = miniBlockIndex * config . miniBlockSizeInValues ; // The end of current mini block could be the end of current block ( deltaValuesToFlush ) buffer when data is not aligned to mini block
int miniEnd = Math . min ( ( miniBlockIndex + 1 ) * config . miniBlockSizeInValues , deltaValuesToFlush ) ; for ( int i = miniStart ; i < miniEnd ; i ++ ) { mask |= deltaBlockBuffer [ i ] ; } bitWidths [ miniBlockIndex ] = 32 - Integer . numberOfLeadingZeros ( mask ) ; }


signature:
public class AndRecordFilter { /** * Returns builder for creating an and filter .
 * @ param filter1 The first filter to check .
 * @ param filter2 The second filter to check .
 * @ return an unbound and filter */
public static final UnboundRecordFilter and ( final UnboundRecordFilter filter1 , final UnboundRecordFilter filter2 ) { } }

implementation: Preconditions . checkNotNull ( filter1 , "filter1" ) ; Preconditions . checkNotNull ( filter2 , "filter2" ) ; return new UnboundRecordFilter ( ) { @ Override public RecordFilter bind ( Iterable < ColumnReader > readers ) { return new AndRecordFilter ( filter1 . bind ( readers ) , filter2 . bind ( readers ) ) ; } } ;


signature:
public class MessageType { /** * { @ inheritDoc } */
@ Override public void writeToStringBuilder ( StringBuilder sb , String indent ) { } }

implementation: sb . append ( "message " ) . append ( getName ( ) ) . append ( getLogicalTypeAnnotation ( ) == null ? "" : " (" + getLogicalTypeAnnotation ( ) . toString ( ) + ")" ) . append ( " {\n" ) ; membersDisplayString ( sb , "  " ) ; sb . append ( "}\n" ) ;


signature:
public class ColumnIndexStoreImpl { /** * Creates a column index store which lazily reads column / offset indexes for the columns in paths . ( paths are the set
 * of columns used for the projection ) */
static ColumnIndexStore create ( ParquetFileReader reader , BlockMetaData block , Set < ColumnPath > paths ) { } }

implementation: try { return new ColumnIndexStoreImpl ( reader , block , paths ) ; } catch ( MissingOffsetIndexException e ) { return EMPTY ; }


signature:
public class NotRecordFilter { /** * Returns builder for creating an and filter .
 * @ param filter The filter to invert .
 * @ return a not record filter */
public static final UnboundRecordFilter not ( final UnboundRecordFilter filter ) { } }

implementation: Preconditions . checkNotNull ( filter , "filter" ) ; return new UnboundRecordFilter ( ) { @ Override public RecordFilter bind ( Iterable < ColumnReader > readers ) { return new NotRecordFilter ( filter . bind ( readers ) ) ; } } ;


signature:
public class Exceptions { /** * If the given throwable is an instance of E , throw it as an E .
 * @ param t an exception instance
 * @ param excClass an exception class t may be an instance of
 * @ param < E > the type of exception that will be thrown if throwable is an instance
 * @ throws E if t is an instance of E */
public static < E extends Exception > void throwIfInstance ( Throwable t , Class < E > excClass ) throws E { } }

implementation:
if ( excClass . isAssignableFrom ( t . getClass ( ) ) ) { // the throwable is already an exception , so return it
throw excClass . cast ( t ) ; }


signature:
public class Statistics { /** * Returns the typed statistics object based on the passed type parameter
 * @ param type PrimitiveTypeName type of the column
 * @ return instance of a typed statistics class
 * @ deprecated Use { @ link # createStats ( Type ) } instead */
@ Deprecated public static Statistics getStatsBasedOnType ( PrimitiveTypeName type ) { } }

implementation: switch ( type ) { case INT32 : return new IntStatistics ( ) ; case INT64 : return new LongStatistics ( ) ; case FLOAT : return new FloatStatistics ( ) ; case DOUBLE : return new DoubleStatistics ( ) ; case BOOLEAN : return new BooleanStatistics ( ) ; case BINARY : return new BinaryStatistics ( ) ; case INT96 : return new BinaryStatistics ( ) ; case FIXED_LEN_BYTE_ARRAY : return new BinaryStatistics ( ) ; default : throw new UnknownColumnTypeException ( type ) ; }


signature:
public class Statistics { /** * Creates an empty { @ code Statistics } instance for the specified type to be
 * used for reading / writing the new min / max statistics used in the V2 format .
 * @ param type
 * type of the column
 * @ return instance of a typed statistics class */
public static Statistics < ? > createStats ( Type type ) { } }

implementation: PrimitiveType primitive = type . asPrimitiveType ( ) ; switch ( primitive . getPrimitiveTypeName ( ) ) { case INT32 : return new IntStatistics ( primitive ) ; case INT64 : return new LongStatistics ( primitive ) ; case FLOAT : return new FloatStatistics ( primitive ) ; case DOUBLE : return new DoubleStatistics ( primitive ) ; case BOOLEAN : return new BooleanStatistics ( primitive ) ; case BINARY : case INT96 : case FIXED_LEN_BYTE_ARRAY : return new BinaryStatistics ( primitive ) ; default : throw new UnknownColumnTypeException ( primitive . getPrimitiveTypeName ( ) ) ; }


signature:
public class Statistics { /** * Returns a builder to create new statistics object . Used to read the statistics from the parquet file .
 * @ param type
 * type of the column
 * @ return builder to create new statistics object */
public static Builder getBuilderForReading ( PrimitiveType type ) { } }

implementation: switch ( type . getPrimitiveTypeName ( ) ) { case FLOAT : return new FloatBuilder ( type ) ; case DOUBLE : return new DoubleBuilder ( type ) ; default : return new Builder ( type ) ; }


signature:
public class Statistics { /** * Method to merge this statistics object with the object passed
 * as parameter . Merging keeps the smallest of min values , largest of max
 * values and combines the number of null counts .
 * @ param stats Statistics object to merge with */
public void mergeStatistics ( Statistics stats ) { } }

implementation:
if ( stats . isEmpty ( ) ) return ; // Merge stats only if they have the same type
if ( type . equals ( stats . type ) ) { incrementNumNulls ( stats . getNumNulls ( ) ) ; if ( stats . hasNonNullValue ( ) ) { mergeStatisticsMinMax ( stats ) ; markAsNotEmpty ( ) ; } } else { throw StatisticsClassException . create ( this , stats ) ; }


signature:
public class AvroSchemaConverter { /** * Given a schema , check to see if it is a union of a null type and a regular schema ,
 * and then return the non - null sub - schema . Otherwise , return the given schema .
 * @ param schema The schema to check
 * @ return The non - null portion of a union schema , or the given schema */
public static Schema getNonNull ( Schema schema ) { } }

implementation: if ( schema . getType ( ) . equals ( Schema . Type . UNION ) ) { List < Schema > schemas = schema . getTypes ( ) ; if ( schemas . size ( ) == 2 ) { if ( schemas . get ( 0 ) . getType ( ) . equals ( Schema . Type . NULL ) ) { return schemas . get ( 1 ) ; } else if ( schemas . get ( 1 ) . getType ( ) . equals ( Schema . Type . NULL ) ) { return schemas . get ( 0 ) ; } else { return schema ; } } else { return schema ; } } else { return schema ; }


signature:
public class AvroSchemaConverter { /** * Implements the rules for interpreting existing data from the logical type
 * spec for the LIST annotation . This is used to produce the expected schema .
 * The AvroArrayConverter will decide whether the repeated type is the array
 * element type by testing whether the element schema and repeated type are
 * the same . This ensures that the LIST rules are followed when there is no
 * schema and that a schema can be provided to override the default behavior . */
private boolean isElementType ( Type repeatedType , String parentName ) { } }

implementation:
return ( // can ' t be a synthetic layer because it would be invalid
repeatedType . isPrimitive ( ) || repeatedType . asGroupType ( ) . getFieldCount ( ) > 1 || repeatedType . asGroupType ( ) . getType ( 0 ) . isRepetition ( REPEATED ) || // known patterns without the synthetic layer
repeatedType . getName ( ) . equals ( "array" ) || repeatedType . getName ( ) . equals ( parentName + "_tuple" ) || // default assumption
assumeRepeatedIsListElement ) ;


signature:
public class ContextUtil { /** * Creates JobContext from a JobConf and jobId using the correct constructor
 * for based on Hadoop version . < code > jobId < / code > could be null .
 * @ param conf a configuration
 * @ param jobId a job id
 * @ return a job context */
public static JobContext newJobContext ( Configuration conf , JobID jobId ) { } }

implementation: try { return ( JobContext ) JOB_CONTEXT_CONSTRUCTOR . newInstance ( conf , jobId ) ; } catch ( InstantiationException e ) { throw new IllegalArgumentException ( "Can't instantiate JobContext" , e ) ; } catch ( IllegalAccessException e ) { throw new IllegalArgumentException ( "Can't instantiate JobContext" , e ) ; } catch ( InvocationTargetException e ) { throw new IllegalArgumentException ( "Can't instantiate JobContext" , e ) ; }


signature:
public class ContextUtil { /** * Creates TaskAttemptContext from a JobConf and jobId using the correct
 * constructor for based on Hadoop version .
 * @ param conf a configuration
 * @ param taskAttemptId a task attempt id
 * @ return a task attempt context */
public static TaskAttemptContext newTaskAttemptContext ( Configuration conf , TaskAttemptID taskAttemptId ) { } }

implementation: try { return ( TaskAttemptContext ) TASK_CONTEXT_CONSTRUCTOR . newInstance ( conf , taskAttemptId ) ; } catch ( InstantiationException e ) { throw new IllegalArgumentException ( "Can't instantiate TaskAttemptContext" , e ) ; } catch ( IllegalAccessException e ) { throw new IllegalArgumentException ( "Can't instantiate TaskAttemptContext" , e ) ; } catch ( InvocationTargetException e ) { throw new IllegalArgumentException ( "Can't instantiate TaskAttemptContext" , e ) ; }


signature:
public class ContextUtil { /** * Invoke getConfiguration ( ) method on JobContext . Works with both
 * Hadoop 1 and 2.
 * @ param context a job context
 * @ return the context ' s configuration */
public static Configuration getConfiguration ( JobContext context ) { } }

implementation: try { return ( Configuration ) GET_CONFIGURATION_METHOD . invoke ( context ) ; } catch ( IllegalAccessException e ) { throw new IllegalArgumentException ( "Can't invoke method" , e ) ; } catch ( InvocationTargetException e ) { throw new IllegalArgumentException ( "Can't invoke method" , e ) ; }


signature:
public class ContextUtil { /** * Invokes a method and rethrows any exception as runtime exceptions .
 * @ param method a method
 * @ param obj an object to run method on
 * @ param args an array of arguments to the method
 * @ return the result of the method call */
private static Object invoke ( Method method , Object obj , Object ... args ) { } }

implementation: try { return method . invoke ( obj , args ) ; } catch ( IllegalAccessException e ) { throw new IllegalArgumentException ( "Can't invoke method " + method . getName ( ) , e ) ; } catch ( InvocationTargetException e ) { throw new IllegalArgumentException ( "Can't invoke method " + method . getName ( ) , e ) ; }


signature:
public class GroupType { /** * appends a display string for of the members of this group to sb
 * @ param sb where to append
 * @ param indent the indentation level */
void membersDisplayString ( StringBuilder sb , String indent ) { } }

implementation: for ( Type field : fields ) { field . writeToStringBuilder ( sb , indent ) ; if ( field . isPrimitive ( ) ) { sb . append ( ";" ) ; } sb . append ( "\n" ) ; }


signature:
public class GroupType { /** * { @ inheritDoc } */
@ Override public void writeToStringBuilder ( StringBuilder sb , String indent ) { } }

implementation: sb . append ( indent ) . append ( getRepetition ( ) . name ( ) . toLowerCase ( Locale . ENGLISH ) ) . append ( " group " ) . append ( getName ( ) ) . append ( getLogicalTypeAnnotation ( ) == null ? "" : " (" + getLogicalTypeAnnotation ( ) . toString ( ) + ")" ) . append ( getId ( ) == null ? "" : " = " + getId ( ) ) . append ( " {\n" ) ; membersDisplayString ( sb , indent + "  " ) ; sb . append ( indent ) . append ( "}" ) ;


signature:
public class GroupType { /** * produces the list of fields resulting from merging toMerge into the fields of this
 * @ param toMerge the group containing the fields to merge
 * @ param strict should schema primitive types match
 * @ return the merged list */
List < Type > mergeFields ( GroupType toMerge , boolean strict ) { } }

implementation:
List < Type > newFields = new ArrayList < Type > ( ) ; // merge existing fields
for ( Type type : this . getFields ( ) ) { Type merged ; if ( toMerge . containsField ( type . getName ( ) ) ) { Type fieldToMerge = toMerge . getType ( type . getName ( ) ) ; if ( type . getLogicalTypeAnnotation ( ) != null && ! type . getLogicalTypeAnnotation ( ) . equals ( fieldToMerge . getLogicalTypeAnnotation ( ) ) ) { throw new IncompatibleSchemaModificationException ( "cannot merge logical type " + fieldToMerge . getLogicalTypeAnnotation ( ) + " into " + type . getLogicalTypeAnnotation ( ) ) ; } merged = type . union ( fieldToMerge , strict ) ; } else { merged = type ; } newFields . add ( merged ) ; } // add new fields
for ( Type type : toMerge . getFields ( ) ) { if ( ! this . containsField ( type . getName ( ) ) ) { newFields . add ( type ) ; } } return newFields ;


signature:
public class BenchmarkCounter { /** * Init counters in hadoop ' s mapred API , which is used by cascading and Hive .
 * @ param reporter a reporter
 * @ param configuration a configuration */
public static void initCounterFromReporter ( Reporter reporter , Configuration configuration ) { } }

implementation: counterLoader = new MapRedCounterLoader ( reporter , configuration ) ; loadCounters ( ) ;


signature:
public class SnappyCompressor { /** * Fills specified buffer with compressed data . Returns actual number
 * of bytes of compressed data . A return value of 0 indicates that
 * needsInput ( ) should be called in order to determine if more input
 * data is required .
 * @ param buffer Buffer for the compressed data
 * @ param off Start offset of the data
 * @ param len Size of the buffer
 * @ return The actual number of bytes of compressed data . */
@ Override public synchronized int compress ( byte [ ] buffer , int off , int len ) throws IOException { } }

implementation:
SnappyUtil . validateBuffer ( buffer , off , len ) ; if ( needsInput ( ) ) { // No buffered output bytes and no input to consume , need more input
return 0 ; } if ( ! outputBuffer . hasRemaining ( ) ) { // There is uncompressed input , compress it now
int maxOutputSize = Snappy . maxCompressedLength ( inputBuffer . position ( ) ) ; if ( maxOutputSize > outputBuffer . capacity ( ) ) { ByteBuffer oldBuffer = outputBuffer ; outputBuffer = ByteBuffer . allocateDirect ( maxOutputSize ) ; CleanUtil . clean ( oldBuffer ) ; } // Reset the previous outputBuffer
outputBuffer . clear ( ) ; inputBuffer . limit ( inputBuffer . position ( ) ) ; inputBuffer . position ( 0 ) ; int size = Snappy . compress ( inputBuffer , outputBuffer ) ; outputBuffer . limit ( size ) ; inputBuffer . limit ( 0 ) ; inputBuffer . rewind ( ) ; } // Return compressed output up to ' len '
int numBytes = Math . min ( len , outputBuffer . remaining ( ) ) ; outputBuffer . get ( buffer , off , numBytes ) ; bytesWritten += numBytes ; return numBytes ;


signature:
public class Closeables { /** * Closes a ( potentially null ) closeable , swallowing any IOExceptions thrown by
 * c . close ( ) . The exception will be logged .
 * @ param c can be null */
public static void closeAndSwallowIOExceptions ( Closeable c ) { } }

implementation: if ( c == null ) { return ; } try { c . close ( ) ; } catch ( IOException e ) { LOG . warn ( "Encountered exception closing closeable" , e ) ; }


signature:
public class ParquetInputFormat { /** * Returns a non - null Filter , which is a wrapper around either a
 * FilterPredicate , an UnboundRecordFilter , or a no - op filter .
 * @ param conf a configuration
 * @ return a filter for the unbound record filter specified in conf */
public static Filter getFilter ( Configuration conf ) { } }

implementation: return FilterCompat . get ( getFilterPredicate ( conf ) , getUnboundRecordFilterInstance ( conf ) ) ;


signature:
public class ParquetInputFormat { /** * { @ inheritDoc } */
@ Override public RecordReader < Void , T > createRecordReader ( InputSplit inputSplit , TaskAttemptContext taskAttemptContext ) throws IOException , InterruptedException { } }

implementation: Configuration conf = ContextUtil . getConfiguration ( taskAttemptContext ) ; ReadSupport < T > readSupport = getReadSupport ( conf ) ; return new ParquetRecordReader < T > ( readSupport , getFilter ( conf ) ) ;


signature:
public class ParquetInputFormat { /** * { @ inheritDoc } */
@ Override public List < InputSplit > getSplits ( JobContext jobContext ) throws IOException { } }

implementation:
Configuration configuration = ContextUtil . getConfiguration ( jobContext ) ; List < InputSplit > splits = new ArrayList < InputSplit > ( ) ; if ( isTaskSideMetaData ( configuration ) ) { // Although not required by the API , some clients may depend on always
// receiving ParquetInputSplit . Translation is required at some point .
for ( InputSplit split : super . getSplits ( jobContext ) ) { Preconditions . checkArgument ( split instanceof FileSplit , "Cannot wrap non-FileSplit: " + split ) ; splits . add ( ParquetInputSplit . from ( ( FileSplit ) split ) ) ; } return splits ; } else { splits . addAll ( getSplits ( configuration , getFooters ( jobContext ) ) ) ; } return splits ;


signature:
public class ParquetInputFormat { /** * This is to support multi - level / recursive directory listing until
 * MAPREDUCE - 1577 is fixed . */
@ Override protected List < FileStatus > listStatus ( JobContext jobContext ) throws IOException { } }

implementation: return getAllFileRecursively ( super . listStatus ( jobContext ) , ContextUtil . getConfiguration ( jobContext ) ) ;


signature:
public class ParquetInputFormat { /** * the footers for the files
 * @ param configuration to connect to the file system
 * @ param statuses the files to open
 * @ return the footers of the files
 * @ throws IOException if there is an error while reading */
public List < Footer > getFooters ( Configuration configuration , Collection < FileStatus > statuses ) throws IOException { } }

implementation: LOG . debug ( "reading {} files" , statuses . size ( ) ) ; boolean taskSideMetaData = isTaskSideMetaData ( configuration ) ; return ParquetFileReader . readAllFootersInParallelUsingSummaryFiles ( configuration , statuses , taskSideMetaData ) ;


signature:
public class ClientSideMetadataSplitStrategy { /** * groups together all the data blocks for the same HDFS block
 * @ param rowGroupBlocks data blocks ( row groups )
 * @ param hdfsBlocksArray hdfs blocks
 * @ param fileStatus the containing file
 * @ param requestedSchema the schema requested by the user
 * @ param readSupportMetadata the metadata provided by the readSupport implementation in init
 * @ param minSplitSize the mapred . min . split . size
 * @ param maxSplitSize the mapred . max . split . size
 * @ return the splits ( one per HDFS block )
 * @ throws IOException If hosts can ' t be retrieved for the HDFS block */
static < T > List < ParquetInputSplit > generateSplits ( List < BlockMetaData > rowGroupBlocks , BlockLocation [ ] hdfsBlocksArray , FileStatus fileStatus , String requestedSchema , Map < String , String > readSupportMetadata , long minSplitSize , long maxSplitSize ) throws IOException { } }

implementation:
List < SplitInfo > splitRowGroups = generateSplitInfo ( rowGroupBlocks , hdfsBlocksArray , minSplitSize , maxSplitSize ) ; // generate splits from rowGroups of each split
List < ParquetInputSplit > resultSplits = new ArrayList < ParquetInputSplit > ( ) ; for ( SplitInfo splitInfo : splitRowGroups ) { ParquetInputSplit split = splitInfo . getParquetInputSplit ( fileStatus , requestedSchema , readSupportMetadata ) ; resultSplits . add ( split ) ; } return resultSplits ;


signature:
public class FilterApi { /** * Keeps records if their value is equal to the provided value .
 * Nulls are treated the same way the java programming language does .
 * For example :
 * eq ( column , null ) will keep all records whose value is null .
 * eq ( column , 7 ) will keep all records whose value is 7 , and will drop records whose value is null
 * @ param column a column reference created by FilterApi
 * @ param value a value that matches the column ' s type
 * @ param < T > the Java type of values in the column
 * @ param < C > the column type that corresponds to values of type T
 * @ return an equals predicate for the given column and value */
public static < T extends Comparable < T > , C extends Column < T > & SupportsEqNotEq > Eq < T > eq ( C column , T value ) { } }

implementation: return new Eq < T > ( column , value ) ;


signature:
public class FilterApi { /** * Keeps records if their value is not equal to the provided value .
 * Nulls are treated the same way the java programming language does .
 * For example :
 * notEq ( column , null ) will keep all records whose value is not null .
 * notEq ( column , 7 ) will keep all records whose value is not 7 , including records whose value is null .
 * NOTE : this is different from how some query languages handle null . For example , SQL and pig will drop
 * nulls when you filter by not equal to 7 . To achieve similar behavior in this api , do :
 * and ( notEq ( column , 7 ) , notEq ( column , null ) )
 * NOTE : be sure to read the { @ link # lt } , { @ link # ltEq } , { @ link # gt } , { @ link # gtEq } operator ' s docs
 * for how they handle nulls
 * @ param column a column reference created by FilterApi
 * @ param value a value that matches the column ' s type
 * @ param < T > the Java type of values in the column
 * @ param < C > the column type that corresponds to values of type T
 * @ return a not - equals predicate for the given column and value */
public static < T extends Comparable < T > , C extends Column < T > & SupportsEqNotEq > NotEq < T > notEq ( C column , T value ) { } }

implementation: return new NotEq < T > ( column , value ) ;


signature:
public class FilterApi { /** * Keeps records if their value is less than ( but not equal to ) the provided value .
 * The provided value cannot be null , as less than null has no meaning .
 * Records with null values will be dropped .
 * For example :
 * lt ( column , 7 ) will keep all records whose value is less than ( but not equal to ) 7 , and not null .
 * @ param column a column reference created by FilterApi
 * @ param value a value that matches the column ' s type
 * @ param < T > the Java type of values in the column
 * @ param < C > the column type that corresponds to values of type T
 * @ return a less - than predicate for the given column and value */
public static < T extends Comparable < T > , C extends Column < T > & SupportsLtGt > Lt < T > lt ( C column , T value ) { } }

implementation: return new Lt < T > ( column , value ) ;


signature:
public class FilterApi { /** * Keeps records if their value is less than or equal to the provided value .
 * The provided value cannot be null , as less than null has no meaning .
 * Records with null values will be dropped .
 * For example :
 * ltEq ( column , 7 ) will keep all records whose value is less than or equal to 7 , and not null .
 * @ param column a column reference created by FilterApi
 * @ param value a value that matches the column ' s type
 * @ param < T > the Java type of values in the column
 * @ param < C > the column type that corresponds to values of type T
 * @ return a less - than - or - equal predicate for the given column and value */
public static < T extends Comparable < T > , C extends Column < T > & SupportsLtGt > LtEq < T > ltEq ( C column , T value ) { } }

implementation: return new LtEq < T > ( column , value ) ;


signature:
public class FilterApi { /** * Keeps records if their value is greater than ( but not equal to ) the provided value .
 * The provided value cannot be null , as less than null has no meaning .
 * Records with null values will be dropped .
 * For example :
 * gt ( column , 7 ) will keep all records whose value is greater than ( but not equal to ) 7 , and not null .
 * @ param column a column reference created by FilterApi
 * @ param value a value that matches the column ' s type
 * @ param < T > the Java type of values in the column
 * @ param < C > the column type that corresponds to values of type T
 * @ return a greater - than predicate for the given column and value */
public static < T extends Comparable < T > , C extends Column < T > & SupportsLtGt > Gt < T > gt ( C column , T value ) { } }

implementation: return new Gt < T > ( column , value ) ;


signature:
public class FilterApi { /** * Keeps records if their value is greater than or equal to the provided value .
 * The provided value cannot be null , as less than null has no meaning .
 * Records with null values will be dropped .
 * For example :
 * gtEq ( column , 7 ) will keep all records whose value is greater than or equal to 7 , and not null .
 * @ param column a column reference created by FilterApi
 * @ param value a value that matches the column ' s type
 * @ param < T > the Java type of values in the column
 * @ param < C > the column type that corresponds to values of type T
 * @ return a greater - than - or - equal predicate for the given column and value */
public static < T extends Comparable < T > , C extends Column < T > & SupportsLtGt > GtEq < T > gtEq ( C column , T value ) { } }

implementation: return new GtEq < T > ( column , value ) ;


signature:
public class FilterApi { /** * Keeps records that pass the provided { @ link UserDefinedPredicate }
 * The provided class must have a default constructor . To use an instance
 * of a UserDefinedPredicate instead , see userDefined below .
 * @ param column a column reference created by FilterApi
 * @ param clazz a user - defined predicate class
 * @ param < T > the Java type of values in the column
 * @ param < U > a user - defined predicate for values of type T
 * @ return a user - defined predicate for the given column */
public static < T extends Comparable < T > , U extends UserDefinedPredicate < T > > UserDefined < T , U > userDefined ( Column < T > column , Class < U > clazz ) { } }

implementation: return new UserDefinedByClass < T , U > ( column , clazz ) ;


signature:
public class FilterApi { /** * Keeps records that pass the provided { @ link UserDefinedPredicate }
 * The provided instance of UserDefinedPredicate must be serializable .
 * @ param column a column reference created by FilterApi
 * @ param udp a user - defined predicate instance
 * @ param < T > the Java type of values in the column
 * @ param < U > a user - defined predicate for values of type T
 * @ return a user - defined predicate for the given column */
public static < T extends Comparable < T > , U extends UserDefinedPredicate < T > & Serializable > UserDefined < T , U > userDefined ( Column < T > column , U udp ) { } }

implementation: return new UserDefinedByInstance < T , U > ( column , udp ) ;


signature:
public class Log { /** * prints a debug message
 * @ param m a log message */
public void debug ( Object m ) { } }

implementation: if ( m instanceof Throwable ) { logger . debug ( "" , ( Throwable ) m ) ; } else { logger . debug ( String . valueOf ( m ) ) ; }


signature:
public class Log { /** * prints a debug message
 * @ param m a log message
 * @ param t a throwable error */
public void debug ( Object m , Throwable t ) { } }

implementation: logger . debug ( String . valueOf ( m ) , t ) ;


signature:
public class Log { /** * prints an info message
 * @ param m a log message */
public void info ( Object m ) { } }

implementation: if ( m instanceof Throwable ) { logger . info ( "" , ( Throwable ) m ) ; } else { logger . info ( String . valueOf ( m ) ) ; }


signature:
public class Log { /** * prints an info message
 * @ param m a log message
 * @ param t a throwable error */
public void info ( Object m , Throwable t ) { } }

implementation: logger . info ( String . valueOf ( m ) , t ) ;


signature:
public class Log { /** * prints a warn message
 * @ param m a log message */
public void warn ( Object m ) { } }

implementation: if ( m instanceof Throwable ) { logger . warn ( "" , ( Throwable ) m ) ; } else { logger . warn ( String . valueOf ( m ) ) ; }


signature:
public class Log { /** * prints a warn message
 * @ param m a log message
 * @ param t a throwable error */
public void warn ( Object m , Throwable t ) { } }

implementation: logger . warn ( String . valueOf ( m ) , t ) ;


signature:
public class Log { /** * prints an error message
 * @ param m a log message */
public void error ( Object m ) { } }

implementation: if ( m instanceof Throwable ) { logger . error ( "" , ( Throwable ) m ) ; } else { logger . error ( String . valueOf ( m ) ) ; }


signature:
public class Log { /** * prints an error message
 * @ param m a log message
 * @ param t a throwable error */
public void error ( Object m , Throwable t ) { } }

implementation: logger . error ( String . valueOf ( m ) , t ) ;


signature:
public class OrRecordFilter { /** * Returns builder for creating an and filter .
 * @ param filter1 The first filter to check .
 * @ param filter2 The second filter to check .
 * @ return an or record filter */
public static final UnboundRecordFilter or ( final UnboundRecordFilter filter1 , final UnboundRecordFilter filter2 ) { } }

implementation: Preconditions . checkNotNull ( filter1 , "filter1" ) ; Preconditions . checkNotNull ( filter2 , "filter2" ) ; return new UnboundRecordFilter ( ) { @ Override public RecordFilter bind ( Iterable < ColumnReader > readers ) { return new OrRecordFilter ( filter1 . bind ( readers ) , filter2 . bind ( readers ) ) ; } } ;


signature:
public class ProtoWriteSupport { /** * Writes Protocol buffer to parquet file .
 * @ param record instance of Message . Builder or Message . */
@ Override public void write ( T record ) { } }

implementation: recordConsumer . startMessage ( ) ; try { messageWriter . writeTopLevelMessage ( record ) ; } catch ( RuntimeException e ) { Message m = ( record instanceof Message . Builder ) ? ( ( Message . Builder ) record ) . build ( ) : ( Message ) record ; LOG . error ( "Cannot write message " + e . getMessage ( ) + " : " + m ) ; throw e ; } recordConsumer . endMessage ( ) ;


signature:
public class ProtoWriteSupport { /** * validates mapping between protobuffer fields and parquet fields . */
private void validatedMapping ( Descriptor descriptor , GroupType parquetSchema ) { } }

implementation: List < FieldDescriptor > allFields = descriptor . getFields ( ) ; for ( FieldDescriptor fieldDescriptor : allFields ) { String fieldName = fieldDescriptor . getName ( ) ; int fieldIndex = fieldDescriptor . getIndex ( ) ; int parquetIndex = parquetSchema . getFieldIndex ( fieldName ) ; if ( fieldIndex != parquetIndex ) { String message = "FieldIndex mismatch name=" + fieldName + ": " + fieldIndex + " != " + parquetIndex ; throw new IncompatibleSchemaModificationException ( message ) ; } }


signature:
public class ProtoWriteSupport { /** * Returns message descriptor as JSON String */
private String serializeDescriptor ( Class < ? extends Message > protoClass ) { } }

implementation: Descriptor descriptor = Protobufs . getMessageDescriptor ( protoClass ) ; DescriptorProtos . DescriptorProto asProto = descriptor . toProto ( ) ; return TextFormat . printToString ( asProto ) ;


signature:
public class CodecFactory { /** * Create a codec factory that will provide compressors and decompressors
 * that will work natively with ByteBuffers backed by direct memory .
 * @ param config configuration options for different compression codecs
 * @ param allocator an allocator for creating result buffers during compression
 * and decompression , must provide buffers backed by Direct
 * memory and return true for the isDirect ( ) method
 * on the ByteBufferAllocator interface
 * @ param pageSize the default page size . This does not set a hard limit on the
 * size of buffers that can be compressed , but performance may
 * be improved by setting it close to the expected size of buffers
 * ( in the case of parquet , pages ) that will be compressed . This
 * setting is unused in the case of decompressing data , as parquet
 * always records the uncompressed size of a buffer . If this
 * CodecFactory is only going to be used for decompressors , this
 * parameter will not impact the function of the factory .
 * @ return a configured direct codec factory */
public static CodecFactory createDirectCodecFactory ( Configuration config , ByteBufferAllocator allocator , int pageSize ) { } }

implementation: return new DirectCodecFactory ( config , allocator , pageSize ) ;


signature:
public class Expressions { /** * a . 2 . b [ 3 ] [ " key " ]
 * * optional ( union with null ) should be ignored
 * * unions should match by position number or short name ( e . g . 2 , user )
 * * fields should match by name
 * * arrays are dereferenced by position [ n ] = & gt ; schema is the element schema
 * * maps are dereferenced by key = & gt ; schema is the value schema
 * @ param schema an Avro schema
 * @ param fieldPaths selected field paths
 * @ return a filtered schema */
public static Schema filterSchema ( Schema schema , String ... fieldPaths ) { } }

implementation: return filterSchema ( schema , Lists . newArrayList ( fieldPaths ) ) ;


signature:
public class ProtocolEventsAmender { /** * Given a thrift definition , protocols events , it checks all the required fields ,
 * and create default value if a required field is missing
 * @ param recordThriftType the Thrift Struct definition for events
 * @ return a list of events
 * @ throws TException if there is an error while amending events */
public List < TProtocol > amendMissingRequiredFields ( StructType recordThriftType ) throws TException { } }

implementation: Iterator < TProtocol > protocolIter = rootEvents . iterator ( ) ; checkStruct ( protocolIter , recordThriftType ) ; return fixedEvents ;


signature:
public class ProtocolEventsAmender { /** * check each element of the Set , make sure all the element contain required fields
 * @ param eventIter
 * @ param setFieldDefinition
 * @ throws TException */
private void checkSet ( Iterator < TProtocol > eventIter , ThriftField setFieldDefinition ) throws TException { } }

implementation: TSet thriftSet = acceptProtocol ( eventIter . next ( ) ) . readSetBegin ( ) ; ThriftField elementFieldDefinition = ( ( ThriftType . SetType ) setFieldDefinition . getType ( ) ) . getValues ( ) ; int setSize = thriftSet . size ; for ( int i = 0 ; i < setSize ; i ++ ) { checkField ( thriftSet . elemType , eventIter , elementFieldDefinition ) ; } acceptProtocol ( eventIter . next ( ) ) . readSetEnd ( ) ;


signature:
public class Util { /** * reads the meta data from the stream
 * @ param from the stream to read the metadata from
 * @ param skipRowGroups whether row groups should be skipped
 * @ return the resulting metadata
 * @ throws IOException if any I / O error occurs during the reading */
public static FileMetaData readFileMetaData ( InputStream from , boolean skipRowGroups ) throws IOException { } }

implementation: FileMetaData md = new FileMetaData ( ) ; if ( skipRowGroups ) { readFileMetaData ( from , new DefaultFileMetaDataConsumer ( md ) , skipRowGroups ) ; } else { read ( from , md ) ; } return md ;


signature:
public class ThriftToParquetFileWriter { /** * close the file
 * @ see java . io . Closeable # close ( ) */
@ Override public void close ( ) throws IOException { } }

implementation: try { recordWriter . close ( taskAttemptContext ) ; } catch ( InterruptedException e ) { Thread . interrupted ( ) ; throw new IOException ( "The thread was interrupted" , e ) ; }


signature:
public class ExampleOutputFormat { /** * set the schema being written to the job conf
 * @ param job a job
 * @ param schema the schema of the data */
public static void setSchema ( Job job , MessageType schema ) { } }

implementation: GroupWriteSupport . setSchema ( schema , ContextUtil . getConfiguration ( job ) ) ;


signature:
public class ValidTypeMap { /** * set up the mapping in both directions */
private static void add ( Class < ? > c , PrimitiveTypeName p ) { } }

implementation: Set < PrimitiveTypeName > descriptors = classToParquetType . get ( c ) ; if ( descriptors == null ) { descriptors = new HashSet < PrimitiveTypeName > ( ) ; classToParquetType . put ( c , descriptors ) ; } descriptors . add ( p ) ; Set < Class < ? > > classes = parquetTypeToClass . get ( p ) ; if ( classes == null ) { classes = new HashSet < Class < ? > > ( ) ; parquetTypeToClass . put ( p , classes ) ; } classes . add ( c ) ;


signature:
public class ValidTypeMap { /** * Asserts that foundColumn was declared as a type that is compatible with the type for this column found
 * in the schema of the parquet file .
 * @ throws java . lang . IllegalArgumentException if the types do not align
 * @ param foundColumn the column as declared by the user
 * @ param primitiveType the primitive type according to the schema
 * @ param < T > the java Type of values in the column , must be Comparable */
public static < T extends Comparable < T > > void assertTypeValid ( Column < T > foundColumn , PrimitiveTypeName primitiveType ) { } }

implementation: Class < T > foundColumnType = foundColumn . getColumnType ( ) ; ColumnPath columnPath = foundColumn . getColumnPath ( ) ; Set < PrimitiveTypeName > validTypeDescriptors = classToParquetType . get ( foundColumnType ) ; if ( validTypeDescriptors == null ) { StringBuilder message = new StringBuilder ( ) ; message . append ( "Column " ) . append ( columnPath . toDotString ( ) ) . append ( " was declared as type: " ) . append ( foundColumnType . getName ( ) ) . append ( " which is not supported in FilterPredicates." ) ; Set < Class < ? > > supportedTypes = parquetTypeToClass . get ( primitiveType ) ; if ( supportedTypes != null ) { message . append ( " Supported types for this column are: " ) . append ( supportedTypes ) ; } else { message . append ( " There are no supported types for columns of " + primitiveType ) ; } throw new IllegalArgumentException ( message . toString ( ) ) ; } if ( ! validTypeDescriptors . contains ( primitiveType ) ) { StringBuilder message = new StringBuilder ( ) ; message . append ( "FilterPredicate column: " ) . append ( columnPath . toDotString ( ) ) . append ( "'s declared type (" ) . append ( foundColumnType . getName ( ) ) . append ( ") does not match the schema found in file metadata. Column " ) . append ( columnPath . toDotString ( ) ) . append ( " is of type: " ) . append ( primitiveType ) . append ( "\nValid types for this column are: " ) . append ( parquetTypeToClass . get ( primitiveType ) ) ; throw new IllegalArgumentException ( message . toString ( ) ) ; }


signature:
public class Strings { /** * Join an Iterable of Strings into a single string with a delimiter .
 * For example , join ( Arrays . asList ( " foo " , " " , " bar " , " x " ) , " | " ) would return
 * " foo | | bar | x "
 * @ param s an iterable of strings
 * @ param on the delimiter
 * @ return a single joined string */
public static String join ( Iterable < String > s , String on ) { } }

implementation: return join ( s . iterator ( ) , on ) ;


signature:
public class Strings { /** * Join an Iterator of Strings into a single string with a delimiter .
 * For example , join ( Arrays . asList ( " foo " , " " , " bar " , " x " ) , " | " ) would return
 * " foo | | bar | x "
 * @ param iter an iterator of strings
 * @ param on the delimiter
 * @ return a single joined string */
public static String join ( Iterator < String > iter , String on ) { } }

implementation: StringBuilder sb = new StringBuilder ( ) ; while ( iter . hasNext ( ) ) { sb . append ( iter . next ( ) ) ; if ( iter . hasNext ( ) ) { sb . append ( on ) ; } } return sb . toString ( ) ;


signature:
public class Strings { /** * Join an Array of Strings into a single string with a delimiter .
 * For example , join ( new String [ ] { " foo " , " " , " bar " , " x " } , " | " ) would return
 * " foo | | bar | x "
 * @ param s an iterable of strings
 * @ param on the delimiter
 * @ return a single joined string */
public static String join ( String [ ] s , String on ) { } }

implementation: return join ( Arrays . asList ( s ) , on ) ;


signature:
public class Strings { /** * Expands a string according to { @ link # expandGlob ( String ) } , and then constructs a { @ link WildcardPath }
 * for each expanded result which can be used to match strings as described in { @ link WildcardPath } .
 * @ param globPattern a String to be passed to { @ link # expandGlob ( String ) }
 * @ param delim the delimeter used by { @ link WildcardPath }
 * @ return a list of wildcard paths , one for each expanded result */
public static List < WildcardPath > expandGlobToWildCardPaths ( String globPattern , char delim ) { } }

implementation: List < WildcardPath > ret = new ArrayList < WildcardPath > ( ) ; for ( String expandedGlob : Strings . expandGlob ( globPattern ) ) { ret . add ( new WildcardPath ( globPattern , expandedGlob , delim ) ) ; } return ret ;


signature:
public class Ints { /** * Cast value to a an int , or throw an exception
 * if there is an overflow .
 * @ param value a long to be casted to an int
 * @ return an int that is = = to value
 * @ throws IllegalArgumentException if value can ' t be casted to an int
 * @ deprecated replaced by { @ link java . lang . Math # toIntExact ( long ) } */
public static int checkedCast ( long value ) { } }

implementation: int valueI = ( int ) value ; if ( valueI != value ) { throw new IllegalArgumentException ( String . format ( "Overflow casting %d to an int" , value ) ) ; } return valueI ;


signature:
public class HiveBindingFactory { /** * Internal method visible for testing purposes */
@ SuppressWarnings ( "rawtypes" ) Class < ? extends HiveBinding > create ( ClassLoader classLoader ) { } }

implementation:
// HiveVersionInfo was added in 0.11 , if the class does
// not exist then return the hive binding for 0.10
Class hiveVersionInfo ; try { hiveVersionInfo = Class . forName ( HIVE_VERSION_CLASS_NAME , true , classLoader ) ; } catch ( ClassNotFoundException e ) { LOG . debug ( "Class " + HIVE_VERSION_CLASS_NAME + ", not found, returning {}" , Hive010Binding . class . getSimpleName ( ) ) ; return Hive010Binding . class ; } return createInternal ( hiveVersionInfo ) ;


signature:
public class HiveBindingFactory { /** * Internal method visible for testing purposes */
@ SuppressWarnings ( { } }

implementation:
"unchecked" , "rawtypes" } ) Class < ? extends HiveBinding > createInternal ( Class hiveVersionInfo ) { String hiveVersion ; try { Method getVersionMethod = hiveVersionInfo . getMethod ( HIVE_VERSION_METHOD_NAME , ( Class [ ] ) null ) ; String rawVersion = ( String ) getVersionMethod . invoke ( null , ( Object [ ] ) null ) ; LOG . debug ( "Raw Version from {} is '{}'" , hiveVersionInfo . getSimpleName ( ) , rawVersion ) ; hiveVersion = trimVersion ( rawVersion ) ; } catch ( Exception e ) { throw new UnexpectedHiveVersionProviderError ( "Unexpected error whilst " + "determining Hive version" , e ) ; } if ( hiveVersion . equalsIgnoreCase ( HIVE_VERSION_UNKNOWN ) ) { LOG . debug ( "Unknown hive version, attempting to guess" ) ; return createBindingForUnknownVersion ( ) ; } if ( hiveVersion . startsWith ( HIVE_VERSION_010 ) ) { LOG . debug ( "Hive version {}, returning {}" , hiveVersion , Hive010Binding . class . getSimpleName ( ) ) ; return Hive010Binding . class ; } else if ( hiveVersion . startsWith ( HIVE_VERSION_011 ) ) { LOG . debug ( "Hive version " + hiveVersion + ", returning " + Hive010Binding . class . getSimpleName ( ) + " as it's expected the 0.10 " + "binding will work with 0.11" ) ; return Hive010Binding . class ; } else if ( hiveVersion . startsWith ( HIVE_VERSION_013 ) ) { throw new HiveBindingInstantiationError ( "Hive 0.13 contains native Parquet support " + "and the parquet-hive jars from the parquet project should not be included " + "in Hive's classpath." ) ; } LOG . debug ( "Hive version {}, returning {}" , hiveVersion , Hive012Binding . class . getSimpleName ( ) ) ; // as of 11/26/2013 it looks like the 0.12 binding will work for 0.13
return Hive012Binding . class ;


signature:
public class ParquetOutputCommitter { /** * TODO : all the catching of Exceptions below - - see PARQUET - 383 */
public static void writeMetaDataFile ( Configuration configuration , Path outputPath ) { } }

implementation:
JobSummaryLevel level = ParquetOutputFormat . getJobSummaryLevel ( configuration ) ; if ( level == JobSummaryLevel . NONE ) { return ; } try { final FileSystem fileSystem = outputPath . getFileSystem ( configuration ) ; FileStatus outputStatus = fileSystem . getFileStatus ( outputPath ) ; List < Footer > footers ; switch ( level ) { case ALL : footers = ParquetFileReader . readAllFootersInParallel ( configuration , outputStatus , false ) ; // don ' t skip row groups
break ; case COMMON_ONLY : footers = ParquetFileReader . readAllFootersInParallel ( configuration , outputStatus , true ) ; // skip row groups
break ; default : throw new IllegalArgumentException ( "Unrecognized job summary level: " + level ) ; } // If there are no footers , _ metadata file cannot be written since there is no way to determine schema !
// Onus of writing any summary files lies with the caller in this case .
if ( footers . isEmpty ( ) ) { return ; } try { ParquetFileWriter . writeMetadataFile ( configuration , outputPath , footers , level ) ; } catch ( Exception e ) { LOG . warn ( "could not write summary file(s) for " + outputPath , e ) ; final Path metadataPath = new Path ( outputPath , ParquetFileWriter . PARQUET_METADATA_FILE ) ; try { if ( fileSystem . exists ( metadataPath ) ) { fileSystem . delete ( metadataPath , true ) ; } } catch ( Exception e2 ) { LOG . warn ( "could not delete metadata file" + outputPath , e2 ) ; } try { final Path commonMetadataPath = new Path ( outputPath , ParquetFileWriter . PARQUET_COMMON_METADATA_FILE ) ; if ( fileSystem . exists ( commonMetadataPath ) ) { fileSystem . delete ( commonMetadataPath , true ) ; } } catch ( Exception e2 ) { LOG . warn ( "could not delete metadata file" + outputPath , e2 ) ; } } } catch ( Exception e ) { LOG . warn ( "could not write summary file for " + outputPath , e ) ; }


signature:
public class BytePacker { /** * Compatibility API
 * @ param input the input bytes
 * @ param inPos where to read from in input
 * @ param output the output values
 * @ param outPos where to write to in output */
@ Deprecated public void unpack8Values ( final byte [ ] input , final int inPos , final int [ ] output , final int outPos ) { } }

implementation: unpack8Values ( ByteBuffer . wrap ( input ) , inPos , output , outPos ) ;


signature:
public class BytePacker { /** * Compatibility API
 * @ param input the input bytes
 * @ param inPos where to read from in input
 * @ param output the output values
 * @ param outPos where to write to in output */
@ Deprecated public void unpack32Values ( byte [ ] input , int inPos , int [ ] output , int outPos ) { } }

implementation: unpack32Values ( ByteBuffer . wrap ( input ) , inPos , output , outPos ) ;


signature:
public class SchemaConverter { /** * Creates a Parquet Schema from an Arrow one and returns the mapping
 * @ param arrowSchema the provided Arrow Schema
 * @ return the mapping between the 2 */
public SchemaMapping fromArrow ( Schema arrowSchema ) { } }

implementation: List < Field > fields = arrowSchema . getFields ( ) ; List < TypeMapping > parquetFields = fromArrow ( fields ) ; MessageType parquetType = addToBuilder ( parquetFields , Types . buildMessage ( ) ) . named ( "root" ) ; return new SchemaMapping ( arrowSchema , parquetType , parquetFields ) ;


signature:
public class SchemaConverter { /** * Creates an Arrow Schema from an Parquet one and returns the mapping
 * @ param parquetSchema the provided Parquet Schema
 * @ return the mapping between the 2 */
public SchemaMapping fromParquet ( MessageType parquetSchema ) { } }

implementation: List < Type > fields = parquetSchema . getFields ( ) ; List < TypeMapping > mappings = fromParquet ( fields ) ; List < Field > arrowFields = fields ( mappings ) ; return new SchemaMapping ( new Schema ( arrowFields ) , parquetSchema , mappings ) ;


signature:
public class SchemaConverter { /** * Maps a Parquet and Arrow Schema
 * For now does not validate primitive type compatibility
 * @ param arrowSchema an Arrow schema
 * @ param parquetSchema a Parquet message type
 * @ return the mapping between the 2 */
public SchemaMapping map ( Schema arrowSchema , MessageType parquetSchema ) { } }

implementation: List < TypeMapping > children = map ( arrowSchema . getFields ( ) , parquetSchema . getFields ( ) ) ; return new SchemaMapping ( arrowSchema , parquetSchema , children ) ;


signature:
public class ParquetMetadataConverter { /** * Visible for testing */
List < SchemaElement > toParquetSchema ( MessageType schema ) { } }

implementation: List < SchemaElement > result = new ArrayList < SchemaElement > ( ) ; addToList ( result , schema ) ; return result ;


signature:
public class ParquetMetadataConverter { /** * Visible for testing */
Set < org . apache . parquet . column . Encoding > fromFormatEncodings ( List < Encoding > encodings ) { } }

implementation:
Set < org . apache . parquet . column . Encoding > converted = new HashSet < org . apache . parquet . column . Encoding > ( ) ; for ( Encoding encoding : encodings ) { converted . add ( getEncoding ( encoding ) ) ; } // make converted unmodifiable , drop reference to modifiable copy
converted = Collections . unmodifiableSet ( converted ) ; // atomically update the cache
Set < org . apache . parquet . column . Encoding > cached = cachedEncodingSets . putIfAbsent ( converted , converted ) ; if ( cached == null ) { // cached = = null signifies that converted was * not * in the cache previously
// so we can return converted instead of throwing it away , it has now
// been cached
cached = converted ; } return cached ;


signature:
public class ParquetMetadataConverter { /** * Visible for testing */
static org . apache . parquet . column . statistics . Statistics fromParquetStatisticsInternal ( String createdBy , Statistics formatStats , PrimitiveType type , SortOrder typeSortOrder ) { } }

implementation:
// create stats object based on the column type
org . apache . parquet . column . statistics . Statistics . Builder statsBuilder = org . apache . parquet . column . statistics . Statistics . getBuilderForReading ( type ) ; if ( formatStats != null ) { // Use the new V2 min - max statistics over the former one if it is filled
if ( formatStats . isSetMin_value ( ) && formatStats . isSetMax_value ( ) ) { byte [ ] min = formatStats . min_value . array ( ) ; byte [ ] max = formatStats . max_value . array ( ) ; if ( isMinMaxStatsSupported ( type ) || Arrays . equals ( min , max ) ) { statsBuilder . withMin ( min ) ; statsBuilder . withMax ( max ) ; } } else { boolean isSet = formatStats . isSetMax ( ) && formatStats . isSetMin ( ) ; boolean maxEqualsMin = isSet ? Arrays . equals ( formatStats . getMin ( ) , formatStats . getMax ( ) ) : false ; boolean sortOrdersMatch = SortOrder . SIGNED == typeSortOrder ; // NOTE : See docs in CorruptStatistics for explanation of why this check is needed
// The sort order is checked to avoid returning min / max stats that are not
// valid with the type ' s sort order . In previous releases , all stats were
// aggregated using a signed byte - wise ordering , which isn ' t valid for all the
// types ( e . g . strings , decimals etc . ) .
if ( ! CorruptStatistics . shouldIgnoreStatistics ( createdBy , type . getPrimitiveTypeName ( ) ) && ( sortOrdersMatch || maxEqualsMin ) ) { if ( isSet ) { statsBuilder . withMin ( formatStats . min . array ( ) ) ; statsBuilder . withMax ( formatStats . max . array ( ) ) ; } } } if ( formatStats . isSetNull_count ( ) ) { statsBuilder . withNumNulls ( formatStats . null_count ) ; } } return statsBuilder . build ( ) ;


signature:
public class ParquetMetadataConverter { /** * Returns whether to use signed order min and max with a type . It is safe to
 * use signed min and max when the type is a string type and contains only
 * ASCII characters ( where the sign bit was 0 ) . This checks whether the type
 * is a string type and uses { @ code useSignedStringMinMax } to determine if
 * only ASCII characters were written .
 * @ param type a primitive type with a logical type annotation
 * @ return true if signed order min / max can be used with this type */
private boolean overrideSortOrderToSigned ( PrimitiveType type ) { } }

implementation:
// even if the override is set , only return stats for string - ish types
// a null type annotation is considered string - ish because some writers
// failed to use the UTF8 annotation .
LogicalTypeAnnotation annotation = type . getLogicalTypeAnnotation ( ) ; return useSignedStringMinMax && PrimitiveTypeName . BINARY == type . getPrimitiveTypeName ( ) && ( annotation == null || STRING_TYPES . contains ( annotation . getClass ( ) ) ) ;