signature:
public class IdentityMultimapList { /** * Adds a value to the set associated with the key . */
public boolean add ( K key , V value ) { } }

implementation: List < V > list = this . get ( key ) ; if ( list == null ) { list = new ArrayList < > ( ) ; this . put ( key , list ) ; } return list . add ( value ) ;


signature:
public class TriggerFuture { /** * This version also checks to see if we are done and we still need to call the trigger .
 * If so , it calls it .
 * @ see java . util . concurrent . Future # isDone ( ) */
@ Override public boolean isDone ( ) { } }

implementation: boolean done = this . raw . isDone ( ) ; if ( ! triggered && done ) { this . triggered = true ; PendingFutures . removePending ( this ) ; this . trigger ( ) ; } return done ;


signature:
public class IfConditionGenerator { /** * Clever enough to recognize that an empty set of conditions means Always . */
public If < ? , ? > [ ] generateIfConditions ( Class < ? extends If < ? , ? > > [ ] ifClasses , Field field ) { } }

implementation:
if ( ifClasses . length == 0 ) return ALWAYS ; If < ? , ? > [ ] result = new If < ? , ? > [ ifClasses . length ] ; for ( int i = 0 ; i < ifClasses . length ; i ++ ) { Class < ? extends If < ? , ? > > ifClass = ifClasses [ i ] ; result [ i ] = this . createIf ( ifClass , field ) ; // Sanity check the generic If class types to ensure that they match the actual types of the field & entity .
Type valueType = GenericTypeReflector . getTypeParameter ( ifClass , If . class . getTypeParameters ( ) [ 0 ] ) ; Class < ? > valueClass = GenericTypeReflector . erase ( valueType ) ; Type pojoType = GenericTypeReflector . getTypeParameter ( ifClass , If . class . getTypeParameters ( ) [ 1 ] ) ; Class < ? > pojoClass = GenericTypeReflector . erase ( pojoType ) ; if ( ! TypeUtils . isAssignableFrom ( valueClass , field . getType ( ) ) ) throw new IllegalStateException ( "Cannot use If class " + ifClass . getName ( ) + " on " + field + " because you cannot assign " + field . getType ( ) . getName ( ) + " to " + valueClass . getName ( ) ) ; if ( ! TypeUtils . isAssignableFrom ( pojoClass , field . getDeclaringClass ( ) ) ) throw new IllegalStateException ( "Cannot use If class " + ifClass . getName ( ) + " on " + field + " because the containing class " + field . getDeclaringClass ( ) . getName ( ) + " is not compatible with " + pojoClass . getName ( ) ) ; } return result ;


signature:
public class LoadContext { /** * Call this when a load process completes . Executes anything in the batch and then executes any delayed operations . */
public void done ( ) { } }

implementation:
engine . execute ( ) ; while ( deferred != null ) { final List < Runnable > runme = deferred ; deferred = null ; // reset this because it might get filled with more
for ( final Runnable run : runme ) { log . trace ( "Executing {}" , run ) ; run . run ( ) ; } }


signature:
public class LoadContext { /** * Create a Ref for the key , and maybe start a load operation depending on current load groups . */
public < T > Ref < T > loadRef ( Key < T > key , LoadConditions loadConditions ) { } }

implementation: return engine . makeRef ( currentRoot , loadConditions , key ) ;


signature:
public class LoadContext { /** * Delays an operation until the context is done ( ) . Typically this is for lifecycle methods . */
public void defer ( Runnable runnable ) { } }

implementation: if ( this . deferred == null ) this . deferred = new ArrayList < > ( ) ; log . trace ( "Deferring: {}" , runnable ) ; this . deferred . add ( runnable ) ;


signature:
public class LoadContext { /** * Get the container object which is appropriate for the specified property . Go up the chain looking for a compatible
 * type ; the first one found is the container . If nothing found , throw an exception . */
public Object getContainer ( Type containerType , Path path ) { } }

implementation:
Class < ? > containerClass = GenericTypeReflector . erase ( containerType ) ; Iterator < Object > containersIt = containers . descendingIterator ( ) ; // We have always entered the current ' this ' context when processing properties , so the first thing
// we get will always be ' this ' . So skip that and the first matching owner should be what we want .
containersIt . next ( ) ; while ( containersIt . hasNext ( ) ) { Object potentialContainer = containersIt . next ( ) ; if ( containerClass . isAssignableFrom ( potentialContainer . getClass ( ) ) ) return potentialContainer ; } throw new IllegalStateException ( "No container matching " + containerType + " in " + containers + " at path " + path ) ;


signature:
public class EntityMemcacheStats { /** * We ' re just tracking statistics so we don ' t really need to worry about these stepping on each other ;
 * if there ' s a hit or miss lost no big deal . */
private Stat getStat ( String kind ) { } }

implementation: Stat stat = this . stats . get ( kind ) ; if ( stat == null ) { stat = new Stat ( ) ; this . stats . put ( kind , stat ) ; } return stat ;


signature:
public class FutureHelper { /** * Quietly perform the get ( ) on a future */
public static < T > T quietGet ( Future < T > future ) { } }

implementation:
try { return future . get ( ) ; } catch ( Exception ex ) { unwrapAndThrow ( ex ) ; return null ; // just to make the compiler happy
}


signature:
public class FutureHelper { /** * Properly unwraps ExecutionException , throwing the relevant original cause . Otherwise
 * RuntimeExceptions get thrown and checked exceptions get wrapped in a RuntimeException . */
public static void unwrapAndThrow ( Throwable ex ) { } }

implementation: if ( ex instanceof RuntimeException ) throw ( RuntimeException ) ex ; else if ( ex instanceof Error ) throw ( Error ) ex ; else if ( ex instanceof ExecutionException ) unwrapAndThrow ( ex . getCause ( ) ) ; else throw new UndeclaredThrowableException ( ex ) ;


signature:
public class FloatTranslatorFactory { /** * Coerces the value to be a number of the specified type ; needed because
 * all numbers come back from the datastore as Long / Double and this screws up
 * any type that expects something smaller . We don ' t need to worry about primitive
 * types because we wrapped the class earlier . */
private Number coerceNumber ( final Number value , final Class < ? > type ) { } }

implementation: if ( type == Float . class ) return value . floatValue ( ) ; else if ( type == Double . class ) return value . doubleValue ( ) ; else throw new IllegalArgumentException ( ) ; // should be impossible


signature:
public class Translators { /** * Obtains the Translator appropriate for this type and annotations . May be a cached
 * translator ; if not , one will be discovered and cached . */
public < P , D > Translator < P , D > get ( final TypeKey tk , final CreateContext ctx , final Path path ) { } }

implementation:
Translator < ? , ? > translator = translators . get ( tk ) ; if ( translator == null ) { translator = create ( tk , ctx , path ) ; translators . put ( tk , translator ) ; } // noinspection unchecked
return ( Translator < P , D > ) translator ;


signature:
public class Translators { /** * Get the translator for a root entity class */
public < P > Translator < P , FullEntity < ? > > getRoot ( final Class < P > clazz ) { } }

implementation: return get ( new TypeKey ( clazz ) , new CreateContext ( fact ) , Path . root ( ) ) ;


signature:
public class Translators { /** * Create a translator from scratch by going through the discovery process . */
private Translator < ? , ? > create ( final TypeKey tk , final CreateContext ctx , final Path path ) { } }

implementation: for ( final TranslatorFactory < ? , ? > trans : this . translatorFactories ) { @ SuppressWarnings ( "unchecked" ) final Translator < ? , ? > soFar = trans . create ( tk , ctx , path ) ; if ( soFar != null ) return soFar ; } throw new IllegalArgumentException ( "Don't know how to translate " + tk . getType ( ) + " with annotations " + Arrays . toString ( tk . getAnnotations ( ) ) ) ;


signature:
public class PropertyPopulator { /** * Gets the appropriate value from the container and sets it on the appropriate field of the pojo . */
@ Override public void load ( final FullEntity < ? > container , final LoadContext ctx , final Path containerPath , final P intoPojo ) { } }

implementation:
try { if ( translator instanceof Recycles ) ctx . recycle ( property . get ( intoPojo ) ) ; final Value < D > value = ( translator instanceof Synthetic ) ? null : getPropertyFromContainer ( container , containerPath ) ; // will throw SkipException if property not present
setValue ( intoPojo , value , ctx , containerPath ) ; } catch ( SkipException ex ) { // Irrelevant
}


signature:
public class PropertyPopulator { /** * Gets the relevant property from the container , detecting alsoload collisions .
 * @ return the value obtained from the container
 * @ throws IllegalStateException if there are multiple alsoload name matches */
private Value < D > getPropertyFromContainer ( final FullEntity < ? > container , final Path containerPath ) { } }

implementation:
String foundName = null ; Value < D > value = null ; for ( String name : property . getLoadNames ( ) ) { if ( container . contains ( name ) ) { if ( foundName != null ) throw new IllegalStateException ( "Collision trying to load field; multiple name matches for '" + property . getName ( ) + "' at '" + containerPath . extend ( foundName ) + "' and '" + containerPath . extend ( name ) + "'" ) ; // noinspection unchecked
value = container . getValue ( name ) ; foundName = name ; } } if ( foundName == null ) throw new SkipException ( ) ; else return value ;


signature:
public class PropertyPopulator { /** * Set this raw datastore value on the relevant property of the pojo , doing whatever translations are necessary . */
public void setValue ( final Object pojo , final Value < D > value , final LoadContext ctx , final Path containerPath ) throws SkipException { } }

implementation: final Path propertyPath = containerPath . extend ( property . getName ( ) ) ; final P loaded = translator . load ( value , ctx , propertyPath ) ; setOnPojo ( pojo , loaded , ctx , propertyPath ) ;


signature:
public class PropertyPopulator { /** * Sets the property on the pojo to the value . The value should already be translated .
 * TODO : Sensitive to the value possibly being a Result < ? > wrapper , in which case it enqueues the set operation until the loadcontext is done . */
private void setOnPojo ( final Object pojo , final P value , final LoadContext ctx , final Path path ) { } }

implementation: if ( log . isTraceEnabled ( ) ) log . trace ( LogUtils . msg ( path , "Setting property " + property . getName ( ) + " to " + value ) ) ; property . set ( pojo , value ) ;


signature:
public class PropertyPopulator { /** * Gets the appropriate field value from the pojo and puts it in the container at the appropriate prop name
 * and with the appropriate indexing .
 * @ param onPojo is the parent pojo which holds the property we represent
 * @ param index is the default state of indexing up to this point
 * @ param containerPath is the path to the container ; each property will extend this path . */
@ Override public void save ( final P onPojo , boolean index , final SaveContext ctx , final Path containerPath , final FullEntity . Builder < ? > into ) { } }

implementation:
if ( property . isSaved ( onPojo ) ) { // Look for an override on indexing
final Boolean propertyIndexInstruction = property . getIndexInstruction ( onPojo ) ; if ( propertyIndexInstruction != null ) index = propertyIndexInstruction ; @ SuppressWarnings ( "unchecked" ) final P value = ( P ) property . get ( onPojo ) ; try { final Path propPath = containerPath . extend ( property . getName ( ) ) ; final Value < D > propValue = translator . save ( value , index , ctx , propPath ) ; into . set ( property . getName ( ) , propValue ) ; } catch ( SkipException ex ) { // No problem , do nothing
} }


signature:
public class PropertyPopulator { /** * Get the value for the property and translate it into datastore format . */
public Value < D > getValue ( final Object pojo , final SaveContext ctx , final Path containerPath ) { } }

implementation: @ SuppressWarnings ( "unchecked" ) final P value = ( P ) property . get ( pojo ) ; return translator . save ( value , false , ctx , containerPath . extend ( property . getName ( ) ) ) ;


signature:
public class AbstractOpenRtbJsonWriter { /** * Write any extensions that may exist in a message .
 * @ param msg A message that may contain extensions
 * @ param gen The JSON generator
 * @ param < EM > Type of message being serialized , containing extensions
 * @ throws IOException any serialization error */
@ SuppressWarnings ( "unchecked" ) protected final < EM extends ExtendableMessage < EM > > void writeExtensions ( EM msg , JsonGenerator gen ) throws IOException { } }

implementation: boolean openExt = false ; for ( Map . Entry < FieldDescriptor , Object > field : msg . getAllFields ( ) . entrySet ( ) ) { FieldDescriptor fd = field . getKey ( ) ; if ( fd . isExtension ( ) ) { if ( fd . isRepeated ( ) ) { List < Object > extValue = ( List < Object > ) field . getValue ( ) ; if ( ! extValue . isEmpty ( ) ) { OpenRtbJsonExtWriter < Object > extWriter = factory . getWriter ( msg . getClass ( ) , extValue . get ( 0 ) . getClass ( ) , fd . getName ( ) ) ; if ( extWriter != null ) { openExt = openExt ( gen , openExt ) ; extWriter . writeRepeated ( extValue , gen ) ; } } } else { Object extValue = field . getValue ( ) ; OpenRtbJsonExtWriter < Object > extWriter = factory . getWriter ( msg . getClass ( ) , extValue . getClass ( ) , fd . getName ( ) ) ; if ( extWriter != null ) { openExt = openExt ( gen , openExt ) ; extWriter . writeSingle ( extValue , gen ) ; } } } } if ( openExt ) { gen . writeEndObject ( ) ; }


signature:
public class AbstractOpenRtbJsonWriter { /** * Writes a string that represents a ContentCategory ' s JSON name , returning success status .
 * If the factory is in strict mode , the category name is validated . */
protected final boolean writeContentCategory ( String cat , JsonGenerator gen ) throws IOException { } }

implementation: if ( ! factory . isStrict ( ) || OpenRtbUtils . categoryFromName ( cat ) != null ) { gen . writeString ( cat ) ; return true ; } else { return false ; }


signature:
public class AbstractOpenRtbJsonWriter { /** * Writes an array of ContentCategory if not empty .
 * @ see # writeContentCategory ( String , JsonGenerator ) */
protected final void writeContentCategories ( String fieldName , List < String > cats , JsonGenerator gen ) throws IOException { } }

implementation: if ( ! cats . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( String cat : cats ) { writeContentCategory ( cat , gen ) ; } gen . writeEndArray ( ) ; }


signature:
public class SnippetProcessor { /** * Processes the raw snippet that was set by the bid , making any transformations necessary . */
public String process ( SnippetProcessorContext ctx , String snippet ) { } }

implementation: checkNotNull ( ctx ) ; StringBuilder sb = ctx . builder ( ) ; sb . setLength ( 0 ) ; String currSnippet = snippet ; boolean processedMacros = false ; int snippetPos = 0 ; int macroPos = currSnippet . indexOf ( "${" ) ; while ( macroPos != - 1 ) { sb . append ( currSnippet . substring ( snippetPos , macroPos ) ) ; int macroEnd = processMacroAt ( ctx , currSnippet , macroPos ) ; if ( macroEnd == - 1 ) { sb . append ( "${" ) ; snippetPos = macroPos + 2 ; } else { snippetPos = macroEnd ; processedMacros = true ; } macroPos = currSnippet . indexOf ( "${" , snippetPos ) ; } if ( processedMacros ) { sb . append ( currSnippet , snippetPos , currSnippet . length ( ) ) ; currSnippet = sb . toString ( ) ; } sb . setLength ( 0 ) ; String ret = urlEncode ( ctx , currSnippet ) ; sb . setLength ( 0 ) ; return ret ;


signature:
public class OpenRtbJsonWriter { /** * Serializes a { @ link BidRequest } to JSON , returned as a { @ code String } . */
public String writeBidRequest ( BidRequest req ) throws IOException { } }

implementation: try ( StringWriter writer = new StringWriter ( ) ) { writeBidRequest ( req , writer ) ; return writer . toString ( ) ; }


signature:
public class OpenRtbJsonWriter { /** * Serializes a { @ link BidRequest } to JSON , streamed into an { @ link Writer } .
 * @ see JsonFactory # createGenerator ( Writer ) */
public void writeBidRequest ( BidRequest req , Writer writer ) throws IOException { } }

implementation: JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( writer ) ; writeBidRequest ( req , gen ) ;


signature:
public class OpenRtbJsonWriter { /** * Serializes a { @ link BidRequest } to JSON , streamed into an { @ link OutputStream } .
 * @ see JsonFactory # createGenerator ( OutputStream ) */
public void writeBidRequest ( BidRequest req , OutputStream os ) throws IOException { } }

implementation: JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( os ) ; writeBidRequest ( req , gen ) ;


signature:
public class OpenRtbJsonWriter { /** * Serializes a { @ link BidRequest } to JSON , with a provided { @ link JsonGenerator }
 * which allows several choices of output and encoding . */
public final void writeBidRequest ( BidRequest req , JsonGenerator gen ) throws IOException { } }

implementation: gen . writeStartObject ( ) ; writeBidRequestFields ( req , gen ) ; writeExtensions ( req , gen ) ; gen . writeEndObject ( ) ; gen . flush ( ) ;


signature:
public class OpenRtbJsonWriter { /** * Serializes a { @ link BidResponse } to JSON , returned as a { @ link String } . */
public String writeBidResponse ( BidResponse resp ) throws IOException { } }

implementation: try ( StringWriter writer = new StringWriter ( ) ) { writeBidResponse ( resp , writer ) ; return writer . toString ( ) ; }


signature:
public class OpenRtbJsonWriter { /** * Serializes a { @ link BidResponse } to JSON , streamed to a { @ link OutputStream } .
 * @ see JsonFactory # createGenerator ( OutputStream ) */
public void writeBidResponse ( BidResponse resp , OutputStream os ) throws IOException { } }

implementation: JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( os ) ; writeBidResponse ( resp , gen ) ;


signature:
public class OpenRtbJsonWriter { /** * Serializes a { @ link BidResponse } to JSON , streamed to a { @ link Writer } .
 * @ see JsonFactory # createGenerator ( Writer ) */
public void writeBidResponse ( BidResponse resp , Writer writer ) throws IOException { } }

implementation: JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( writer ) ; writeBidResponse ( resp , gen ) ;


signature:
public class OpenRtbJsonWriter { /** * Serializes a { @ link BidResponse } to JSON , with a provided { @ link JsonGenerator }
 * which allows several choices of output and encoding . */
public final void writeBidResponse ( BidResponse resp , JsonGenerator gen ) throws IOException { } }

implementation: gen . writeStartObject ( ) ; writeBidResponseFields ( resp , gen ) ; writeExtensions ( resp , gen ) ; gen . writeEndObject ( ) ; gen . flush ( ) ;


signature:
public class OpenRtbUtils { /** * @ return The OpenRTB SeatBid with the specified ID ; will be created if not existent .
 * The ID should be present in the request ' s wseat .
 * @ see # seatBid ( com . google . openrtb . OpenRtb . BidResponse . Builder )
 * Use for the anonymous seat */
public static SeatBid . Builder seatBid ( BidResponse . Builder response , String seat ) { } }

implementation: checkNotNull ( seat ) ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { if ( seatbid . hasSeat ( ) && seat . equals ( seatbid . getSeat ( ) ) ) { return seatbid ; } } return response . addSeatbidBuilder ( ) . setSeat ( seat ) ;


signature:
public class OpenRtbUtils { /** * Iterates all bids .
 * @ return Read - only sequence of all bis in the response .
 * May have bids from multiple seats , grouped by seat */
public static Iterable < Bid . Builder > bids ( BidResponse . Builder response ) { } }

implementation: return new ResponseBidsIterator ( response , SEAT_ANY , null ) ;


signature:
public class OpenRtbUtils { /** * Iterates all bids from a specific seat .
 * @ param seatFilter Filter for seatFilter . You can use { @ code null } to select the anonymous seat ,
 * or { @ link # SEAT _ ANY } to not filter by seat
 * @ return View for the seat ' s internal sequence of bids ; or an empty , read - only
 * view if that seat doesn ' t exist . */
public static List < Bid . Builder > bids ( BidResponse . Builder response , @ Nullable String seatFilter ) { } }

implementation: for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { if ( filterSeat ( seatbid , seatFilter ) ) { return seatbid . getBidBuilderList ( ) ; } } return ImmutableList . of ( ) ;


signature:
public class OpenRtbUtils { /** * Finds a bid by ID .
 * @ param id Bid ID , assumed to be unique within the response
 * @ return Matching bid ' s builder , or { @ code null } if not found */
@ Nullable public static Bid . Builder bidWithId ( BidResponse . Builder response , String id ) { } }

implementation: checkNotNull ( id ) ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { for ( Bid . Builder bid : seatbid . getBidBuilderList ( ) ) { if ( id . equals ( bid . getId ( ) ) ) { return bid ; } } } return null ;


signature:
public class OpenRtbUtils { /** * Finds bids by a custom criteria .
 * @ param bidFilter Filter for bids
 * @ param seatFilter Filter for seat . You can use { @ code null } to select the anonymous seat ,
 * or { @ link # SEAT _ ANY } to not filter by seat
 * @ return Read - only sequence of bids that satisfy the filter .
 * May have bids from multiple seats , grouped by seat */
public static Stream < Bid . Builder > bidStreamWith ( BidResponse . Builder response , @ Nullable String seatFilter , @ Nullable Predicate < Bid . Builder > bidFilter ) { } }

implementation: return StreamSupport . stream ( new ResponseBidsIterator ( response , seatFilter , bidFilter ) . spliterator ( ) , false ) ;


signature:
public class OpenRtbUtils { /** * Finds bids by a custom criteria .
 * @ param bidFilter Filter for bids
 * @ param seatFilter Filter for seat . You can use { @ code null } to select the anonymous seat ,
 * or { @ link # SEAT _ ANY } to not filter by seat
 * @ return Sequence of all bids that satisfy the filter .
 * May have bids from multiple seats , grouped by seat */
public static Iterable < Bid . Builder > bidsWith ( BidResponse . Builder response , @ Nullable String seatFilter , @ Nullable Predicate < Bid . Builder > bidFilter ) { } }

implementation: return new ResponseBidsIterator ( response , seatFilter , bidFilter ) ;


signature:
public class OpenRtbUtils { /** * Updates bids , from all seats .
 * @ param updater Update function . The { @ code apply ( ) } method can decide or not to update each
 * object , and it ' s expected to return { @ code true } for objects that were updated
 * @ return { @ code true } if at least one bid was updated
 * @ see ProtoUtils # update ( Iterable , Function ) for more general updating support */
public static boolean updateBids ( BidResponse . Builder response , Function < Bid . Builder , Boolean > updater ) { } }

implementation: checkNotNull ( updater ) ; boolean updated = false ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { updated |= ProtoUtils . update ( seatbid . getBidBuilderList ( ) , updater ) ; } return updated ;


signature:
public class OpenRtbUtils { /** * Updates bids from a given seat .
 * @ param seatFilter Seat ID , or { @ code null } to select the anonymous seat
 * @ param updater Update function . The { @ code apply ( ) } method can decide or not to update each
 * object , and it ' s expected to return { @ code true } for objects that were updated
 * @ return { @ code true } if at least one bid was updated
 * @ see ProtoUtils # update ( Iterable , Function ) for more general updating support */
public static boolean updateBids ( BidResponse . Builder response , @ Nullable String seatFilter , Function < Bid . Builder , Boolean > updater ) { } }

implementation: checkNotNull ( updater ) ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { if ( filterSeat ( seatbid , seatFilter ) ) { return ProtoUtils . update ( seatbid . getBidBuilderList ( ) , updater ) ; } } return false ;


signature:
public class OpenRtbUtils { /** * Remove bids by bid .
 * @ param filter Returns { @ code true } to keep bid , { @ code false } to remove
 * @ return { @ code true } if any bid was removed
 * @ see ProtoUtils # filter ( List , Predicate ) for more general filtering support */
public static boolean removeBids ( BidResponse . Builder response , Predicate < Bid . Builder > filter ) { } }

implementation: checkNotNull ( filter ) ; boolean updated = false ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { updated |= removeBids ( seatbid , filter ) ; } return updated ;


signature:
public class OpenRtbUtils { /** * Remove bids by seat and bid .
 * @ param seatFilter Seat ID , or { @ code null } to select the anonymous seat
 * @ param bidFilter Returns { @ code true } to keep bid , { @ code false } to remove
 * @ return { @ code true } if any bid was removed
 * @ see ProtoUtils # filter ( List , Predicate ) for more general filtering support */
public static boolean removeBids ( BidResponse . Builder response , @ Nullable String seatFilter , Predicate < Bid . Builder > bidFilter ) { } }

implementation: checkNotNull ( bidFilter ) ; boolean updated = false ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { if ( filterSeat ( seatbid , seatFilter ) ) { updated |= removeBids ( seatbid , bidFilter ) ; } } return updated ;


signature:
public class OpenRtbUtils { /** * Finds an { @ link Imp } by ID .
 * @ return The { @ link Imp } s that has the given id , or { @ code null } if not found . */
@ Nullable public static Imp impWithId ( BidRequest request , String id ) { } }

implementation: checkNotNull ( id ) ; for ( Imp imp : request . getImpList ( ) ) { if ( imp . getId ( ) . equals ( id ) ) { return imp ; } } return null ;


signature:
public class OpenRtbUtils { /** * Find an { @ link Imp } by its ID and its { @ link Banner } ' s ID .
 * @ param impId Imp ID ; optional if the Banner IDs are unique within the request
 * @ param bannerId Banner ID
 * @ return The { @ link Imp } for a given impression ID x banner ID , or { @ code null } if not found */
@ Nullable public static Imp bannerImpWithId ( BidRequest request , @ Nullable String impId , String bannerId ) { } }

implementation: checkNotNull ( bannerId ) ; for ( Imp imp : request . getImpList ( ) ) { if ( ( impId == null || imp . getId ( ) . equals ( impId ) ) && imp . hasBanner ( ) && imp . getBanner ( ) . getId ( ) . equals ( bannerId ) ) { return imp ; } } return null ;


signature:
public class OpenRtbUtils { /** * Optimized code for most filtered lookups . This is worth the effort
 * because bidder code may invoke these lookup methods intensely ;
 * common cases like everything - filtered or nothing - filtered are very dominant ;
 * and simpler code previously used needed lots of temporary collections .
 * @ param request Container of impressions
 * @ param impFilter Filters impressions ; will be executed exactly once ,
 * and only for impressions that pass the banner / video type filters .
 * The constants { @ link # IMP _ NONE } and { @ link # IMP _ ALL } allow
 * more efficient execution when you want to filter none / all impressions .
 * @ return Immutable or unmodifiable view for the filtered impressions */
public static Iterable < Imp > impsWith ( BidRequest request , Predicate < Imp > impFilter ) { } }

implementation:
checkNotNull ( impFilter ) ; List < Imp > imps = request . getImpList ( ) ; if ( imps . isEmpty ( ) || impFilter == IMP_ALL ) { return imps ; } else if ( impFilter == IMP_NONE ) { return ImmutableList . of ( ) ; } boolean included = impFilter . test ( imps . get ( 0 ) ) ; int size = imps . size ( ) , i ; for ( i = 1 ; i < size ; ++ i ) { if ( impFilter . test ( imps . get ( i ) ) != included ) { break ; } } if ( i == size ) { return included ? imps // Unmodifiable , comes from protobuf
: ImmutableList . < Imp > of ( ) ; } int headingSize = i ; return new FluentIterable < Imp > ( ) { @ Override public Iterator < Imp > iterator ( ) { Iterator < Imp > unfiltered = imps . iterator ( ) ; return new AbstractIterator < Imp > ( ) { private int heading = 0 ; @ Override protected Imp computeNext ( ) { while ( unfiltered . hasNext ( ) ) { Imp imp = unfiltered . next ( ) ; if ( ( heading ++ < headingSize ) ? included : impFilter . test ( imp ) ) { return imp ; } } return endOfData ( ) ; } } ; } } ;


signature:
public class OpenRtbUtils { /** * Adds " impression type " subfilters to a base filter , to further restricts impressions
 * that contain a banner , video and / or native object .
 * @ param baseFilter base filter for impressions
 * @ param banner { @ code true } to include impressions with
 * { @ link com . google . openrtb . OpenRtb . BidRequest . Imp . Banner } s
 * @ param video { @ code true } to include impressions with
 * { @ link com . google . openrtb . OpenRtb . BidRequest . Imp . Video } s
 * @ param nativ { @ code true } to include impressions with
 * { @ link com . google . openrtb . OpenRtb . BidRequest . Imp . Native } s
 * @ return A filter in the form : { @ code baseFilter AND ( banner OR video OR native ) } */
public static Predicate < Imp > addFilters ( Predicate < Imp > baseFilter , boolean banner , boolean video , boolean nativ ) { } }

implementation: int orCount = ( banner ? 1 : 0 ) + ( video ? 1 : 0 ) + ( nativ ? 1 : 0 ) ; if ( baseFilter == IMP_NONE || orCount == 0 ) { return baseFilter ; } Predicate < Imp > typeFilter = null ; if ( banner ) { typeFilter = Imp :: hasBanner ; } if ( video ) { typeFilter = typeFilter == null ? Imp :: hasVideo : typeFilter . or ( Imp :: hasVideo ) ; } if ( nativ ) { typeFilter = typeFilter == null ? Imp :: hasNative : typeFilter . or ( Imp :: hasNative ) ; } return baseFilter == IMP_ALL ? typeFilter : baseFilter . and ( typeFilter ) ;


signature:
public class OpenRtbUtils { /** * Performs a filter by seat .
 * @ param seatbid Seat to filter
 * @ param seatFilter Filter for seat . You can use { @ code null } to select the anonymous seat ,
 * or { @ link # SEAT _ ANY } to not filter by seat
 * @ return { @ code true } if the seat passes the filter */
public static boolean filterSeat ( SeatBidOrBuilder seatbid , @ Nullable String seatFilter ) { } }

implementation: return seatFilter == null ? ! seatbid . hasSeat ( ) : seatFilter == SEAT_ANY || seatFilter . equals ( seatbid . getSeat ( ) ) ;


signature:
public class OpenRtbNativeJsonWriter { /** * Serializes a { @ link NativeRequest } to JSON , returned as a { @ code String } . */
public String writeNativeRequest ( NativeRequest req ) throws IOException { } }

implementation: try ( StringWriter writer = new StringWriter ( ) ) { writeNativeRequest ( req , writer ) ; return writer . toString ( ) ; }


signature:
public class OpenRtbNativeJsonWriter { /** * Serializes a { @ link NativeRequest } to JSON , streamed into an { @ link Writer } .
 * @ see JsonFactory # createGenerator ( Writer ) */
public void writeNativeRequest ( NativeRequest req , Writer writer ) throws IOException { } }

implementation: JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( writer ) ; writeNativeRequest ( req , gen ) ;


signature:
public class OpenRtbNativeJsonWriter { /** * Serializes a { @ link NativeRequest } to JSON , streamed into an { @ link OutputStream } .
 * @ see JsonFactory # createGenerator ( OutputStream ) */
public void writeNativeRequest ( NativeRequest req , OutputStream os ) throws IOException { } }

implementation: JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( os ) ; writeNativeRequest ( req , gen ) ;


signature:
public class OpenRtbNativeJsonWriter { /** * Serializes a { @ link NativeRequest } to JSON , with a provided { @ link JsonGenerator }
 * which allows several choices of output and encoding . */
public final void writeNativeRequest ( NativeRequest req , JsonGenerator gen ) throws IOException { } }

implementation: gen . writeStartObject ( ) ; if ( factory ( ) . isRootNativeField ( ) ) { gen . writeObjectFieldStart ( "native" ) ; } writeNativeRequestFields ( req , gen ) ; writeExtensions ( req , gen ) ; if ( factory ( ) . isRootNativeField ( ) ) { gen . writeEndObject ( ) ; } gen . writeEndObject ( ) ; gen . flush ( ) ;


signature:
public class OpenRtbNativeJsonWriter { /** * Serializes a { @ link NativeResponse } to JSON , returned as a { @ link String } . */
public String writeNativeResponse ( NativeResponse resp ) throws IOException { } }

implementation: try ( StringWriter writer = new StringWriter ( ) ) { writeNativeResponse ( resp , writer ) ; return writer . toString ( ) ; }


signature:
public class OpenRtbNativeJsonWriter { /** * Serializes a { @ link NativeResponse } to JSON , streamed to a { @ link OutputStream } .
 * @ see JsonFactory # createGenerator ( OutputStream ) */
public void writeNativeResponse ( NativeResponse resp , OutputStream os ) throws IOException { } }

implementation: JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( os ) ; writeNativeResponse ( resp , gen ) ;


signature:
public class OpenRtbNativeJsonWriter { /** * Serializes a { @ link NativeResponse } to JSON , streamed to a { @ link Writer } .
 * @ see JsonFactory # createGenerator ( Writer ) */
public void writeNativeResponse ( NativeResponse resp , Writer writer ) throws IOException { } }

implementation: JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( writer ) ; writeNativeResponse ( resp , gen ) ;


signature:
public class OpenRtbNativeJsonWriter { /** * Serializes a { @ link NativeResponse } to JSON , with a provided { @ link JsonGenerator }
 * which allows several choices of output and encoding . */
public final void writeNativeResponse ( NativeResponse resp , JsonGenerator gen ) throws IOException { } }

implementation: gen . writeStartObject ( ) ; if ( factory ( ) . isRootNativeField ( ) ) { gen . writeObjectFieldStart ( "native" ) ; } writeNativeResponseFields ( resp , gen ) ; writeExtensions ( resp , gen ) ; if ( factory ( ) . isRootNativeField ( ) ) { gen . writeEndObject ( ) ; } gen . writeEndObject ( ) ; gen . flush ( ) ;


signature:
public class AbstractOpenRtbJsonReader { /** * Read any extensions that may exist in a message .
 * @ param msg Builder of a message that may contain extensions
 * @ param par The JSON parser , positioned at the " ext " field
 * @ param < EB > Type of message builder being constructed
 * @ throws IOException any parsing error */
protected final < EB extends ExtendableBuilder < ? , EB > > void readExtensions ( EB msg , JsonParser par ) throws IOException { } }

implementation:
@ SuppressWarnings ( "unchecked" ) Set < OpenRtbJsonExtReader < EB > > extReaders = factory . getReaders ( ( Class < EB > ) msg . getClass ( ) ) ; if ( extReaders . isEmpty ( ) ) { par . skipChildren ( ) ; return ; } startObject ( par ) ; JsonToken tokLast = par . getCurrentToken ( ) ; JsonLocation locLast = par . getCurrentLocation ( ) ; while ( true ) { boolean extRead = false ; for ( OpenRtbJsonExtReader < EB > extReader : extReaders ) { if ( extReader . filter ( par ) ) { extReader . read ( msg , par ) ; JsonToken tokNew = par . getCurrentToken ( ) ; JsonLocation locNew = par . getCurrentLocation ( ) ; boolean advanced = tokNew != tokLast || ! locNew . equals ( locLast ) ; extRead |= advanced ; if ( ! endObject ( par ) ) { return ; } else if ( advanced && par . getCurrentToken ( ) != JsonToken . FIELD_NAME ) { tokLast = par . nextToken ( ) ; locLast = par . getCurrentLocation ( ) ; } else { tokLast = tokNew ; locLast = locNew ; } } } if ( ! endObject ( par ) ) { // Can ' t rely on this exit condition inside the for loop because no readers may filter .
return ; } if ( ! extRead ) { // No field was consumed by any reader , so we need to skip the field to make progress .
if ( logger . isDebugEnabled ( ) ) { logger . debug ( "Extension field not consumed by any reader, skipping: {} @{}:{}" , par . getCurrentName ( ) , locLast . getLineNr ( ) , locLast . getCharOffset ( ) ) ; } par . nextToken ( ) ; par . skipChildren ( ) ; tokLast = par . nextToken ( ) ; locLast = par . getCurrentLocation ( ) ; } // Else loop , try all readers again
}


signature:
public class AbstractOpenRtbJsonReader { /** * Special case for empty - string input . Returning null in non - @ Nullable method ,
 * but this is non - strict mode anyway . */
protected final boolean emptyToNull ( JsonParser par ) throws IOException { } }

implementation: JsonToken token = par . getCurrentToken ( ) ; if ( token == null ) { token = par . nextToken ( ) ; } return ! factory ( ) . isStrict ( ) && token == null ;


signature:
public class OpenRtbJsonFactory { /** * Register an extension reader .
 * @ param extReader code to desserialize some extension properties
 * @ param msgKlass class of extension message ' s builder , e . g . { @ code MyImp . Builder . class } */
public final < EB extends ExtendableBuilder < ? , EB > > OpenRtbJsonFactory register ( OpenRtbJsonExtReader < EB > extReader , Class < EB > msgKlass ) { } }

implementation: extReaders . put ( msgKlass . getName ( ) , extReader ) ; return this ;


signature:
public class OpenRtbJsonFactory { /** * Register an extension writer , bound to a specific field name . This writer will be
 * used in preference to a non - field - specific writer that may exist for the same class .
 * @ param extWriter code to serialize some { @ code extKlass } ' s properties
 * @ param extKlass class of extension message , e . g . { @ code MyImp . class }
 * @ param msgKlass class of container message , e . g . { @ code Imp . class }
 * @ param fieldName name of the field containing the extension
 * @ param < T > Type of value for the extension
 * @ see # register ( OpenRtbJsonExtWriter , Class , Class ) */
public final < T > OpenRtbJsonFactory register ( OpenRtbJsonExtWriter < T > extWriter , Class < T > extKlass , Class < ? extends Message > msgKlass , String fieldName ) { } }

implementation: Map < String , Map < String , OpenRtbJsonExtWriter < ? > > > mapMsg = extWriters . get ( msgKlass . getName ( ) ) ; if ( mapMsg == null ) { extWriters . put ( msgKlass . getName ( ) , mapMsg = new LinkedHashMap < > ( ) ) ; } Map < String , OpenRtbJsonExtWriter < ? > > mapKlass = mapMsg . get ( extKlass . getName ( ) ) ; if ( mapKlass == null ) { mapMsg . put ( extKlass . getName ( ) , mapKlass = new LinkedHashMap < > ( ) ) ; } mapKlass . put ( fieldName == null ? FIELDNAME_ALL : fieldName , extWriter ) ; return this ;


signature:
public class OpenRtbJsonFactory { /** * Register an extension writer , not bound to any a field name ( so this serializer
 * can be used for any extension of the provided class ) .
 * @ param extWriter code to serialize some { @ code extKlass } ' s properties
 * @ param extKlass class of extension message , e . g . { @ code MyImp . class }
 * @ param msgKlass class of container message , e . g . { @ code Imp . class }
 * @ param < T > Type of value for the extension
 * @ see # register ( OpenRtbJsonExtWriter , Class , Class , String ) */
public final < T > OpenRtbJsonFactory register ( OpenRtbJsonExtWriter < T > extWriter , Class < T > extKlass , Class < ? extends Message > msgKlass ) { } }

implementation: return register ( extWriter , extKlass , msgKlass , FIELDNAME_ALL ) ;


signature:
public class OpenRtbNativeJsonReader { /** * Desserializes a { @ link NativeRequest } from a JSON string , provided as a { @ link CharSequence } . */
public NativeRequest readNativeRequest ( CharSequence chars ) throws IOException { } }

implementation: return readNativeRequest ( CharSource . wrap ( chars ) . openStream ( ) ) ;


signature:
public class OpenRtbNativeJsonReader { /** * Desserializes a { @ link NativeRequest } from JSON , streamed from a { @ link Reader } . */
public NativeRequest readNativeRequest ( Reader reader ) throws IOException { } }

implementation: return ProtoUtils . built ( readNativeRequest ( factory ( ) . getJsonFactory ( ) . createParser ( reader ) ) ) ;


signature:
public class OpenRtbNativeJsonReader { /** * Desserializes a { @ link NativeRequest } from JSON , streamed from an { @ link InputStream } . */
public NativeRequest readNativeRequest ( InputStream is ) throws IOException { } }

implementation: try { return ProtoUtils . built ( readNativeRequest ( factory ( ) . getJsonFactory ( ) . createParser ( is ) ) ) ; } finally { Closeables . closeQuietly ( is ) ; }


signature:
public class OpenRtbNativeJsonReader { /** * Desserializes a { @ link NativeRequest } from JSON , with a provided { @ link JsonParser }
 * which allows several choices of input and encoding . */
public final NativeRequest . Builder readNativeRequest ( JsonParser par ) throws IOException { } }

implementation: if ( emptyToNull ( par ) ) { return null ; } NativeRequest . Builder req = NativeRequest . newBuilder ( ) ; boolean rootNativeField = false ; boolean firstField = true ; for ( startObject ( par ) ; endObject ( par ) ; par . nextToken ( ) ) { String fieldName = getCurrentName ( par ) ; if ( par . nextToken ( ) != JsonToken . VALUE_NULL ) { if ( firstField ) { firstField = false ; if ( ( rootNativeField = "native" . equals ( fieldName ) ) == true ) { startObject ( par ) ; fieldName = getCurrentName ( par ) ; par . nextToken ( ) ; } } if ( par . getCurrentToken ( ) != JsonToken . VALUE_NULL ) { readNativeRequestField ( par , req , fieldName ) ; } } } if ( rootNativeField && ! endObject ( par ) ) { par . nextToken ( ) ; } return req ;


signature:
public class OpenRtbNativeJsonReader { /** * Desserializes a { @ link NativeResponse } from a JSON string , provided as a { @ link CharSequence } . */
public NativeResponse readNativeResponse ( CharSequence chars ) throws IOException { } }

implementation: return readNativeResponse ( CharSource . wrap ( chars ) . openStream ( ) ) ;


signature:
public class OpenRtbNativeJsonReader { /** * Desserializes a { @ link NativeResponse } from JSON , streamed from a { @ link Reader } . */
public NativeResponse readNativeResponse ( Reader reader ) throws IOException { } }

implementation: return ProtoUtils . built ( readNativeResponse ( factory ( ) . getJsonFactory ( ) . createParser ( reader ) ) ) ;


signature:
public class OpenRtbNativeJsonReader { /** * Desserializes a { @ link NativeResponse } from JSON , streamed from an { @ link InputStream } . */
public NativeResponse readNativeResponse ( InputStream is ) throws IOException { } }

implementation: try { return ProtoUtils . built ( readNativeResponse ( factory ( ) . getJsonFactory ( ) . createParser ( is ) ) ) ; } finally { Closeables . closeQuietly ( is ) ; }


signature:
public class OpenRtbNativeJsonReader { /** * Desserializes a { @ link NativeResponse } from JSON , with a provided { @ link JsonParser }
 * which allows several choices of input and encoding . */
public final NativeResponse . Builder readNativeResponse ( JsonParser par ) throws IOException { } }

implementation: if ( emptyToNull ( par ) ) { return null ; } NativeResponse . Builder resp = NativeResponse . newBuilder ( ) ; boolean rootNativeField = false ; boolean firstField = true ; for ( startObject ( par ) ; endObject ( par ) ; par . nextToken ( ) ) { String fieldName = getCurrentName ( par ) ; if ( par . nextToken ( ) != JsonToken . VALUE_NULL ) { if ( firstField ) { firstField = false ; if ( ( rootNativeField = "native" . equals ( fieldName ) ) == true ) { startObject ( par ) ; fieldName = getCurrentName ( par ) ; par . nextToken ( ) ; } } if ( par . getCurrentToken ( ) != JsonToken . VALUE_NULL ) { readNativeResponseField ( par , resp , fieldName ) ; } } } if ( rootNativeField && ! endObject ( par ) ) { par . nextToken ( ) ; } return resp ;


signature:
public class OpenRtbSnippetProcessor { /** * Processes the context ' s response in - place , modifying properties that may contain macros . */
public void process ( SnippetProcessorContext bidCtx ) { } }

implementation: for ( SeatBid . Builder seat : bidCtx . response ( ) . getSeatbidBuilderList ( ) ) { for ( Bid . Builder bid : seat . getBidBuilderList ( ) ) { bidCtx . setBid ( bid ) ; processFields ( bidCtx ) ; } }


signature:
public class OpenRtbSnippetProcessor { /** * Processes all fields of a bid that should support macro expansion . */
protected void processFields ( SnippetProcessorContext bidCtx ) { } }

implementation:
Bid . Builder bid = bidCtx . getBid ( ) ; // Properties that can also be in the RHS of macros used by other properties .
if ( extendedFields ) { if ( bid . hasAdid ( ) ) { bid . setAdid ( process ( bidCtx , bid . getAdid ( ) ) ) ; } bid . setId ( process ( bidCtx , bid . getId ( ) ) ) ; } // Properties that are NOT the RHS of any macro .
if ( bid . hasAdm ( ) ) { bid . setAdm ( process ( bidCtx , bid . getAdm ( ) ) ) ; } if ( extendedFields ) { if ( bid . hasBurl ( ) ) { bid . setBurl ( process ( bidCtx , bid . getBurl ( ) ) ) ; } if ( bid . hasCid ( ) ) { bid . setCid ( process ( bidCtx , bid . getCid ( ) ) ) ; } if ( bid . hasCrid ( ) ) { bid . setCrid ( process ( bidCtx , bid . getCrid ( ) ) ) ; } if ( bid . hasDealid ( ) ) { bid . setDealid ( process ( bidCtx , bid . getDealid ( ) ) ) ; } bid . setImpid ( process ( bidCtx , bid . getImpid ( ) ) ) ; if ( bid . hasIurl ( ) ) { bid . setIurl ( process ( bidCtx , bid . getIurl ( ) ) ) ; } if ( bid . hasLurl ( ) ) { bid . setIurl ( process ( bidCtx , bid . getLurl ( ) ) ) ; } if ( bid . hasNurl ( ) ) { bid . setNurl ( process ( bidCtx , bid . getNurl ( ) ) ) ; } }


signature:
public class OpenRtbJsonReader { /** * Desserializes a { @ link BidRequest } from a JSON string , provided as a { @ link CharSequence } . */
public BidRequest readBidRequest ( CharSequence chars ) throws IOException { } }

implementation: return readBidRequest ( CharSource . wrap ( chars ) . openStream ( ) ) ;


signature:
public class OpenRtbJsonReader { /** * Desserializes a { @ link BidRequest } from JSON , streamed from a { @ link Reader } . */
public BidRequest readBidRequest ( Reader reader ) throws IOException { } }

implementation: return ProtoUtils . built ( readBidRequest ( factory ( ) . getJsonFactory ( ) . createParser ( reader ) ) ) ;


signature:
public class OpenRtbJsonReader { /** * Desserializes a { @ link BidRequest } from JSON , streamed from an { @ link InputStream } . */
public BidRequest readBidRequest ( InputStream is ) throws IOException { } }

implementation: try { return ProtoUtils . built ( readBidRequest ( factory ( ) . getJsonFactory ( ) . createParser ( is ) ) ) ; } finally { Closeables . closeQuietly ( is ) ; }


signature:
public class OpenRtbJsonReader { /** * Desserializes a { @ link BidRequest } from JSON , with a provided { @ link JsonParser }
 * which allows several choices of input and encoding . */
public final BidRequest . Builder readBidRequest ( JsonParser par ) throws IOException { } }

implementation: if ( emptyToNull ( par ) ) { return null ; } BidRequest . Builder req = BidRequest . newBuilder ( ) ; for ( startObject ( par ) ; endObject ( par ) ; par . nextToken ( ) ) { String fieldName = getCurrentName ( par ) ; if ( par . nextToken ( ) != JsonToken . VALUE_NULL ) { readBidRequestField ( par , req , fieldName ) ; } } return req ;


signature:
public class OpenRtbJsonReader { /** * Desserializes a { @ link BidResponse } from a JSON string , provided as a { @ link CharSequence } . */
public BidResponse readBidResponse ( CharSequence chars ) throws IOException { } }

implementation: return readBidResponse ( CharSource . wrap ( chars ) . openStream ( ) ) ;


signature:
public class OpenRtbJsonReader { /** * Desserializes a { @ link BidResponse } from JSON , streamed from a { @ link Reader } . */
public BidResponse readBidResponse ( Reader reader ) throws IOException { } }

implementation: return ProtoUtils . built ( readBidResponse ( factory ( ) . getJsonFactory ( ) . createParser ( reader ) ) ) ;


signature:
public class OpenRtbJsonReader { /** * Desserializes a { @ link BidResponse } from JSON , streamed from an { @ link InputStream } . */
public BidResponse readBidResponse ( InputStream is ) throws IOException { } }

implementation: try { return ProtoUtils . built ( readBidResponse ( factory ( ) . getJsonFactory ( ) . createParser ( is ) ) ) ; } finally { Closeables . closeQuietly ( is ) ; }


signature:
public class OpenRtbJsonReader { /** * Desserializes a { @ link BidResponse } from JSON , with a provided { @ link JsonParser }
 * which allows several choices of input and encoding . */
public final BidResponse . Builder readBidResponse ( JsonParser par ) throws IOException { } }

implementation: if ( emptyToNull ( par ) ) { return null ; } BidResponse . Builder resp = BidResponse . newBuilder ( ) ; for ( startObject ( par ) ; endObject ( par ) ; par . nextToken ( ) ) { String fieldName = getCurrentName ( par ) ; if ( par . nextToken ( ) != JsonToken . VALUE_NULL ) { readBidResponseField ( par , resp , fieldName ) ; } } return resp ;


signature:
public class ProtoUtils { /** * Given a message - or - builder , returns a message , invoking the builder if necessary . */
@ SuppressWarnings ( "unchecked" ) public static < I extends MessageLiteOrBuilder , O extends MessageLite > O built ( @ Nullable I msg ) { } }

implementation: return msg instanceof MessageLite . Builder ? ( O ) ( ( MessageLite . Builder ) msg ) . build ( ) : ( O ) msg ;


signature:
public class ProtoUtils { /** * Given a message - or - builder , return a builder , invoking toBuilder ( ) if necessary . */
@ SuppressWarnings ( "unchecked" ) public static < I extends MessageLiteOrBuilder , O extends MessageLite . Builder > O builder ( @ Nullable I msg ) { } }

implementation: return msg instanceof MessageLite ? ( O ) ( ( MessageLite ) msg ) . toBuilder ( ) : ( O ) msg ;


signature:
public class ProtoUtils { /** * Updates every builder from a sequence .
 * @ param objs List of builders to update
 * @ param updater Update function . The { @ code apply ( ) } method can decide or not to update each
 * object , and it ' s expected to return { @ code true } if some update was made
 * @ return { @ code true } if at least one object was updated */
public static < B extends MessageLite . Builder > boolean update ( Iterable < B > objs , Function < B , Boolean > updater ) { } }

implementation: checkNotNull ( updater ) ; boolean updated = false ; for ( B obj : objs ) { updated |= updater . apply ( obj ) ; } return updated ;


signature:
public class ProtoUtils { /** * Runs a filter through a sequence of objects .
 * @ param objs Message - or - builder objects
 * @ param filter Function that returns { @ code true } to retain an object , { @ code false } to discard
 * @ return Retained objects . If some elements are retained and others are discarded ,
 * this will be a new , mutable { @ link List } that contains only the retained elements .
 * If all elements are retained , returns the same , unmodified input sequence .
 * If all elements are discarded , returns an immutable , empty sequence */
public static < M extends MessageLiteOrBuilder > List < M > filter ( List < M > objs , Predicate < M > filter ) { } }

implementation:
checkNotNull ( filter ) ; for ( int i = 0 ; i < objs . size ( ) ; ++ i ) { if ( ! filter . test ( objs . get ( i ) ) ) { // At least one discarded object , go to slow - path .
return filterFrom ( objs , filter , i ) ; } } // Optimized common case : all items filtered , return the input sequence .
return objs ;


signature:
public class ProtoUtils { /** * Returns a copy of a { @ link Message } that contains only fields that pass a filter .
 * This will be executed recursively for fields which are child messages .
 * @ param msg Message object
 * @ param clearEmpty { @ code true } will cause { @ code null } to be returned if all fields from
 * { @ code msg } are removed ; { @ code false } will return an " empty " message in that case
 * @ param filter Function that returns { @ code true } to retain a field , { @ code false } to discard
 * @ return Message with the retained fieldsfrom { @ code msg } .
 * If some fields are retained and others discarded , returns a new message object .
 * If all fields are retained , returns the same { @ code msg } object .
 * If all fields are discarded , returns { @ code null } if { @ code clearEmpty = = true }
 * or a default instance of { @ code msg } ' s message type if { @ code clearEmpty = = false } */
@ Nullable public static < M extends Message > M filter ( M msg , boolean clearEmpty , Predicate < FieldDescriptor > filter ) { } }

implementation:
checkNotNull ( filter ) ; int i = 0 ; for ( Map . Entry < FieldDescriptor , Object > entry : msg . getAllFields ( ) . entrySet ( ) ) { FieldDescriptor fd = entry . getKey ( ) ; if ( ! filter . test ( fd ) ) { // At least one field discarded , go to slow - path .
return filterFrom ( msg , clearEmpty , filter , i , true ) ; } else if ( fd . getType ( ) == FieldDescriptor . Type . MESSAGE ) { // At least one field may have children , go to slow - path .
return filterFrom ( msg , clearEmpty , filter , i , false ) ; } ++ i ; } // Optimized common case : all items filtered , return the input sequence .
return msg ;


signature:
public class OpenRtbJsonUtils { /** * Returns the current field name , or empty string if none . */
public static String getCurrentName ( JsonParser par ) throws IOException { } }

implementation: String name = par . getCurrentName ( ) ; return name == null ? "" : name ;


signature:
public class OpenRtbJsonUtils { /** * Starts an Object , skipping the ' { ' token , and if necessary a field name before it . */
public static void startObject ( JsonParser par ) throws IOException { } }

implementation: JsonToken token = par . getCurrentToken ( ) ; if ( token == null || token == JsonToken . FIELD_NAME ) { token = par . nextToken ( ) ; } if ( token == JsonToken . START_OBJECT ) { par . nextToken ( ) ; } else { throw new JsonParseException ( par , "Expected start of object" ) ; }


signature:
public class OpenRtbJsonUtils { /** * Returns { @ code true } if the JSON Object is NOT finished .
 * Logic is inverted so this is used as a loop - end condition . */
public static boolean endObject ( JsonParser par ) { } }

implementation: JsonToken token = par . getCurrentToken ( ) ; return token != null && token != JsonToken . END_OBJECT ;


signature:
public class OpenRtbJsonUtils { /** * Starts an Array , skipping the ' [ ' token , and if necessary a field name before it . */
public static void startArray ( JsonParser par ) throws IOException { } }

implementation: JsonToken token = par . getCurrentToken ( ) ; if ( token == null || token == JsonToken . FIELD_NAME ) { token = par . nextToken ( ) ; } if ( token == JsonToken . START_ARRAY ) { par . nextToken ( ) ; } else { throw new JsonParseException ( par , "Expected start of array" ) ; }


signature:
public class OpenRtbJsonUtils { /** * Returns { @ code true } if the JSON Object is NOT finished .
 * Logic is inverted so this is used as a loop - end condition . */
public static boolean endArray ( JsonParser par ) { } }

implementation: JsonToken token = par . getCurrentToken ( ) ; return token != null && token != JsonToken . END_ARRAY ;


signature:
public class OpenRtbJsonUtils { /** * Skips a field name if necessary , returning the current token then . */
public static JsonToken peekToken ( JsonParser par ) throws IOException { } }

implementation: JsonToken token = par . getCurrentToken ( ) ; if ( token == null || token == JsonToken . FIELD_NAME ) { token = par . nextToken ( ) ; } return token ;


signature:
public class OpenRtbJsonUtils { /** * Skips a field name if necessary , returning the current token then , which must be
 * the start of an Array or Object : ' { ' or ' [ ' . */
public static JsonToken peekStructStart ( JsonParser par ) throws IOException { } }

implementation: JsonToken token = peekToken ( par ) ; if ( token . isStructStart ( ) ) { return token ; } else { throw new JsonParseException ( par , "Expected start of array or object" ) ; }


signature:
public class OpenRtbJsonUtils { /** * Writes a boolean as int , where false = 0 and true = 1. */
public static void writeIntBoolField ( String fieldName , boolean data , JsonGenerator gen ) throws IOException { } }

implementation: gen . writeNumberField ( fieldName , data ? 1 : 0 ) ;


signature:
public class OpenRtbJsonUtils { /** * Writes a string array if not empty . */
public static void writeStrings ( String fieldName , List < String > data , JsonGenerator gen ) throws IOException { } }

implementation: if ( ! data . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( String d : data ) { gen . writeString ( d ) ; } gen . writeEndArray ( ) ; }


signature:
public class OpenRtbJsonUtils { /** * Writes an int array if not empty . */
public static void writeInts ( String fieldName , List < Integer > data , JsonGenerator gen ) throws IOException { } }

implementation: if ( ! data . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( Integer d : data ) { gen . writeNumber ( d ) ; } gen . writeEndArray ( ) ; }


signature:
public class OpenRtbJsonUtils { /** * Writes a long , using quotes only if it ' s too big ( over 53 - bit mantissa ) . */
public static void writeLong ( long data , JsonGenerator gen ) throws IOException { } }

implementation: if ( data > MAX_JSON_INT || data < - MAX_JSON_INT ) { gen . writeString ( Long . toString ( data ) ) ; } else { gen . writeNumber ( data ) ; }


signature:
public class OpenRtbJsonUtils { /** * Writes a long , using quotes only if it ' s too big ( over 53 - bit mantissa ) . */
public static void writeLongField ( String fieldName , long data , JsonGenerator gen ) throws IOException { } }

implementation: gen . writeFieldName ( fieldName ) ; writeLong ( data , gen ) ;


signature:
public class OpenRtbJsonUtils { /** * Writes a long array if not empty , using quotes for values that are too big .
 * @ see # writeLong ( long , JsonGenerator ) */
public static void writeLongs ( String fieldName , List < Long > data , JsonGenerator gen ) throws IOException { } }

implementation: if ( ! data . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( long d : data ) { writeLong ( d , gen ) ; } gen . writeEndArray ( ) ; }


signature:
public class OpenRtbJsonUtils { /** * Writes a enum value as an int , using its Protobuf number . */
public static void writeEnum ( ProtocolMessageEnum e , JsonGenerator gen ) throws IOException { } }

implementation: gen . writeNumber ( e . getNumber ( ) ) ;


signature:
public class OpenRtbJsonUtils { /** * Writes a enum value as an int , using its Protobuf number . */
public static void writeEnumField ( String fieldName , ProtocolMessageEnum e , JsonGenerator gen ) throws IOException { } }

implementation: gen . writeNumberField ( fieldName , e . getNumber ( ) ) ;


signature:
public class OpenRtbJsonUtils { /** * Writes a enum array if not empty .
 * @ see # writeEnum ( ProtocolMessageEnum , JsonGenerator ) */
public static void writeEnums ( String fieldName , List < ? extends ProtocolMessageEnum > enums , JsonGenerator gen ) throws IOException { } }

implementation: if ( ! enums . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( ProtocolMessageEnum e : enums ) { writeEnum ( e , gen ) ; } gen . writeEndArray ( ) ; }


signature:
public class OpenRtbJsonUtils { /** * Reads from either a JSON Value String ( containing CSV ) or a JSON Array .
 * The dual input format is needed because some fields ( e . g . keywords ) were allowed
 * to be of either type in OpenRTB 2.2 ; now in 2.3 they are all CSV strings only .
 * TODO : Simplify this to only accept CSV strings after 2.2 compatibility is dropped . */
public static String readCsvString ( JsonParser par ) throws IOException { } }

implementation: JsonToken currentToken = par . getCurrentToken ( ) ; if ( currentToken == JsonToken . START_ARRAY ) { StringBuilder keywords = new StringBuilder ( ) ; for ( startArray ( par ) ; endArray ( par ) ; par . nextToken ( ) ) { if ( keywords . length ( ) != 0 ) { keywords . append ( ',' ) ; } keywords . append ( par . getText ( ) ) ; } return keywords . toString ( ) ; } else if ( currentToken == JsonToken . VALUE_STRING ) { return par . getText ( ) ; } else { throw new JsonParseException ( par , "Expected string or array" ) ; }


signature:
public class URLUtil { /** * Resolve relative URL - s and fix a few java . net . URL errors in handling of
 * URLs with embedded params and pure query targets .
 * @ param base
 * base url
 * @ param target
 * target url ( may be relative )
 * @ return resolved absolute url .
 * @ throws MalformedURLException */
public static URL resolveURL ( URL base , String target ) throws MalformedURLException { } }

implementation: target = target . trim ( ) ; if ( target . startsWith ( "?" ) ) { return fixPureQueryTargets ( base , target ) ; } return new URL ( base , target ) ;


signature:
public class URLUtil { /** * Handle the case in RFC3986 section 5.4.1 example 7 , and similar . */
static URL fixPureQueryTargets ( URL base , String target ) throws MalformedURLException { } }

implementation: if ( ! target . startsWith ( "?" ) ) return new URL ( base , target ) ; String basePath = base . getPath ( ) ; String baseRightMost = "" ; int baseRightMostIdx = basePath . lastIndexOf ( "/" ) ; if ( baseRightMostIdx != - 1 ) { baseRightMost = basePath . substring ( baseRightMostIdx + 1 ) ; } if ( target . startsWith ( "?" ) ) target = baseRightMost + target ; return new URL ( base , target ) ;


signature:
public class URLUtil { /** * Handles cases where the url param information is encoded into the base
 * url as opposed to the target .
 * If the taget contains params ( i . e . ' ; xxxx ' ) information then the target
 * params information is assumed to be correct and any base params
 * information is ignored . If the base contains params information but the
 * tareget does not , then the params information is moved to the target
 * allowing it to be correctly determined by the java . net . URL class .
 * @ param base
 * The base URL .
 * @ param target
 * The target path from the base URL .
 * @ return URL A URL with the params information correctly encoded .
 * @ throws MalformedURLException
 * If the url is not a well formed URL . */
private static URL fixEmbeddedParams ( URL base , String target ) throws MalformedURLException { } }

implementation:
// the target contains params information or the base doesn ' t then no
// conversion necessary , return regular URL
if ( target . indexOf ( ';' ) >= 0 || base . toString ( ) . indexOf ( ';' ) == - 1 ) { return new URL ( base , target ) ; } // get the base url and it params information
String baseURL = base . toString ( ) ; int startParams = baseURL . indexOf ( ';' ) ; String params = baseURL . substring ( startParams ) ; // if the target has a query string then put the params information
// after
// any path but before the query string , otherwise just append to the
// path
int startQS = target . indexOf ( '?' ) ; if ( startQS >= 0 ) { target = target . substring ( 0 , startQS ) + params + target . substring ( startQS ) ; } else { target += params ; } return new URL ( base , target ) ;


signature:
public class URLUtil { /** * Partitions of the hostname of the url by " . " */
public static String [ ] getHostSegments ( URL url ) { } }

implementation:
String host = url . getHost ( ) ; // return whole hostname , if it is an ipv4
// TODO : handle ipv6
if ( IP_PATTERN . matcher ( host ) . matches ( ) ) return new String [ ] { host } ; return host . split ( "\\." ) ;


signature:
public class URLUtil { /** * Returns the lowercased hostname for the url or null if the url is not
 * well formed .
 * @ param url
 * The url to check .
 * @ return String The hostname for the url . */
public static String getHost ( String url ) { } }

implementation: try { return new URL ( url ) . getHost ( ) . toLowerCase ( Locale . ROOT ) ; } catch ( MalformedURLException e ) { return null ; }


signature:
public class URLUtil { /** * Returns the page for the url . The page consists of the protocol , host ,
 * and path , but does not include the query string . The host is lowercased
 * but the path is not .
 * @ param url
 * The url to check .
 * @ return String The page for the url . */
public static String getPage ( String url ) { } }

implementation:
try { // get the full url , and replace the query string with and empty
// string
url = url . toLowerCase ( Locale . ROOT ) ; String queryStr = new URL ( url ) . getQuery ( ) ; return ( queryStr != null ) ? url . replace ( "?" + queryStr , "" ) : url ; } catch ( MalformedURLException e ) { return null ; }


signature:
public class ConfUtils { /** * Return one or more Strings regardless of whether they are represented as
 * a single String or a list in the config or an empty List if no value
 * could be found for that key . */
public static List < String > loadListFromConf ( String paramKey , Map stormConf ) { } }

implementation:
Object obj = stormConf . get ( paramKey ) ; List < String > list = new LinkedList < > ( ) ; if ( obj == null ) return list ; if ( obj instanceof PersistentVector ) { list . addAll ( ( PersistentVector ) obj ) ; } else { // single value ?
list . add ( obj . toString ( ) ) ; } return list ;


signature:
public class ConfUtils { /** * If the config consists of a single key ' config ' , its values are used
 * instead */
public static Map extractConfigElement ( Map conf ) { } }

implementation: if ( conf . size ( ) == 1 ) { Object confNode = conf . get ( "config" ) ; if ( confNode != null && confNode instanceof Map ) { conf = ( Map ) confNode ; } } return conf ;


signature:
public class ProtocolFactory { /** * Returns an instance of the protocol to use for a given URL */
public synchronized Protocol getProtocol ( URL url ) { } }

implementation:
// get the protocol
String protocol = url . getProtocol ( ) ; return cache . get ( protocol ) ;


signature:
public class StatusEmitterBolt { /** * Used for redirections or when discovering sitemap URLs . The custom key /
 * values are added to the target metadata post - filtering . */
protected void emitOutlink ( Tuple t , URL sURL , String newUrl , Metadata sourceMetadata , String ... customKeyVals ) { } }

implementation: Outlink ol = filterOutlink ( sURL , newUrl , sourceMetadata , customKeyVals ) ; if ( ol == null ) return ; collector . emit ( com . digitalpebble . stormcrawler . Constants . StatusStreamName , t , new Values ( ol . getTargetURL ( ) , ol . getMetadata ( ) , Status . DISCOVERED ) ) ;


signature:
public class WARCRecordFormat { /** * Generates a WARC info entry which can be stored at the beginning of each
 * WARC file . */
public static byte [ ] generateWARCInfo ( Map < String , String > fields ) { } }

implementation:
StringBuffer buffer = new StringBuffer ( ) ; buffer . append ( WARC_VERSION ) ; buffer . append ( CRLF ) ; buffer . append ( "WARC-Type: warcinfo" ) . append ( CRLF ) ; String mainID = UUID . randomUUID ( ) . toString ( ) ; // retrieve the date and filename from the map
String date = fields . get ( "WARC-Date" ) ; buffer . append ( "WARC-Date: " ) . append ( date ) . append ( CRLF ) ; String filename = fields . get ( "WARC-Filename" ) ; buffer . append ( "WARC-Filename: " ) . append ( filename ) . append ( CRLF ) ; buffer . append ( "WARC-Record-ID" ) . append ( ": " ) . append ( "<urn:uuid:" ) . append ( mainID ) . append ( ">" ) . append ( CRLF ) ; buffer . append ( "Content-Type" ) . append ( ": " ) . append ( "application/warc-fields" ) . append ( CRLF ) ; StringBuilder fieldsBuffer = new StringBuilder ( ) ; // add WARC fields
// http : / / bibnum . bnf . fr / warc / WARC _ ISO _ 28500 _ version1 _ latestdraft . pdf
Iterator < Entry < String , String > > iter = fields . entrySet ( ) . iterator ( ) ; while ( iter . hasNext ( ) ) { Entry < String , String > entry = iter . next ( ) ; String key = entry . getKey ( ) ; if ( key . startsWith ( "WARC-" ) ) continue ; fieldsBuffer . append ( key ) . append ( ": " ) . append ( entry . getValue ( ) ) . append ( CRLF ) ; } buffer . append ( "Content-Length" ) . append ( ": " ) . append ( fieldsBuffer . toString ( ) . getBytes ( StandardCharsets . UTF_8 ) . length ) . append ( CRLF ) ; buffer . append ( CRLF ) ; buffer . append ( fieldsBuffer . toString ( ) ) ; buffer . append ( CRLF ) ; buffer . append ( CRLF ) ; return buffer . toString ( ) . getBytes ( StandardCharsets . UTF_8 ) ;


signature:
public class WARCRecordFormat { /** * Modify verbatim HTTP response headers : remove or replace headers
 * < code > Content - Length < / code > , < code > Content - Encoding < / code > and
 * < code > Transfer - Encoding < / code > which may confuse WARC readers . Ensure
 * that the header end with a single empty line ( < code > \ r \ n \ r \ n < / code > ) .
 * @ param headers
 * HTTP 1.1 or 1.0 response header string , CR - LF - separated lines ,
 * first line is status line
 * @ return safe HTTP response header */
public static String fixHttpHeaders ( String headers , int contentLength ) { } }

implementation:
int start = 0 , lineEnd = 0 , last = 0 , trailingCrLf = 0 ; StringBuilder replace = new StringBuilder ( ) ; while ( start < headers . length ( ) ) { lineEnd = headers . indexOf ( CRLF , start ) ; trailingCrLf = 1 ; if ( lineEnd == - 1 ) { lineEnd = headers . length ( ) ; trailingCrLf = 0 ; } int colonPos = - 1 ; for ( int i = start ; i < lineEnd ; i ++ ) { if ( headers . charAt ( i ) == ':' ) { colonPos = i ; break ; } } if ( colonPos == - 1 ) { boolean valid = true ; if ( start == 0 ) { // status line ( without colon )
} else if ( ( lineEnd + 4 ) == headers . length ( ) && headers . endsWith ( CRLF + CRLF ) ) { // ok , trailing empty line
trailingCrLf = 2 ; } else if ( start == lineEnd ) { // skip / remove empty line
valid = false ; } else { LOG . warn ( "Invalid header line: {}" , headers . substring ( start , lineEnd ) ) ; valid = false ; } if ( ! valid ) { if ( last < start ) { replace . append ( headers . substring ( last , start ) ) ; } last = lineEnd + 2 * trailingCrLf ; } start = lineEnd + 2 * trailingCrLf ; /* * skip over invalid header line , no further check for
 * problematic headers required */
continue ; } String name = headers . substring ( start , colonPos ) ; if ( PROBLEMATIC_HEADERS . matcher ( name ) . matches ( ) ) { boolean needsFix = true ; if ( name . equalsIgnoreCase ( "content-length" ) ) { String value = headers . substring ( colonPos + 1 , lineEnd ) . trim ( ) ; try { int l = Integer . parseInt ( value ) ; if ( l == contentLength ) { needsFix = false ; } } catch ( NumberFormatException e ) { // needs to be fixed
} } if ( needsFix ) { if ( last < start ) { replace . append ( headers . substring ( last , start ) ) ; } last = lineEnd + 2 * trailingCrLf ; replace . append ( X_HIDE_HEADER ) . append ( headers . substring ( start , lineEnd + 2 * trailingCrLf ) ) ; if ( trailingCrLf == 0 ) { replace . append ( CRLF ) ; trailingCrLf = 1 ; } if ( name . equalsIgnoreCase ( "content-length" ) ) { // add effective uncompressed and unchunked length of
// content
replace . append ( "Content-Length" ) . append ( ": " ) . append ( contentLength ) . append ( CRLF ) ; } } } start = lineEnd + 2 * trailingCrLf ; } if ( last > 0 || trailingCrLf != 2 ) { if ( last < headers . length ( ) ) { // append trailing headers
replace . append ( headers . substring ( last ) ) ; } while ( trailingCrLf < 2 ) { replace . append ( CRLF ) ; trailingCrLf ++ ; } return replace . toString ( ) ; } return headers ;


signature:
public class WARCRecordFormat { /** * Get the actual fetch time from metadata and format it as required by the
 * WARC - Date field . If no fetch time is found in metadata ( key
 * { @ link REQUEST _ TIME _ KEY } ) , the current time is taken . */
protected static String getCaptureTime ( Metadata metadata ) { } }

implementation: String captureTimeMillis = metadata . getFirstValue ( REQUEST_TIME_KEY ) ; Instant capturedAt = Instant . now ( ) ; if ( captureTimeMillis != null ) { try { long millis = Long . parseLong ( captureTimeMillis ) ; capturedAt = Instant . ofEpochMilli ( millis ) ; } catch ( NumberFormatException | DateTimeException e ) { LOG . warn ( "Failed to parse capture time:" , e ) ; } } return WARC_DF . format ( capturedAt ) ;


signature:
public class RefreshTag { /** * Returns a normalised value of the content attribute for the refresh tag */
public static String extractRefreshURL ( String value ) { } }

implementation:
if ( StringUtils . isBlank ( value ) ) return null ; // 0 ; URL = http : / / www . apollocolors . com / site
try { if ( matcher . reset ( value ) . matches ( ) ) { return matcher . group ( 1 ) ; } } catch ( Exception e ) { } return null ;


signature:
public class MetadataTransfer { /** * Determine which metadata should be transfered to an outlink . Adds
 * additional metadata like the URL path . */
public Metadata getMetaForOutlink ( String targetURL , String sourceURL , Metadata parentMD ) { } }

implementation:
Metadata md = _filter ( parentMD , mdToTransfer ) ; // keep the path ?
if ( trackPath ) { md . addValue ( urlPathKeyName , sourceURL ) ; } // track depth
if ( trackDepth ) { String existingDepth = md . getFirstValue ( depthKeyName ) ; int depth ; try { depth = Integer . parseInt ( existingDepth ) ; } catch ( Exception e ) { depth = 0 ; } md . setValue ( depthKeyName , Integer . toString ( ++ depth ) ) ; } return md ;


signature:
public class MetadataTransfer { /** * Determine which metadata should be persisted for a given document
 * including those which are not necessarily transferred to the outlinks */
public Metadata filter ( Metadata metadata ) { } }

implementation:
Metadata filtered_md = _filter ( metadata , mdToTransfer ) ; // add the features that are only persisted but
// not transfered like _ _ redirTo _
filtered_md . putAll ( _filter ( metadata , mdToPersistOnly ) ) ; return filtered_md ;


signature:
public class RegexURLFilterBase { /** * Populates a List of Rules off of JsonNode . */
private List < RegexRule > readRules ( ArrayNode rulesList ) { } }

implementation: List < RegexRule > rules = new ArrayList < > ( ) ; for ( JsonNode urlFilterNode : rulesList ) { try { RegexRule rule = createRule ( urlFilterNode . asText ( ) ) ; if ( rule != null ) { rules . add ( rule ) ; } } catch ( IOException e ) { LOG . error ( "There was an error reading regex filter {}" , urlFilterNode . asText ( ) , e ) ; } } return rules ;


signature:
public class RegexURLFilterBase { /** * - - - - - * < implementation : URLFilter > * */
@ Override public String filter ( URL pageUrl , Metadata sourceMetadata , String url ) { } }

implementation: for ( RegexRule rule : rules ) { if ( rule . match ( url ) ) { return rule . accept ( ) ? url : null ; } } return null ;


signature:
public class ParseResult { /** * Add the key value to the metadata object for a given URL * */
public void put ( String URL , String key , String value ) { } }

implementation: get ( URL ) . getMetadata ( ) . addValue ( key , value ) ;


signature:
public class MemorySpout { /** * Add a new URL
 * @ param nextFetch */
public static void add ( String url , Metadata md , Date nextFetch ) { } }

implementation: LOG . debug ( "Adding {} with md {} and nextFetch {}" , url , md , nextFetch ) ; ScheduledURL tuple = new ScheduledURL ( url , md , nextFetch ) ; synchronized ( queue ) { queue . add ( tuple ) ; }


signature:
public class CloudSearchUtils { /** * Returns a normalised doc ID based on the URL of a document * */
public static String getID ( String url ) { } }

implementation:
// the document needs an ID
// see
// http : / / docs . aws . amazon . com / cloudsearch / latest / developerguide / preparing - data . html # creating - document - batches
// A unique ID for the document . A document ID can contain any
// letter or number and the following characters : _ - = # ; : / ? @
// & . Document IDs must be at least 1 and no more than 128
// characters long .
byte [ ] dig = digester . digest ( url . getBytes ( StandardCharsets . UTF_8 ) ) ; String ID = Hex . encodeHexString ( dig ) ; // is that even possible ?
if ( ID . length ( ) > 128 ) { throw new RuntimeException ( "ID larger than max 128 chars" ) ; } return ID ;


signature:
public class CloudSearchUtils { /** * Remove the non - cloudSearch - legal characters . Note that this might convert
 * two fields to the same name .
 * @ see < a
 * href = " http : / / docs . aws . amazon . com / cloudsearch / latest / developerguide / configuring - index - fields . html " >
 * configuring - index - fields . html < / a >
 * @ param name
 * @ return */
public static String cleanFieldName ( String name ) { } }

implementation: String lowercase = name . toLowerCase ( ) ; lowercase = lowercase . replaceAll ( "[^a-z_0-9]" , "_" ) ; if ( lowercase . length ( ) < 3 || lowercase . length ( ) > 64 ) throw new RuntimeException ( "Field name must be between 3 and 64 chars : " + lowercase ) ; if ( lowercase . equals ( "score" ) ) throw new RuntimeException ( "Field name must be score" ) ; return lowercase ;


signature:
public class ElasticSearchConnection { /** * Creates a connection with a default listener . The values for bolt type
 * are [ indexer , status , metrics ] */
public static ElasticSearchConnection getConnection ( Map stormConf , String boltType ) { } }

implementation: BulkProcessor . Listener listener = new BulkProcessor . Listener ( ) { @ Override public void afterBulk ( long arg0 , BulkRequest arg1 , BulkResponse arg2 ) { } @ Override public void afterBulk ( long arg0 , BulkRequest arg1 , Throwable arg2 ) { } @ Override public void beforeBulk ( long arg0 , BulkRequest arg1 ) { } } ; return getConnection ( stormConf , boltType , listener ) ;


signature:
public class CharsetIdentification { /** * Identifies the charset of a document based on the following logic : guess
 * from the ByteOrderMark - else if the same charset is specified in the
 * http headers and the html metadata then use it - otherwise use ICU ' s
 * charset detector to make an educated guess and if that fails too returns
 * UTF - 8. */
public static String getCharset ( Metadata metadata , byte [ ] content , int maxLengthCharsetDetection ) { } }

implementation:
// let ' s look at the BOM first
String BOMCharset = getCharsetFromBOM ( content ) ; if ( BOMCharset != null ) { return BOMCharset ; } // then look at what we get from HTTP headers and HTML content
String httpCharset = getCharsetFromHTTP ( metadata ) ; String htmlCharset = getCharsetFromMeta ( content , maxLengthCharsetDetection ) ; // both exist and agree
if ( httpCharset != null && htmlCharset != null && httpCharset . equalsIgnoreCase ( htmlCharset ) ) { return httpCharset ; } // let ' s guess from the text - using a hint or not
String hintCharset = null ; if ( httpCharset != null && htmlCharset == null ) { hintCharset = httpCharset ; } else if ( httpCharset == null && htmlCharset != null ) { hintCharset = htmlCharset ; } String textCharset = getCharsetFromText ( content , hintCharset , maxLengthCharsetDetection ) ; if ( textCharset != null ) { return textCharset ; } // return the default charset
return DEFAULT_CHARSET . name ( ) ;


signature:
public class CharsetIdentification { /** * Detects any BOMs and returns the corresponding charset */
private static String getCharsetFromBOM ( final byte [ ] byteData ) { } }

implementation: BOMInputStream bomIn = new BOMInputStream ( new ByteArrayInputStream ( byteData ) ) ; try { ByteOrderMark bom = bomIn . getBOM ( ) ; if ( bom != null ) { return bom . getCharsetName ( ) ; } } catch ( IOException e ) { return null ; } return null ;


signature:
public class CharsetIdentification { /** * Use a third party library as last resort to guess the charset from the
 * bytes . */
private static String getCharsetFromText ( byte [ ] content , String declaredCharset , int maxLengthCharsetDetection ) { } }

implementation:
String charset = null ; // filter HTML tags
CharsetDetector charsetDetector = new CharsetDetector ( ) ; charsetDetector . enableInputFilter ( true ) ; // give it a hint
if ( declaredCharset != null ) charsetDetector . setDeclaredEncoding ( declaredCharset ) ; // trim the content of the text for the detection
byte [ ] subContent = content ; if ( maxLengthCharsetDetection != - 1 && content . length > maxLengthCharsetDetection ) { subContent = Arrays . copyOfRange ( content , 0 , maxLengthCharsetDetection ) ; } charsetDetector . setText ( subContent ) ; try { CharsetMatch charsetMatch = charsetDetector . detect ( ) ; charset = validateCharset ( charsetMatch . getName ( ) ) ; } catch ( Exception e ) { charset = null ; } return charset ;


signature:
public class CharsetIdentification { /** * Attempt to find a META tag in the HTML that hints at the character set
 * used to write the document . */
private static String getCharsetFromMeta ( byte buffer [ ] , int maxlength ) { } }

implementation:
// convert to UTF - 8 String - - which hopefully will not mess up the
// characters we ' re interested in . . .
int len = buffer . length ; if ( maxlength > 0 && maxlength < len ) { len = maxlength ; } String html = new String ( buffer , 0 , len , DEFAULT_CHARSET ) ; Document doc = Parser . htmlParser ( ) . parseInput ( html , "dummy" ) ; // look for < meta http - equiv = " Content - Type "
// content = " text / html ; charset = gb2312 " > or HTML5 < meta charset = " gb2312 " >
Elements metaElements = doc . select ( "meta[http-equiv=content-type], meta[charset]" ) ; String foundCharset = null ; for ( Element meta : metaElements ) { if ( meta . hasAttr ( "http-equiv" ) ) foundCharset = getCharsetFromContentType ( meta . attr ( "content" ) ) ; if ( foundCharset == null && meta . hasAttr ( "charset" ) ) foundCharset = meta . attr ( "charset" ) ; if ( foundCharset != null ) return foundCharset ; } return foundCharset ;


signature:
public class SiteMapParserBolt { /** * Examines the first bytes of the content for a clue of whether this
 * document is a sitemap , based on namespaces . Works for XML and
 * non - compressed documents only . */
private final boolean sniff ( byte [ ] content ) { } }

implementation: byte [ ] beginning = content ; if ( content . length > maxOffsetGuess && maxOffsetGuess > 0 ) { beginning = Arrays . copyOfRange ( content , 0 , maxOffsetGuess ) ; } int position = Bytes . indexOf ( beginning , clue ) ; if ( position != - 1 ) { return true ; } return false ;


signature:
public class URLPartitioner { /** * Returns the host , domain , IP of a URL so that it can be partitioned for
 * politeness , depending on the value of the config
 * < i > partition . url . mode < / i > . */
public String getPartition ( String url , Metadata metadata ) { } }

implementation:
String partitionKey = null ; String host = "" ; // IP in metadata ?
if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_IP ) ) { String ip_provided = metadata . getFirstValue ( "ip" ) ; if ( StringUtils . isNotBlank ( ip_provided ) ) { partitionKey = ip_provided ; } } if ( partitionKey == null ) { URL u ; try { u = new URL ( url ) ; host = u . getHost ( ) ; } catch ( MalformedURLException e1 ) { LOG . warn ( "Invalid URL: {}" , url ) ; return null ; } } // partition by hostname
if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_HOST ) ) partitionKey = host ; // partition by domain : needs fixing
else if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_DOMAIN ) ) { partitionKey = PaidLevelDomain . getPLD ( host ) ; } // partition by IP
if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_IP ) && partitionKey == null ) { try { long start = System . currentTimeMillis ( ) ; final InetAddress addr = InetAddress . getByName ( host ) ; partitionKey = addr . getHostAddress ( ) ; long end = System . currentTimeMillis ( ) ; LOG . debug ( "Resolved IP {} in {} msec for : {}" , partitionKey , end - start , url ) ; } catch ( final Exception e ) { LOG . warn ( "Unable to resolve IP for: {}" , host ) ; return null ; } } LOG . debug ( "Partition Key for: {} > {}" , url , partitionKey ) ; return partitionKey ;


signature:
public class Metadata { /** * Set the value for a given key . The value can be null . */
public void setValue ( String key , String value ) { } }

implementation: md . put ( key , new String [ ] { value } ) ;


signature:
public class Metadata { /** * Returns the first non empty value found for the keys or null if none
 * found . */
public static String getFirstValue ( Metadata md , String ... keys ) { } }

implementation: for ( String key : keys ) { String val = md . getFirstValue ( key ) ; if ( StringUtils . isBlank ( val ) ) continue ; return val ; } return null ;


signature:
public class SeleniumProtocol { /** * Returns the first available driver * */
private final RemoteWebDriver getDriver ( ) { } }

implementation: try { return drivers . take ( ) ; } catch ( InterruptedException e ) { Thread . currentThread ( ) . interrupt ( ) ; } return null ;


signature:
public class AbstractHttpProtocol { /** * Called by extensions of this class * */
protected static void main ( AbstractHttpProtocol protocol , String args [ ] ) throws Exception { } }

implementation:
Config conf = new Config ( ) ; // loads the default configuration file
Map defaultSCConfig = Utils . findAndReadConfigFile ( "crawler-default.yaml" , false ) ; conf . putAll ( ConfUtils . extractConfigElement ( defaultSCConfig ) ) ; Options options = new Options ( ) ; options . addOption ( "c" , true , "configuration file" ) ; CommandLineParser parser = new DefaultParser ( ) ; CommandLine cmd = parser . parse ( options , args ) ; if ( cmd . hasOption ( "c" ) ) { String confFile = cmd . getOptionValue ( "c" ) ; ConfUtils . loadConf ( confFile , conf ) ; } protocol . configure ( conf ) ; Set < Runnable > threads = new HashSet < > ( ) ; class Fetchable implements Runnable { String url ; Metadata md ; Fetchable ( String line ) { StringTabScheme scheme = new StringTabScheme ( ) ; List < Object > tuple = scheme . deserialize ( ByteBuffer . wrap ( line . getBytes ( StandardCharsets . UTF_8 ) ) ) ; this . url = ( String ) tuple . get ( 0 ) ; this . md = ( Metadata ) tuple . get ( 1 ) ; } public void run ( ) { StringBuilder stringB = new StringBuilder ( ) ; stringB . append ( url ) . append ( "\n" ) ; if ( ! protocol . skipRobots ) { BaseRobotRules rules = protocol . getRobotRules ( url ) ; stringB . append ( "robots allowed: " ) . append ( rules . isAllowed ( url ) ) . append ( "\n" ) ; if ( rules instanceof RobotRules ) { stringB . append ( "robots requests: " ) . append ( ( ( RobotRules ) rules ) . getContentLengthFetched ( ) . length ) . append ( "\n" ) ; } stringB . append ( "sitemaps identified: " ) . append ( rules . getSitemaps ( ) . size ( ) ) . append ( "\n" ) ; } long start = System . currentTimeMillis ( ) ; ProtocolResponse response ; try { response = protocol . getProtocolOutput ( url , md ) ; stringB . append ( response . getMetadata ( ) ) . append ( "\n" ) ; stringB . append ( "status code: " ) . append ( response . getStatusCode ( ) ) . append ( "\n" ) ; stringB . append ( "content length: " ) . append ( response . getContent ( ) . length ) . append ( "\n" ) ; long timeFetching = System . currentTimeMillis ( ) - start ; stringB . append ( "fetched in : " ) . append ( timeFetching ) . append ( " msec" ) ; System . out . println ( stringB ) ; } catch ( Exception e ) { e . printStackTrace ( ) ; } finally { threads . remove ( this ) ; } } } for ( String arg : cmd . getArgs ( ) ) { Fetchable p = new Fetchable ( arg ) ; threads . add ( p ) ; new Thread ( p ) . start ( ) ; } while ( threads . size ( ) > 0 ) { Thread . sleep ( 1000 ) ; } protocol . cleanup ( ) ; System . exit ( 0 ) ;


signature:
public class CookieConverter { /** * Get a list of cookies based on the cookies string taken from response
 * header and the target url .
 * @ param cookiesString
 * the value of the http header for " Cookie " in the http
 * response .
 * @ param targetURL
 * the url for which we wish to pass the cookies in the request .
 * @ return List off cookies to add to the request . */
public static List < Cookie > getCookies ( String [ ] cookiesStrings , URL targetURL ) { } }

implementation:
ArrayList < Cookie > list = new ArrayList < Cookie > ( ) ; for ( String cs : cookiesStrings ) { String name = null ; String value = null ; String expires = null ; String domain = null ; String path = null ; boolean secure = false ; String [ ] tokens = cs . split ( ";" ) ; int equals = tokens [ 0 ] . indexOf ( "=" ) ; name = tokens [ 0 ] . substring ( 0 , equals ) ; value = tokens [ 0 ] . substring ( equals + 1 ) ; for ( int i = 1 ; i < tokens . length ; i ++ ) { String ti = tokens [ i ] . trim ( ) ; if ( ti . equalsIgnoreCase ( "secure" ) ) secure = true ; if ( ti . toLowerCase ( ) . startsWith ( "path=" ) ) { path = ti . substring ( 5 ) ; } if ( ti . toLowerCase ( ) . startsWith ( "domain=" ) ) { domain = ti . substring ( 7 ) ; } if ( ti . toLowerCase ( ) . startsWith ( "expires=" ) ) { expires = ti . substring ( 8 ) ; } } BasicClientCookie cookie = new BasicClientCookie ( name , value ) ; // check domain
if ( domain != null ) { cookie . setDomain ( domain ) ; if ( ! checkDomainMatchToUrl ( domain , targetURL . getHost ( ) ) ) continue ; } // check path
if ( path != null ) { cookie . setPath ( path ) ; if ( ! path . equals ( "" ) && ! path . equals ( "/" ) && ! targetURL . getPath ( ) . startsWith ( path ) ) continue ; } // check secure
if ( secure ) { cookie . setSecure ( secure ) ; if ( ! targetURL . getProtocol ( ) . equalsIgnoreCase ( "https" ) ) continue ; } // check expiration
if ( expires != null ) { try { Date expirationDate = DATE_FORMAT . parse ( expires ) ; cookie . setExpiryDate ( expirationDate ) ; // check that it hasn ' t expired ?
if ( cookie . isExpired ( new Date ( ) ) ) continue ; cookie . setExpiryDate ( expirationDate ) ; } catch ( ParseException e ) { // ignore exceptions
} } // attach additional infos to cookie
list . add ( cookie ) ; } return list ;


signature:
public class CookieConverter { /** * Helper method to check if url matches a cookie domain .
 * @ param cookieDomain
 * the domain in the cookie
 * @ param urlHostName
 * the host name of the url
 * @ return does the cookie match the host name */
public static boolean checkDomainMatchToUrl ( String cookieDomain , String urlHostName ) { } }

implementation: try { if ( cookieDomain . startsWith ( "." ) ) { cookieDomain = cookieDomain . substring ( 1 ) ; } String [ ] domainTokens = cookieDomain . split ( "\\." ) ; String [ ] hostTokens = urlHostName . split ( "\\." ) ; int tokenDif = hostTokens . length - domainTokens . length ; if ( tokenDif < 0 ) { return false ; } for ( int i = domainTokens . length - 1 ; i >= 0 ; i -- ) { if ( ! domainTokens [ i ] . equalsIgnoreCase ( hostTokens [ i + tokenDif ] ) ) { return false ; } } return true ; } catch ( Exception e ) { return true ; }


signature:
public class HttpRobotRulesParser { /** * Compose unique key to store and access robot rules in cache for given URL */
protected static String getCacheKey ( URL url ) { } }

implementation:
String protocol = url . getProtocol ( ) . toLowerCase ( Locale . ROOT ) ; String host = url . getHost ( ) . toLowerCase ( Locale . ROOT ) ; int port = url . getPort ( ) ; if ( port == - 1 ) { port = url . getDefaultPort ( ) ; } /* * Robot rules apply only to host , protocol , and port where robots . txt
 * is hosted ( cf . NUTCH - 1752 ) . Consequently */
return protocol + ":" + host + ":" + port ;


signature:
public class HttpRobotRulesParser { /** * Returns the robots rules from the cache or empty rules if not found
 * @ see com . digitalpebble . stormcrawler . filtering . robots . RobotsFilter */
public BaseRobotRules getRobotRulesSetFromCache ( URL url ) { } }

implementation: String cacheKey = getCacheKey ( url ) ; BaseRobotRules robotRules = CACHE . getIfPresent ( cacheKey ) ; if ( robotRules != null ) { return robotRules ; } return EMPTY_RULES ;


signature:
public class HttpRobotRulesParser { /** * Get the rules from robots . txt which applies for the given { @ code url } .
 * Robot rules are cached for a unique combination of host , protocol , and
 * port . If no rules are found in the cache , a HTTP request is send to fetch
 * { { protocol : / / host : port / robots . txt } } . The robots . txt is then parsed and
 * the rules are cached to avoid re - fetching and re - parsing it again .
 * @ param http
 * The { @ link Protocol } object
 * @ param url
 * URL robots . txt applies to
 * @ return { @ link BaseRobotRules } holding the rules from robots . txt */
@ Override public BaseRobotRules getRobotRulesSet ( Protocol http , URL url ) { } }

implementation:
String cacheKey = getCacheKey ( url ) ; // check in the error cache first
BaseRobotRules robotRules = ERRORCACHE . getIfPresent ( cacheKey ) ; if ( robotRules != null ) { return robotRules ; } // now try the proper cache
robotRules = CACHE . getIfPresent ( cacheKey ) ; if ( robotRules != null ) { return robotRules ; } boolean cacheRule = true ; URL redir = null ; String keyredir = null ; LOG . debug ( "Cache miss {} for {}" , cacheKey , url ) ; List < Integer > bytesFetched = new LinkedList < > ( ) ; try { ProtocolResponse response = http . getProtocolOutput ( new URL ( url , "/robots.txt" ) . toString ( ) , Metadata . empty ) ; int code = response . getStatusCode ( ) ; bytesFetched . add ( response . getContent ( ) != null ? response . getContent ( ) . length : 0 ) ; // try one level of redirection ?
if ( code == 301 || code == 302 || code == 307 || code == 308 ) { String redirection = response . getMetadata ( ) . getFirstValue ( HttpHeaders . LOCATION ) ; if ( StringUtils . isNotBlank ( redirection ) ) { if ( ! redirection . startsWith ( "http" ) ) { // RFC says it should be absolute , but apparently it
// isn ' t
redir = new URL ( url , redirection ) ; } else { redir = new URL ( redirection ) ; } // try from the cache
keyredir = getCacheKey ( redir ) ; if ( cacheKey . equalsIgnoreCase ( keyredir ) ) { keyredir = null ; } else { RobotRules cachedRediRobotRules = CACHE . getIfPresent ( keyredir ) ; if ( cachedRediRobotRules != null ) { // cache also for the redirected host
// but only if the robots . txt file is at the root
if ( redir . getPath ( ) . equals ( "/robots.txt" ) ) { LOG . debug ( "Caching robots for {} under key {} in cache" , redir , keyredir ) ; CACHE . put ( keyredir , cachedRediRobotRules ) ; } return cachedRediRobotRules ; } } response = http . getProtocolOutput ( redir . toString ( ) , Metadata . empty ) ; code = response . getStatusCode ( ) ; bytesFetched . add ( response . getContent ( ) != null ? response . getContent ( ) . length : 0 ) ; } } if ( code == 200 ) // found rules : parse them
{ String ct = response . getMetadata ( ) . getFirstValue ( HttpHeaders . CONTENT_TYPE ) ; robotRules = parseRules ( url . toString ( ) , response . getContent ( ) , ct , agentNames ) ; } else if ( ( code == 403 ) && ( ! allowForbidden ) ) { robotRules = FORBID_ALL_RULES ; // use forbid all
} else if ( code >= 500 ) { cacheRule = false ; robotRules = EMPTY_RULES ; } else robotRules = EMPTY_RULES ; // use default rules
} catch ( Throwable t ) { LOG . info ( "Couldn't get robots.txt for {} : {}" , url , t . toString ( ) ) ; cacheRule = false ; robotRules = EMPTY_RULES ; } Cache < String , RobotRules > cacheToUse = CACHE ; String cacheName = "success" ; if ( ! cacheRule ) { cacheToUse = ERRORCACHE ; cacheName = "error" ; } RobotRules cached = new RobotRules ( robotRules ) ; LOG . debug ( "Caching robots for {} under key {} in cache {}" , url , cacheKey , cacheName ) ; cacheToUse . put ( cacheKey , cached ) ; // cache robot for redirections
// get here only if the target has not been found in the cache
if ( keyredir != null ) { // cache also for the redirected host
// but only if the robots . txt file is at the root
if ( redir . getPath ( ) . equals ( "/robots.txt" ) ) { LOG . debug ( "Caching robots for {} under key {} in cache {}" , redir , keyredir , cacheName ) ; cacheToUse . put ( keyredir , cached ) ; } } RobotRules live = new RobotRules ( robotRules ) ; live . setContentLengthFetched ( Ints . toArray ( bytesFetched ) ) ; return live ;


signature:
public class RobotsTags { /** * called by the parser bolts */
public void extractMetaTags ( DocumentFragment doc ) throws XPathExpressionException { } }

implementation:
NodeList nodes = ( NodeList ) expression . evaluate ( doc , XPathConstants . NODESET ) ; if ( nodes == null ) return ; int numNodes = nodes . getLength ( ) ; for ( int i = 0 ; i < numNodes ; i ++ ) { Node n = ( Node ) nodes . item ( i ) ; // iterate on the attributes
// and check that it has name = robots and content
// whatever the case is
boolean isRobots = false ; String content = null ; NamedNodeMap attrs = n . getAttributes ( ) ; for ( int att = 0 ; att < attrs . getLength ( ) ; att ++ ) { Node keyval = attrs . item ( att ) ; if ( "name" . equalsIgnoreCase ( keyval . getNodeName ( ) ) && "robots" . equalsIgnoreCase ( keyval . getNodeValue ( ) ) ) { isRobots = true ; continue ; } if ( "content" . equalsIgnoreCase ( keyval . getNodeName ( ) ) ) { content = keyval . getNodeValue ( ) ; continue ; } } if ( isRobots && content != null ) { // got a value - split it
String [ ] vals = content . split ( " *, *" ) ; parseValues ( vals ) ; return ; } }


signature:
public class RobotsTags { /** * Extracts meta tags based on the value of the content attribute * */
public void extractMetaTags ( String content ) { } }

implementation: if ( content == null ) return ; String [ ] vals = content . split ( " *, *" ) ; parseValues ( vals ) ;


signature:
public class RobotsTags { /** * Adds a normalised representation of the directives in the metadata * */
public void normaliseToMetadata ( Metadata metadata ) { } }

implementation: metadata . setValue ( ROBOTS_NO_INDEX , Boolean . toString ( noIndex ) ) ; metadata . setValue ( ROBOTS_NO_CACHE , Boolean . toString ( noCache ) ) ; metadata . setValue ( ROBOTS_NO_FOLLOW , Boolean . toString ( noFollow ) ) ;


signature:
public class AbstractStatusUpdaterBolt { /** * Must be called by extending classes to store and collect in one go */
protected final void ack ( Tuple t , String url ) { } }

implementation:
// keep the URL in the cache
if ( useCache ) { cache . put ( url , "" ) ; } _collector . ack ( t ) ;