



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__snake_case						:Dict      		=     logging.getLogger(__name__)





def      __snake_case	(			):
    __a						       =				argparse.ArgumentParser(
        description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''				)
    parser.add_argument(
        '''--dataset_name'''   , type=_UpperCAmelCase   , default='''wikitext'''   , help='''Name of the training. Explore datasets at: hf.co/datasets.'''   , )
    parser.add_argument(
        '''--dataset_config'''   , type=_UpperCAmelCase   , default='''wikitext-103-raw-v1'''   , help='''Configuration name of the dataset.'''				)
    parser.add_argument(
        '''--tokenizer_name_or_path'''   , type=_UpperCAmelCase   , default='''sayakpaul/unigram-tokenizer-wikitext'''   , help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''   , )
    parser.add_argument(
        '''--shard_size'''   , type=_UpperCAmelCase   , default=1000   , help='''Number of entries to go in a single shard.'''   , )
    parser.add_argument('''--split'''   , type=_UpperCAmelCase   , default='''train'''   , choices=['''train''', '''test''', '''validation''']				)
    parser.add_argument(
        '''--limit'''   , default=_UpperCAmelCase   , type=_UpperCAmelCase   , help='''Limit the number of shards (used for debugging).'''   , )
    parser.add_argument(
        '''--max_length'''   , type=_UpperCAmelCase   , default=512   , help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
        ''' sequence length that is a multiple of 8.'''   , )
    parser.add_argument(
        '''--output_dir'''   , default='''tf-tpu'''   , type=_UpperCAmelCase   , help='''Output directory where the TFRecord shards will be saved. If the'''
        ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
        ''' shards will be directly saved to a Google Cloud Storage bucket.'''   , )

    __a						       =				parser.parse_args()
    return args





def      __snake_case	(			_UpperCAmelCase				):
    def fn(_UpperCAmelCase				):
        return tokenizer(examples['''text''']				)

    return fn





def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				[]
    for i in range(len(tokenized_data['''input_ids''']				)				):
        __a						       =				{
            '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]				)				),
            '''attention_mask''': tf.train.Feature(
                intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]				)				),
        }
        __a						       =				tf.train.Features(feature=_UpperCAmelCase				)
        __a						       =				tf.train.Example(features=_UpperCAmelCase				)
        __a						       =				example.SerializeToString()
        records.append(_UpperCAmelCase				)
    return records





def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				datasets.load_dataset(args.dataset_name   , args.dataset_config   , split=args.split				)

    if args.limit is not None:
        __a						       =				min(len(_UpperCAmelCase				)   , args.limit				)
        __a						       =				dataset.select(range(_UpperCAmelCase				)				)
        print(f'Limiting the dataset to {args.limit} entries.'				)

    __a						       =				AutoTokenizer.from_pretrained(args.tokenizer_name_or_path				)

    # Handle output directory creation.
    # For serializing into a Google Cloud Storage Bucket, one needs to first
    # create a bucket.
    if "gs" not in args.output_dir:
        if not os.path.exists(args.output_dir				):
            os.makedirs(args.output_dir				)
        __a						       =				os.path.join(args.output_dir   , args.split				)
        if not os.path.exists(_UpperCAmelCase				):
            os.makedirs(_UpperCAmelCase				)
    else:
        __a						       =				os.path.join(args.output_dir   , args.split				)

    # Tokenize the whole dataset at once.
    __a						       =				tokenize_function(_UpperCAmelCase				)
    __a						       =				dataset.map(_UpperCAmelCase   , batched=_UpperCAmelCase   , num_proc=4   , remove_columns=['''text''']				)

    # We need to concatenate all our texts together, and then split the result
    # into chunks of a fixed size, which we will call block_size. To do this, we
    # will use the map method again, with the option batched=True. When we use batched=True,
    # the function we pass to map() will be passed multiple inputs at once, allowing us
    # to group them into more or fewer examples than we had in the input.
    # This allows us to create our new fixed-length samples. The advantage of this
    # method is that we don't lose a whole lot of content from the dataset compared to the
    # case where we simply tokenize with a pre-defined max_length.

    def group_texts(_UpperCAmelCase				):
        # Concatenate all texts.
        __a						       =				{k: sum(examples[k]   , []				) for k in examples.keys()}
        __a						       =				len(concatenated_examples[list(examples.keys()				)[0]]				)
        # We drop the small remainder, though you could add padding instead if the model supports it
        # In this, as in all things, we advise you to follow your heart 🫀
        __a						       =				(total_length // args.max_length) * args.max_length
        # Split by chunks of max_len.
        __a						       =				{
            k: [t[i : i + args.max_length] for i in range(0   , _UpperCAmelCase   , args.max_length				)]
            for k, t in concatenated_examples.items()
        }
        return result

    __a						       =				dataset_tokenized.map(_UpperCAmelCase   , batched=_UpperCAmelCase   , batch_size=1000   , num_proc=4				)

    __a						       =				0
    __a						       =				0
    for shard in range(0   , len(_UpperCAmelCase				)   , args.shard_size				):
        __a						       =				grouped_dataset[shard : shard + args.shard_size]
        __a						       =				len(dataset_snapshot['''input_ids''']				)
        __a						       =				os.path.join(_UpperCAmelCase   , f'dataset-{shard_count}-{records_containing}.tfrecord'				)
        __a						       =				get_serialized_examples(_UpperCAmelCase				)

        with tf.io.TFRecordWriter(_UpperCAmelCase				) as out_file:
            for i in range(len(_UpperCAmelCase				)				):
                __a						       =				serialized_examples[i]
                out_file.write(_UpperCAmelCase				)
            print('''Wrote file {} containing {} records'''.format(_UpperCAmelCase   , _UpperCAmelCase				)				)

        shard_count += 1
        total_records += records_containing

    with open(f'split-{args.split}-records-count.txt'   , '''w'''				) as f:
        print(f'Total {args.split} records: {total_records}'   , file=_UpperCAmelCase				)


if __name__ == "__main__":
 __snake_case						:List[str]      		=     parse_args()
 main(args)










import argparse

from transformers import BigBirdConfig, BigBirdForPreTraining, BigBirdForQuestionAnswering, load_tf_weights_in_big_bird
from transformers.utils import logging


logging.set_verbosity_info()





def      __snake_case	(			_UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase				):
    # Initialise PyTorch model
    __a						       =				BigBirdConfig.from_json_file(_UpperCAmelCase				)
    print(f'Building PyTorch model from configuration: {config}'				)

    if is_trivia_qa:
        __a						       =				BigBirdForQuestionAnswering(_UpperCAmelCase				)
    else:
        __a						       =				BigBirdForPreTraining(_UpperCAmelCase				)

    # Load weights from tf checkpoint
    load_tf_weights_in_big_bird(_UpperCAmelCase   , _UpperCAmelCase   , is_trivia_qa=_UpperCAmelCase				)

    # Save pytorch-model
    print(f'Save PyTorch model to {pytorch_dump_path}'				)
    model.save_pretrained(_UpperCAmelCase				)


if __name__ == "__main__":
 __snake_case						:Tuple      		=     argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
 )
 parser.add_argument(
     '''--big_bird_config_file''',
     default=None,
     type=str,
     required=True,
     help=(
         '''The config json file corresponding to the pre-trained BERT model. \n'''
         '''This specifies the model architecture.'''
     ),
 )
 parser.add_argument(
     '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
 )
 parser.add_argument(
     '''--is_trivia_qa''', action='''store_true''', help='''Whether to convert a model with a trivia_qa head.'''
 )
 __snake_case						:Any      		=     parser.parse_args()
 convert_tf_checkpoint_to_pytorch(
     args.tf_checkpoint_path, args.big_bird_config_file, args.pytorch_dump_path, args.is_trivia_qa
 )





