






'''simple docstring'''







import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_A	:			int         =logging.getLogger(__name__)





def  SCREAMING_SNAKE_CASE_       ()      ->       List[Any]:
							lowerCamelCase__					:	str         =       argparse.ArgumentParser(
							    description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""      )
							parser.add_argument(
							    """--dataset_name"""				,			type=UpperCamelCase				,			default="""wikitext"""				,			help="""Name of the training. Explore datasets at: hf.co/datasets."""				,			)
							parser.add_argument(
							    """--dataset_config"""				,			type=UpperCamelCase				,			default="""wikitext-103-raw-v1"""				,			help="""Configuration name of the dataset."""      )
							parser.add_argument(
							    """--tokenizer_name_or_path"""				,			type=UpperCamelCase				,			default="""sayakpaul/unigram-tokenizer-wikitext"""				,			help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""				,			)
							parser.add_argument(
							    """--shard_size"""				,			type=UpperCamelCase				,			default=1000				,			help="""Number of entries to go in a single shard."""				,			)
							parser.add_argument("""--split"""				,			type=UpperCamelCase				,			default="""train"""				,			choices=["""train""", """test""", """validation"""]      )
							parser.add_argument(
							    """--limit"""				,			default=UpperCamelCase				,			type=UpperCamelCase				,			help="""Limit the number of shards (used for debugging)."""				,			)
							parser.add_argument(
							    """--max_length"""				,			type=UpperCamelCase				,			default=512				,			help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
							    """ sequence length that is a multiple of 8."""				,			)
							parser.add_argument(
							    """--output_dir"""				,			default="""tf-tpu"""				,			type=UpperCamelCase				,			help="""Output directory where the TFRecord shards will be saved. If the"""
							    """ path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"""
							    """ shards will be directly saved to a Google Cloud Storage bucket."""				,			)

							lowerCamelCase__					:	str         =       parser.parse_args()
							return args





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase      )      ->       str:







							def fn(UpperCamelCase      ):
														return tokenizer(examples["""text"""]      )

							return fn





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase      )      ->       Union[str, Any]:
							lowerCamelCase__					:	Any         =       []
							for i in range(len(tokenized_data["""input_ids"""]      )      ):
														lowerCamelCase__					:	List[str]         =       {
														    """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]      )      ),
														    """attention_mask""": tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]      )      ),
														}
														lowerCamelCase__					:	str         =       tf.train.Features(feature=UpperCamelCase      )
														lowerCamelCase__					:	List[Any]         =       tf.train.Example(features=UpperCamelCase      )
														lowerCamelCase__					:	Union[str, Any]         =       example.SerializeToString()
														records.append(UpperCamelCase      )
							return records





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase      )      ->       Tuple:
							lowerCamelCase__					:	Any         =       datasets.load_dataset(args.dataset_name				,			args.dataset_config				,			split=args.split      )

							if args.limit is not None:
														lowerCamelCase__					:	Tuple         =       min(len(UpperCamelCase      )				,			args.limit      )
														lowerCamelCase__					:	Union[str, Any]         =       dataset.select(range(UpperCamelCase      )      )
														print(f'''Limiting the dataset to {args.limit} entries.'''      )

							lowerCamelCase__					:	List[str]         =       AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir      ):
																					os.makedirs(args.output_dir      )
														lowerCamelCase__					:	Optional[Any]         =       os.path.join(args.output_dir				,			args.split      )
														if not os.path.exists(UpperCamelCase      ):
																					os.makedirs(UpperCamelCase      )
							else:
														lowerCamelCase__					:	Optional[Any]         =       os.path.join(args.output_dir				,			args.split      )

							# Tokenize the whole dataset at once.
							lowerCamelCase__					:	Tuple         =       tokenize_function(UpperCamelCase      )
							lowerCamelCase__					:	Optional[Any]         =       dataset.map(UpperCamelCase				,			batched=UpperCamelCase				,			num_proc=4				,			remove_columns=["""text"""]      )

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(UpperCamelCase      ):
														# Concatenate all texts.
														lowerCamelCase__					:	List[Any]         =       {k: sum(examples[k]				,			[]      ) for k in examples.keys()}
														lowerCamelCase__					:	List[str]         =       len(concatenated_examples[list(examples.keys()      )[0]]      )
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														lowerCamelCase__					:	Union[str, Any]         =       (total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														lowerCamelCase__					:	Any         =       {
														    k: [t[i : i + args.max_length] for i in range(0				,			UpperCamelCase				,			args.max_length      )]
														    for k, t in concatenated_examples.items()
														}
														return result

							lowerCamelCase__					:	Tuple         =       dataset_tokenized.map(UpperCamelCase				,			batched=UpperCamelCase				,			batch_size=1000				,			num_proc=4      )

							lowerCamelCase__					:	Optional[Any]         =       0
							lowerCamelCase__					:	Union[str, Any]         =       0
							for shard in range(0				,			len(UpperCamelCase      )				,			args.shard_size      ):
														lowerCamelCase__					:	str         =       grouped_dataset[shard : shard + args.shard_size]
														lowerCamelCase__					:	Union[str, Any]         =       len(dataset_snapshot["""input_ids"""]      )
														lowerCamelCase__					:	Dict         =       os.path.join(UpperCamelCase				,			f'''dataset-{shard_count}-{records_containing}.tfrecord'''      )
														lowerCamelCase__					:	str         =       get_serialized_examples(UpperCamelCase      )

														with tf.io.TFRecordWriter(UpperCamelCase      ) as out_file:
																					for i in range(len(UpperCamelCase      )      ):
																												lowerCamelCase__					:	List[Any]         =       serialized_examples[i]
																												out_file.write(UpperCamelCase      )
																					print("""Wrote file {} containing {} records""".format(UpperCamelCase				,			UpperCamelCase      )      )

														shard_count += 1
														total_records += records_containing

							with open(f'''split-{args.split}-records-count.txt'''				,			"""w"""      ) as f:
														print(f'''Total {args.split} records: {total_records}'''				,			file=UpperCamelCase      )


if __name__ == "__main__":
			_A	:			List[Any]         =parse_args()
			main(args)





'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase							:   Tuple  						=	logging.get_logger(__name__)

lowerCamelCase							:   Dict  						=	{
    'microsoft/cvt-13': 'https://huggingface.co/microsoft/cvt-13/resolve/main/config.json',
    # See all Cvt models at https://huggingface.co/models?filter=cvt
}
class   __lowerCAmelCase						(lowercase_     ):





      '''simple docstring'''






      lowerCAmelCase__					:     Tuple  					=							"""cvt"""



      def __init__(self    :    int   ,						UpperCamelCase    :    List[Any]=3   ,						UpperCamelCase    :    int=[7, 3, 3]   ,						UpperCamelCase    :    str=[4, 2, 2]   ,						UpperCamelCase    :    Dict=[2, 1, 1]   ,						UpperCamelCase    :    Dict=[64, 192, 384]   ,						UpperCamelCase    :    Dict=[1, 3, 6]   ,						UpperCamelCase    :    Dict=[1, 2, 10]   ,						UpperCamelCase    :    Any=[4.0, 4.0, 4.0]   ,						UpperCamelCase    :    Optional[int]=[0.0, 0.0, 0.0]   ,						UpperCamelCase    :    Optional[int]=[0.0, 0.0, 0.0]   ,						UpperCamelCase    :    int=[0.0, 0.0, 0.1]   ,						UpperCamelCase    :    Any=[True, True, True]   ,						UpperCamelCase    :    int=[False, False, True]   ,						UpperCamelCase    :    Union[str, Any]=["dw_bn", "dw_bn", "dw_bn"]   ,						UpperCamelCase    :    Optional[int]=[3, 3, 3]   ,						UpperCamelCase    :    Tuple=[1, 1, 1]   ,						UpperCamelCase    :    Any=[2, 2, 2]   ,						UpperCamelCase    :    Dict=[1, 1, 1]   ,						UpperCamelCase    :    List[str]=[1, 1, 1]   ,						UpperCamelCase    :    str=0.02   ,						UpperCamelCase    :    int=1E-12   ,						**UpperCamelCase    :    Union[str, Any]   ,						):

             '''simple docstring'''







             super().__init__(**UpperCamelCase )
             lowercase__      			= num_channels
             lowercase__      			= patch_sizes
             lowercase__      			= patch_stride
             lowercase__      			= patch_padding
             lowercase__      			= embed_dim
             lowercase__      			= num_heads
             lowercase__      			= depth
             lowercase__      			= mlp_ratio
             lowercase__      			= attention_drop_rate
             lowercase__      			= drop_rate
             lowercase__      			= drop_path_rate
             lowercase__      			= qkv_bias
             lowercase__      			= cls_token
             lowercase__      			= qkv_projection_method
             lowercase__      			= kernel_qkv
             lowercase__      			= padding_kv
             lowercase__      			= stride_kv
             lowercase__      			= padding_q
             lowercase__      			= stride_q
             lowercase__      			= initializer_range
             lowercase__      			= layer_norm_eps




