'''simple docstring'''


import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case__	:     str      =    logging.get_logger(__name__)

snake_case__	:     Optional[Any]      =    {
    '''src_vocab_file''': '''vocab-src.json''',
    '''tgt_vocab_file''': '''vocab-tgt.json''',
    '''merges_file''': '''merges.txt''',
}

snake_case__	:     Tuple      =    {
    '''src_vocab_file''': {
        '''stas/tiny-wmt19-en-de''': '''https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json'''
    },
    '''tgt_vocab_file''': {
        '''stas/tiny-wmt19-en-de''': '''https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json'''
    },
    '''merges_file''': {'''stas/tiny-wmt19-en-de''': '''https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt'''},
}

snake_case__	:     Dict      =    {'''stas/tiny-wmt19-en-de''': 1024}
snake_case__	:     List[Any]      =    {
    '''stas/tiny-wmt19-en-de''': {
        '''langs''': ['''en''', '''de'''],
        '''model_max_length''': 1024,
        '''special_tokens_map_file''': None,
        '''full_tokenizer_file''': None,
    }
}


def    _lowerCamelCase       ( lowerCamelCase_   :				Optional[Any]  ):



     """simple docstring"""





     UpperCAmelCase_				:  Optional[Any]  = set()
     UpperCAmelCase_				:  List[str]  = word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)  )
          UpperCAmelCase_				:  Dict  = char
     return pairs


def    _lowerCamelCase       ( lowerCamelCase_   :				Optional[int]  ):



     """simple docstring"""





     UpperCAmelCase_				:  str  = text.replace('，'      ,	','  )
     UpperCAmelCase_				:  str  = re.sub(R'。\s*'      ,	'. '      ,	lowerCamelCase_  )
     UpperCAmelCase_				:  Optional[Any]  = text.replace('、'      ,	','  )
     UpperCAmelCase_				:  Union[str, Any]  = text.replace('”'      ,	'"'  )
     UpperCAmelCase_				:  int  = text.replace('“'      ,	'"'  )
     UpperCAmelCase_				:  Optional[Any]  = text.replace('∶'      ,	':'  )
     UpperCAmelCase_				:  List[Any]  = text.replace('：'      ,	':'  )
     UpperCAmelCase_				:  List[str]  = text.replace('？'      ,	'?'  )
     UpperCAmelCase_				:  List[Any]  = text.replace('《'      ,	'"'  )
     UpperCAmelCase_				:  Optional[Any]  = text.replace('》'      ,	'"'  )
     UpperCAmelCase_				:  str  = text.replace('）'      ,	')'  )
     UpperCAmelCase_				:  str  = text.replace('！'      ,	'!'  )
     UpperCAmelCase_				:  Optional[Any]  = text.replace('（'      ,	'('  )
     UpperCAmelCase_				:  List[Any]  = text.replace('；'      ,	';'  )
     UpperCAmelCase_				:  int  = text.replace('１'      ,	'1'  )
     UpperCAmelCase_				:  Tuple  = text.replace('」'      ,	'"'  )
     UpperCAmelCase_				:  List[Any]  = text.replace('「'      ,	'"'  )
     UpperCAmelCase_				:  List[str]  = text.replace('０'      ,	'0'  )
     UpperCAmelCase_				:  Dict  = text.replace('３'      ,	'3'  )
     UpperCAmelCase_				:  Tuple  = text.replace('２'      ,	'2'  )
     UpperCAmelCase_				:  Tuple  = text.replace('５'      ,	'5'  )
     UpperCAmelCase_				:  int  = text.replace('６'      ,	'6'  )
     UpperCAmelCase_				:  Any  = text.replace('９'      ,	'9'  )
     UpperCAmelCase_				:  Dict  = text.replace('７'      ,	'7'  )
     UpperCAmelCase_				:  Optional[Any]  = text.replace('８'      ,	'8'  )
     UpperCAmelCase_				:  Any  = text.replace('４'      ,	'4'  )
     UpperCAmelCase_				:  int  = re.sub(R'．\s*'      ,	'. '      ,	lowerCamelCase_  )
     UpperCAmelCase_				:  Union[str, Any]  = text.replace('～'      ,	'~'  )
     UpperCAmelCase_				:  List[Any]  = text.replace('’'      ,	'\''  )
     UpperCAmelCase_				:  List[str]  = text.replace('…'      ,	'...'  )
     UpperCAmelCase_				:  List[str]  = text.replace('━'      ,	'-'  )
     UpperCAmelCase_				:  Union[str, Any]  = text.replace('〈'      ,	'<'  )
     UpperCAmelCase_				:  Dict  = text.replace('〉'      ,	'>'  )
     UpperCAmelCase_				:  Union[str, Any]  = text.replace('【'      ,	'['  )
     UpperCAmelCase_				:  Optional[int]  = text.replace('】'      ,	']'  )
     UpperCAmelCase_				:  str  = text.replace('％'      ,	'%'  )
     return text


def    _lowerCamelCase       ( lowerCamelCase_   :				str  ):



     """simple docstring"""





     UpperCAmelCase_				:  List[Any]  = []
     for char in text:
          UpperCAmelCase_				:  List[str]  = unicodedata.category(lowerCamelCase_  )
          if cat.startswith('C'  ):
               continue
          output.append(lowerCamelCase_  )
     return "".join(lowerCamelCase_  )






class 	__SCREAMING_SNAKE_CASE		(			lowerCamelCase_			):







  '''simple docstring'''






  lowerCamelCase_						:int		       =						VOCAB_FILES_NAMES
  lowerCamelCase_						:Dict		       =						PRETRAINED_VOCAB_FILES_MAP
  lowerCamelCase_						:int		       =						PRETRAINED_INIT_CONFIGURATION
  lowerCamelCase_						:str		       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCamelCase_						:List[Any]		       =						['''input_ids''', '''attention_mask''']


  def __init__(					self ,						snake_case_=None ,						snake_case_=None ,						snake_case_=None ,						snake_case_=None ,						snake_case_=False ,						snake_case_="<unk>" ,						snake_case_="<s>" ,						snake_case_="</s>" ,						snake_case_="<pad>" ,						**snake_case_ ,						):




       '''simple docstring'''

       super().__init__(
           langs=snake_case_ ,						src_vocab_file=snake_case_ ,						tgt_vocab_file=snake_case_ ,						merges_file=snake_case_ ,						do_lower_case=snake_case_ ,						unk_token=snake_case_ ,						bos_token=snake_case_ ,						sep_token=snake_case_ ,						pad_token=snake_case_ ,						**snake_case_ ,						)

       try:
            import sacremoses
       except ImportError:
            raise ImportError(
                'You need to install sacremoses to use XLMTokenizer. '
                'See https://pypi.org/project/sacremoses/ for installation.'		)

       UpperCAmelCase_				:  Any  = sacremoses

       UpperCAmelCase_				:  Any  = src_vocab_file
       UpperCAmelCase_				:  List[str]  = tgt_vocab_file
       UpperCAmelCase_				:  Optional[Any]  = merges_file
       UpperCAmelCase_				:  str  = do_lower_case

       # cache of sm.MosesPunctNormalizer instance
       UpperCAmelCase_				:  Tuple  = {}
       # cache of sm.MosesTokenizer instance
       UpperCAmelCase_				:  str  = {}
       UpperCAmelCase_				:  Optional[int]  = {}

       if langs and len(snake_case_		) == 2:
            UpperCAmelCase_  ,     UpperCAmelCase_				:  Optional[int]  = langs
       else:
            raise ValueError(
                F'''arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '''
                'Usually that means that tokenizer can\'t find a mapping for the given model path '
                'in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.'		)

       with open(snake_case_ ,						encoding='utf-8'		) as src_vocab_handle:
            UpperCAmelCase_				:  Union[str, Any]  = json.load(snake_case_		)
       with open(snake_case_ ,						encoding='utf-8'		) as tgt_vocab_handle:
            UpperCAmelCase_				:  Tuple  = json.load(snake_case_		)
            UpperCAmelCase_				:  Union[str, Any]  = {v: k for k, v in tgt_vocab.items()}
       with open(snake_case_ ,						encoding='utf-8'		) as merges_handle:
            UpperCAmelCase_				:  Optional[int]  = merges_handle.read().split('\n'		)[:-1]
       UpperCAmelCase_				:  int  = [tuple(merge.split()[:2]		) for merge in merges]
       UpperCAmelCase_				:  List[str]  = dict(zip(snake_case_ ,						range(len(snake_case_		)		)		)		)
       UpperCAmelCase_				:  Optional[int]  = {}


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       return self.get_src_vocab()


  @property
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       return self.src_vocab_size


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_		):




       '''simple docstring'''

       if lang not in self.cache_moses_punct_normalizer:
            UpperCAmelCase_				:  Optional[int]  = self.sm.MosesPunctNormalizer(lang=snake_case_		)
            UpperCAmelCase_				:  Optional[Any]  = punct_normalizer
       return self.cache_moses_punct_normalizer[lang].normalize(snake_case_		)


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_		):




       '''simple docstring'''

       if lang not in self.cache_moses_tokenizer:
            UpperCAmelCase_				:  List[Any]  = self.sm.MosesTokenizer(lang=snake_case_		)
            UpperCAmelCase_				:  str  = moses_tokenizer
       return self.cache_moses_tokenizer[lang].tokenize(
           snake_case_ ,						aggressive_dash_splits=snake_case_ ,						return_str=snake_case_ ,						escape=snake_case_		)


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_		):




       '''simple docstring'''

       if lang not in self.cache_moses_detokenizer:
            UpperCAmelCase_				:  List[str]  = self.sm.MosesDetokenizer(lang=snake_case_		)
            UpperCAmelCase_				:  Any  = moses_detokenizer
       return self.cache_moses_detokenizer[lang].detokenize(snake_case_		)


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  Optional[Any]  = replace_unicode_punct(snake_case_		)
       UpperCAmelCase_				:  Optional[Any]  = self.moses_punct_norm(snake_case_ ,						snake_case_		)
       UpperCAmelCase_				:  Tuple  = remove_non_printing_char(snake_case_		)
       return text


  @property
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       return len(self.encoder		)


  @property
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       return len(self.decoder		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       return dict(self.encoder ,						**self.added_tokens_encoder		)


  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       return dict(self.decoder ,						**self.added_tokens_decoder		)


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  str  = tuple(token[:-1]		) + (token[-1] + '</w>',)
       if token in self.cache:
            return self.cache[token]
       UpperCAmelCase_				:  Any  = get_pairs(snake_case_		)

       if not pairs:
            return token + "</w>"

       while True:
            UpperCAmelCase_				:  Dict  = min(snake_case_ ,						key=lambda snake_case_		: self.bpe_ranks.get(snake_case_ ,						float('inf'		)		)		)
            if bigram not in self.bpe_ranks:
                 break
            UpperCAmelCase_  ,     UpperCAmelCase_				:  str  = bigram
            UpperCAmelCase_				:  Any  = []
            UpperCAmelCase_				:  str  = 0
            while i < len(snake_case_		):
                 try:
                      UpperCAmelCase_				:  Union[str, Any]  = word.index(snake_case_ ,						snake_case_		)
                 except ValueError:
                      new_word.extend(word[i:]		)
                      break
                 else:
                      new_word.extend(word[i:j]		)
                      UpperCAmelCase_				:  str  = j

                 if word[i] == first and i < len(snake_case_		) - 1 and word[i + 1] == second:
                      new_word.append(first + second		)
                      i += 2
                 else:
                      new_word.append(word[i]		)
                      i += 1
            UpperCAmelCase_				:  Tuple  = tuple(snake_case_		)
            UpperCAmelCase_				:  List[Any]  = new_word
            if len(snake_case_		) == 1:
                 break
            else:
                 UpperCAmelCase_				:  Optional[Any]  = get_pairs(snake_case_		)
       UpperCAmelCase_				:  str  = ' '.join(snake_case_		)
       if word == "\n  </w>":
            UpperCAmelCase_				:  str  = '\n</w>'
       UpperCAmelCase_				:  str  = word
       return word


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_="en" ,						snake_case_=False		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = self.src_lang

       if self.do_lower_case:
            UpperCAmelCase_				:  Dict  = text.lower()

       if bypass_tokenizer:
            UpperCAmelCase_				:  Optional[int]  = text.split()
       else:
            UpperCAmelCase_				:  Optional[Any]  = self.moses_pipeline(snake_case_ ,						lang=snake_case_		)
            UpperCAmelCase_				:  Union[str, Any]  = self.moses_tokenize(snake_case_ ,						lang=snake_case_		)

       UpperCAmelCase_				:  Any  = []
       for token in text:
            if token:
                 split_tokens.extend(list(self.bpe(snake_case_		).split(' '		)		)		)

       return split_tokens


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       return self.encoder.get(snake_case_ ,						self.encoder.get(self.unk_token		)		)


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       return self.decoder.get(snake_case_ ,						self.unk_token		)


  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  int  = [t.replace(' ' ,						''		).replace('</w>' ,						' '		) for t in tokens]
       UpperCAmelCase_				:  Tuple  = ''.join(snake_case_		).split()
       # detokenize
       UpperCAmelCase_				:  Optional[int]  = self.moses_detokenize(snake_case_ ,						self.tgt_lang		)
       return text


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = [self.sep_token_id]

       # no bos used in fairseq
       if token_ids_a is None:
            return token_ids_a + sep
       return token_ids_a + sep + token_ids_a + sep


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None ,						snake_case_ = False		):




       '''simple docstring'''

       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=snake_case_ ,						token_ids_a=snake_case_ ,						already_has_special_tokens=snake_case_		)
       # no bos used in fairseq
       if token_ids_a is not None:
            return ([0] * len(snake_case_		)) + [1] + ([0] * len(snake_case_		)) + [1]
       return ([0] * len(snake_case_		)) + [1]


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None		):




       '''simple docstring'''

       UpperCAmelCase_				:  str  = [self.sep_token_id]

       # no bos used in fairseq
       if token_ids_a is None:
            return len(token_ids_a + sep		) * [0]
       return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None		):




       '''simple docstring'''

       if not os.path.isdir(snake_case_		):
            logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
            return

       UpperCAmelCase_				:  int  = os.path.join(
           snake_case_ ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['src_vocab_file']		)
       UpperCAmelCase_				:  List[Any]  = os.path.join(
           snake_case_ ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['tgt_vocab_file']		)
       UpperCAmelCase_				:  Any  = os.path.join(
           snake_case_ ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']		)

       with open(snake_case_ ,						'w' ,						encoding='utf-8'		) as f:
            f.write(json.dumps(self.encoder ,						indent=2 ,						sort_keys=snake_case_ ,						ensure_ascii=snake_case_		) + '\n'		)

       with open(snake_case_ ,						'w' ,						encoding='utf-8'		) as f:
            UpperCAmelCase_				:  Optional[int]  = {v: k for k, v in self.decoder.items()}
            f.write(json.dumps(snake_case_ ,						indent=2 ,						sort_keys=snake_case_ ,						ensure_ascii=snake_case_		) + '\n'		)

       UpperCAmelCase_				:  Tuple  = 0
       with open(snake_case_ ,						'w' ,						encoding='utf-8'		) as writer:
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,						key=lambda snake_case_		: kv[1]		):
                 if index != token_index:
                      logger.warning(
                          F'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
                          ' Please check that the tokenizer is not corrupted!'		)
                      UpperCAmelCase_				:  Any  = token_index
                 writer.write(' '.join(snake_case_		) + '\n'		)
                 index += 1

       return src_vocab_file, tgt_vocab_file, merges_file


  def __getstate__(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  int  = self.__dict__.copy()
       UpperCAmelCase_				:  Union[str, Any]  = None
       return state






  def __setstate__(					self ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  Tuple  = d

       try:
            import sacremoses
       except ImportError:
            raise ImportError(
                'You need to install sacremoses to use XLMTokenizer. '
                'See https://pypi.org/project/sacremoses/ for installation.'		)

       UpperCAmelCase_				:  List[str]  = sacremoses







'''simple docstring'''


import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata


def    _lowerCamelCase       ( lowerCamelCase_   :				str  ):



     """simple docstring"""





     UpperCAmelCase_				:  str  = min(re.search('^ +'      ,	lowerCamelCase_  ).end() if t.startswith(' '  ) else 0 for t in string.splitlines()  )
     return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(lowerCamelCase_  )]  )


snake_case__	:     Dict      =    '''\
---
language:
- zh
- en
task_ids:
- sentiment-classification
---
# Begin of markdown

Some cool dataset card
'''

snake_case__	:     Union[str, Any]      =    '''\
---
---
# Begin of markdown

Some cool dataset card
'''


snake_case__	:     int      =    '''\
# Begin of markdown

Some cool dataset card
'''






class 	__SCREAMING_SNAKE_CASE		(			unittest.TestCase			):







  '''simple docstring'''






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       with tempfile.TemporaryDirectory() as tmp_dir:
            UpperCAmelCase_				:  Tuple  = Path(snake_case_		) / 'README.md'
            with open(snake_case_ ,						'w+'		) as readme_file:
                 readme_file.write(snake_case_		)
            UpperCAmelCase_				:  Optional[Any]  = DatasetMetadata.from_readme(snake_case_		)
            self.assertDictEqual(snake_case_ ,						{'language': ['zh', 'en'], 'task_ids': ['sentiment-classification']}		)

            with open(snake_case_ ,						'w+'		) as readme_file:
                 readme_file.write(snake_case_		)
            UpperCAmelCase_				:  Union[str, Any]  = DatasetMetadata.from_readme(snake_case_		)
            self.assertDictEqual(snake_case_ ,						{}		)

            with open(snake_case_ ,						'w+'		) as readme_file:
                 readme_file.write(snake_case_		)
            UpperCAmelCase_				:  List[str]  = DatasetMetadata.from_readme(snake_case_		)
            self.assertEqual(snake_case_ ,						{}		)






  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       UpperCAmelCase_				:  int  = _dedent(
           '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            '		)
       assert DatasetMetadata.from_yaml_string(snake_case_		)

       UpperCAmelCase_				:  int  = _dedent(
           '\\n            annotations_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            task_ids:\n            - open-domain-qa\n            '		)
       with self.assertRaises(snake_case_		):
            DatasetMetadata.from_yaml_string(snake_case_		)

       UpperCAmelCase_				:  str  = _dedent(
           '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - text-classification\n            task_ids:\n            - multi-class-classification\n            paperswithcode_id:\n            - squad\n            configs:\n            - en\n            train-eval-index:\n            - config: en\n              task: text-classification\n              task_id: multi_class_classification\n              splits:\n                train_split: train\n                eval_split: test\n              col_mapping:\n                text: text\n                label: target\n              metrics:\n                - type: accuracy\n                  name: Accuracy\n            extra_gated_prompt: |\n              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:\n              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:\n              1. Researcher shall use the Database only for non-commercial research and educational purposes.\n            extra_gated_fields:\n              Company: text\n              Country: text\n              I agree to use this model for non-commerical use ONLY: checkbox\n            '		)
       assert DatasetMetadata.from_yaml_string(snake_case_		)






