"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
    if is_torch_available():
        import torch
    if is_tf_available():
        import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__UpperCamelCase										=     logging.get_logger(__name__)

__UpperCamelCase										=     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__UpperCamelCase										=     {
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

__UpperCamelCase										=     {
    '''Salesforce/codegen-350M-mono''': 2048,
}







@lru_cache()
def    UpperCAmelCase       (						)							-> Dict:
     snake_case_   			=   (
         list(range(ord('!'   )		,   ord('~'   ) + 1   )   ) + list(range(ord('¡'   )		,   ord('¬'   ) + 1   )   ) + list(range(ord('®'   )		,   ord('ÿ'   ) + 1   )   )
     )
     snake_case_   			=   bs[:]
     snake_case_   			=   0
     for b in range(2**8   ):
          if b not in bs:
               bs.append(UpperCAmelCase   )
               cs.append(2**8 + n   )
               n += 1
     snake_case_   			=   [chr(UpperCAmelCase   ) for n in cs]
     return dict(zip(UpperCAmelCase		,   UpperCAmelCase   )   )







def    UpperCAmelCase       (						UpperCAmelCase   )							-> Optional[int]:
     snake_case_   			=   set()
     snake_case_   			=   word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)   )
          snake_case_   			=   char
     return pairs




class 					UpperCamelCase   (							lowerCAmelCase__  ):
       SCREAMING_SNAKE_CASE_						      = VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE_						      = PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE_						      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE_						      = ["input_ids", "attention_mask"]







       def __init__(					self,				lowerCAmelCase__,				lowerCAmelCase__,				lowerCAmelCase__="replace",				lowerCAmelCase__="<|endoftext|>",				lowerCAmelCase__="<|endoftext|>",				lowerCAmelCase__="<|endoftext|>",				lowerCAmelCase__=None,				lowerCAmelCase__=False,				lowerCAmelCase__=False,				**lowerCAmelCase__,				) -> Optional[Any]:
            snake_case_   			=   AddedToken(lowerCAmelCase__,				lstrip=lowerCAmelCase__,				rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__,				lowerCAmelCase__) else bos_token
            snake_case_   			=   AddedToken(lowerCAmelCase__,				lstrip=lowerCAmelCase__,				rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__,				lowerCAmelCase__) else eos_token
            snake_case_   			=   AddedToken(lowerCAmelCase__,				lstrip=lowerCAmelCase__,				rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__,				lowerCAmelCase__) else unk_token
            snake_case_   			=   AddedToken(lowerCAmelCase__,				lstrip=lowerCAmelCase__,				rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__,				lowerCAmelCase__) else pad_token
            super().__init__(
                errors=lowerCAmelCase__,				unk_token=lowerCAmelCase__,				bos_token=lowerCAmelCase__,				eos_token=lowerCAmelCase__,				pad_token=lowerCAmelCase__,				add_prefix_space=lowerCAmelCase__,				add_bos_token=lowerCAmelCase__,				**lowerCAmelCase__,				)
            snake_case_   			=   add_bos_token

            with open(lowerCAmelCase__,				encoding='utf-8') as vocab_handle:
                 snake_case_   			=   json.load(lowerCAmelCase__)
            snake_case_   			=   {v: k for k, v in self.encoder.items()}
            snake_case_   			=   errors  # how to handle errors in decoding
            snake_case_   			=   bytes_to_unicode()
            snake_case_   			=   {v: k for k, v in self.byte_encoder.items()}
            with open(lowerCAmelCase__,				encoding='utf-8') as merges_handle:
                 snake_case_   			=   merges_handle.read().split('\n')[1:-1]
            snake_case_   			=   [tuple(merge.split()) for merge in bpe_merges]
            snake_case_   			=   dict(zip(lowerCAmelCase__,				range(len(lowerCAmelCase__))))
            snake_case_   			=   {}
            snake_case_   			=   add_prefix_space

            # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
            snake_case_   			=   re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+')







       @property
       def 						a_		(					self) -> Optional[int]:
            return len(self.encoder)







       def 						a_		(					self) -> Any:
            return dict(self.encoder,				**self.added_tokens_encoder)







       def 						a_		(					self,				lowerCAmelCase__) -> Tuple:
            if token in self.cache:
                 return self.cache[token]
            snake_case_   			=   tuple(lowerCAmelCase__)
            snake_case_   			=   get_pairs(lowerCAmelCase__)

            if not pairs:
                 return token

            while True:
                 snake_case_   			=   min(lowerCAmelCase__,				key=lambda lowerCAmelCase__: self.bpe_ranks.get(lowerCAmelCase__,				float('inf')))
                 if bigram not in self.bpe_ranks:
                      break
                 snake_case_			,      snake_case_   			=   bigram
                 snake_case_   			=   []
                 snake_case_   			=   0
                 while i < len(lowerCAmelCase__):
                      try:
                           snake_case_   			=   word.index(lowerCAmelCase__,				lowerCAmelCase__)
                      except ValueError:
                           new_word.extend(word[i:])
                           break
                      else:
                           new_word.extend(word[i:j])
                           snake_case_   			=   j

                      if word[i] == first and i < len(lowerCAmelCase__) - 1 and word[i + 1] == second:
                           new_word.append(first + second)
                           i += 2
                      else:
                           new_word.append(word[i])
                           i += 1
                 snake_case_   			=   tuple(lowerCAmelCase__)
                 snake_case_   			=   new_word
                 if len(lowerCAmelCase__) == 1:
                      break
                 else:
                      snake_case_   			=   get_pairs(lowerCAmelCase__)
            snake_case_   			=   ' '.join(lowerCAmelCase__)
            snake_case_   			=   word
            return word







       def 						a_		(					self,				lowerCAmelCase__,				lowerCAmelCase__=None) -> List[str]:
            if self.add_bos_token:
                 snake_case_   			=   [self.bos_token_id]
            else:
                 snake_case_   			=   []

            snake_case_   			=   bos_token_ids + token_ids_a

            if token_ids_a is None:
                 return output

            return output + bos_token_ids + token_ids_a







       def 						a_		(					self,				lowerCAmelCase__) -> Optional[int]:
            snake_case_   			=   []
            for token in re.findall(self.pat,				lowerCAmelCase__):
                 snake_case_   			=   ''.join(
                     self.byte_encoder[b] for b in token.encode('utf-8'))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                 bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase__).split(' '))
            return bpe_tokens







       def 						a_		(					self,				lowerCAmelCase__) -> str:
            return self.encoder.get(lowerCAmelCase__,				self.encoder.get(self.unk_token))







       def 						a_		(					self,				lowerCAmelCase__) -> List[str]:
            return self.decoder.get(lowerCAmelCase__)







       def 						a_		(					self,				lowerCAmelCase__) -> Any:
            snake_case_   			=   ''.join(lowerCAmelCase__)
            snake_case_   			=   bytearray([self.byte_decoder[c] for c in text]).decode('utf-8',				errors=self.errors)
            return text







       def 						a_		(					self,				lowerCAmelCase__,				lowerCAmelCase__ = None) -> Tuple[str]:
            if not os.path.isdir(lowerCAmelCase__):
                 logger.error(f'Vocabulary path ({save_directory}) should be a directory')
                 return
            snake_case_   			=   os.path.join(
                lowerCAmelCase__,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
            snake_case_   			=   os.path.join(
                lowerCAmelCase__,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

            with open(lowerCAmelCase__,				'w',				encoding='utf-8') as f:
                 f.write(json.dumps(self.encoder,				indent=2,				sort_keys=lowerCAmelCase__,				ensure_ascii=lowerCAmelCase__) + '\n')

            snake_case_   			=   0
            with open(lowerCAmelCase__,				'w',				encoding='utf-8') as writer:
                 writer.write('#version: 0.2\n')
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),				key=lambda lowerCAmelCase__: kv[1]):
                      if index != token_index:
                           logger.warning(
                               f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                               ' Please check that the tokenizer is not corrupted!')
                           snake_case_   			=   token_index
                      writer.write(' '.join(lowerCAmelCase__) + '\n')
                      index += 1

            return vocab_file, merge_file







       def 						a_		(					self,				lowerCAmelCase__,				lowerCAmelCase__=False,				**lowerCAmelCase__) -> List[Any]:
            snake_case_   			=   kwargs.pop('add_prefix_space',				self.add_prefix_space)
            if is_split_into_words or add_prefix_space:
                 snake_case_   			=   ' ' + text
            return (text, kwargs)







       def 						a_		(					self,				lowerCAmelCase__,				lowerCAmelCase__ = False,				lowerCAmelCase__ = None,				lowerCAmelCase__ = None,				**lowerCAmelCase__,				) -> str:
            snake_case_   			=   super()._decode(
                token_ids=lowerCAmelCase__,				skip_special_tokens=lowerCAmelCase__,				clean_up_tokenization_spaces=lowerCAmelCase__,				**lowerCAmelCase__,				)

            if truncate_before_pattern is not None and len(lowerCAmelCase__) > 0:
                 snake_case_   			=   self.truncate(lowerCAmelCase__,				lowerCAmelCase__)

            return decoded_text







       def 						a_		(					self,				lowerCAmelCase__,				lowerCAmelCase__) -> Optional[Any]:
            def find_re(lowerCAmelCase__,				lowerCAmelCase__,				lowerCAmelCase__):
                 snake_case_   			=   pattern.search(lowerCAmelCase__,				lowerCAmelCase__)
                 return m.start() if m else -1

            snake_case_   			=   [re.compile(lowerCAmelCase__,				re.MULTILINE) for pattern in truncate_before_pattern]

            snake_case_   			=   list(re.finditer('^print',				lowerCAmelCase__,				re.MULTILINE))

            if len(lowerCAmelCase__) > 1:
                 snake_case_   			=   completion[: prints[1].start()]

            snake_case_   			=   list(re.finditer('^def',				lowerCAmelCase__,				re.MULTILINE))

            if len(lowerCAmelCase__) > 1:
                 snake_case_   			=   completion[: defs[1].start()]

            snake_case_   			=   0

            snake_case_   			=   [
                pos for pos in [find_re(lowerCAmelCase__,				lowerCAmelCase__,				lowerCAmelCase__) for terminal in terminals] if pos != -1
            ]

            if len(lowerCAmelCase__) > 0:
                 return completion[: min(lowerCAmelCase__)]
            else:
                 return completion


"""simple docstring"""


from typing import Dict, Tuple, overload

import torch
import torch.types
from torch import nn

from . import residue_constants as rc
from .rigid_utils import Rigid, Rotation
from .tensor_utils import batched_gather







@overload
def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase   )							-> torch.Tensor:
     ...







@overload
def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase   )							-> Tuple[torch.Tensor, torch.Tensor]:
     ...







def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase   )							-> Dict:
     snake_case_   			=   aatype == rc.restype_order['G']
     snake_case_   			=   rc.atom_order['CA']
     snake_case_   			=   rc.atom_order['CB']
     snake_case_   			=   torch.where(
         is_gly[..., None].expand(*((-1,) * len(is_gly.shape   ))		,   3   )		,   all_atom_positions[..., ca_idx, :]		,   all_atom_positions[..., cb_idx, :]		,   )

     if all_atom_masks is not None:
          snake_case_   			=   torch.where(
              UpperCAmelCase		,   all_atom_masks[..., ca_idx]		,   all_atom_masks[..., cb_idx]		,   )
          return pseudo_beta, pseudo_beta_mask
     else:
          return pseudo_beta







def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase   )							-> torch.Tensor:
     snake_case_   			=   batched_gather(
         UpperCAmelCase		,   batch['residx_atom37_to_atom14']		,   dim=-2		,   no_batch_dims=len(atomaa.shape[:-2]   )		,   )

     snake_case_   			=   atomaa_data * batch['atom37_atom_exists'][..., None]

     return atomaa_data







def    UpperCAmelCase       (						UpperCAmelCase   )							-> torch.Tensor:
     snake_case_   			=   template_feats['template_aatype']
     snake_case_   			=   template_feats['template_torsion_angles_sin_cos']
     snake_case_   			=   template_feats['template_alt_torsion_angles_sin_cos']
     snake_case_   			=   template_feats['template_torsion_angles_mask']
     snake_case_   			=   torch.cat(
         [
             nn.functional.one_hot(UpperCAmelCase		,   22   ),
             torsion_angles_sin_cos.reshape(*torsion_angles_sin_cos.shape[:-2]		,   14   ),
             alt_torsion_angles_sin_cos.reshape(*alt_torsion_angles_sin_cos.shape[:-2]		,   14   ),
             torsion_angles_mask,
         ]		,   dim=-1		,   )

     return template_angle_feat







def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase = False		,   UpperCAmelCase = 1e-20		,   UpperCAmelCase = 1e8		,   )							-> torch.Tensor:
     snake_case_   			=   batch['template_pseudo_beta_mask']
     snake_case_   			=   template_mask[..., None] * template_mask[..., None, :]

     # Compute distogram (this seems to differ slightly from Alg. 5)
     snake_case_   			=   batch['template_pseudo_beta']
     snake_case_   			=   torch.sum((tpb[..., None, :] - tpb[..., None, :, :]) ** 2		,   dim=-1		,   keepdim=UpperCAmelCase   )
     snake_case_   			=   torch.linspace(UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase		,   device=tpb.device   ) ** 2
     snake_case_   			=   torch.cat([lower[1:], lower.new_tensor([inf]   )]		,   dim=-1   )
     snake_case_   			=   ((dgram > lower) * (dgram < upper)).type(dgram.dtype   )

     snake_case_   			=   [dgram, template_mask_ad[..., None]]

     snake_case_   			=   nn.functional.one_hot(
         batch['template_aatype']		,   rc.restype_num + 2		,   )

     snake_case_   			=   batch['template_aatype'].shape[-1]
     to_concat.append(aatype_one_hot[..., None, :, :].expand(*aatype_one_hot.shape[:-2]		,   UpperCAmelCase		,   -1		,   -1   )   )
     to_concat.append(aatype_one_hot[..., None, :].expand(*aatype_one_hot.shape[:-2]		,   -1		,   UpperCAmelCase		,   -1   )   )

     snake_case_			,      snake_case_			,      snake_case_   			=   [rc.atom_order[a] for a in ['N', 'CA', 'C']]
     snake_case_   			=   Rigid.make_transform_from_reference(
         n_xyz=batch['template_all_atom_positions'][..., n, :]		,   ca_xyz=batch['template_all_atom_positions'][..., ca, :]		,   c_xyz=batch['template_all_atom_positions'][..., c, :]		,   eps=UpperCAmelCase		,   )
     snake_case_   			=   rigids.get_trans()[..., None, :, :]
     snake_case_   			=   rigids[..., None].invert_apply(UpperCAmelCase   )

     snake_case_   			=   torch.rsqrt(eps + torch.sum(rigid_vec**2		,   dim=-1   )   )

     snake_case_   			=   batch['template_all_atom_mask']
     snake_case_   			=   t_aa_masks[..., n] * t_aa_masks[..., ca] * t_aa_masks[..., c]
     snake_case_   			=   template_mask[..., None] * template_mask[..., None, :]

     snake_case_   			=   inv_distance_scalar * template_mask_ad
     snake_case_   			=   rigid_vec * inv_distance_scalar[..., None]

     if not use_unit_vector:
          snake_case_   			=   unit_vector * 0.0

     to_concat.extend(torch.unbind(unit_vector[..., None, :]		,   dim=-1   )   )
     to_concat.append(template_mask_ad[..., None]   )

     snake_case_   			=   torch.cat(UpperCAmelCase		,   dim=-1   )
     snake_case_   			=   act * template_mask_ad[..., None]

     return act







def    UpperCAmelCase       (						UpperCAmelCase   )							-> torch.Tensor:
     snake_case_   			=   nn.functional.one_hot(batch['extra_msa']		,   23   )
     snake_case_   			=   [
         msa_ahot,
         batch['extra_has_deletion'].unsqueeze(-1   ),
         batch['extra_deletion_value'].unsqueeze(-1   ),
     ]
     return torch.cat(UpperCAmelCase		,   dim=-1   )







def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase		,   )							-> Rigid:
     # [*, N, 8, 4, 4]
     snake_case_   			=   rrgdf[aatype, ...]

     # [*, N, 8] transformations, i.e.
     #   One [*, N, 8, 3, 3] rotation matrix and
     #   One [*, N, 8, 3]    translation matrix
     snake_case_   			=   r.from_tensor_axa(UpperCAmelCase   )

     snake_case_   			=   alpha.new_zeros((*((1,) * len(alpha.shape[:-1]   )), 2)   )
     snake_case_   			=   1

     # [*, N, 8, 2]
     snake_case_   			=   torch.cat([bb_rot.expand(*alpha.shape[:-2]		,   -1		,   -1   ), alpha]		,   dim=-2   )

     # [*, N, 8, 3, 3]
     # Produces rotation matrices of the form:
     # [
     #   [1, 0  , 0  ],
     #   [0, a_2,-a_1],
     #   [0, a_1, a_2]
     # ]
     # This follows the original code rather than the supplement, which uses
     # different indices.

     snake_case_   			=   alpha.new_zeros(default_r.get_rots().get_rot_mats().shape   )
     snake_case_   			=   1
     snake_case_   			=   alpha[..., 1]
     snake_case_   			=   -alpha[..., 0]
     snake_case_   			=   alpha

     snake_case_   			=   default_r.compose(Rigid(Rotation(rot_mats=UpperCAmelCase   )		,   UpperCAmelCase   )   )

     snake_case_   			=   all_frames[..., 5]
     snake_case_   			=   all_frames[..., 6]
     snake_case_   			=   all_frames[..., 7]

     snake_case_   			=   all_frames[..., 4]
     snake_case_   			=   chia_frame_to_bb.compose(UpperCAmelCase   )
     snake_case_   			=   chia_frame_to_bb.compose(UpperCAmelCase   )
     snake_case_   			=   chia_frame_to_bb.compose(UpperCAmelCase   )

     snake_case_   			=   Rigid.cat(
         [
             all_frames[..., :5],
             chia_frame_to_bb.unsqueeze(-1   ),
             chia_frame_to_bb.unsqueeze(-1   ),
             chia_frame_to_bb.unsqueeze(-1   ),
         ]		,   dim=-1		,   )

     snake_case_   			=   r[..., None].compose(UpperCAmelCase   )

     return all_frames_to_global







def    UpperCAmelCase       (						UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase		,   UpperCAmelCase		,   )							-> torch.Tensor:
     # [*, N, 14]
     snake_case_   			=   group_idx[aatype, ...]

     # [*, N, 14, 8]
     snake_case_   			=   nn.functional.one_hot(
         UpperCAmelCase		,   num_classes=default_frames.shape[-3]		,   )

     # [*, N, 14, 8]
     snake_case_   			=   r[..., None, :] * group_mask_one_hot

     # [*, N, 14]
     snake_case_   			=   t_atoms_to_global.map_tensor_fn(lambda UpperCAmelCase   : torch.sum(UpperCAmelCase		,   dim=-1   )   )

     # [*, N, 14, 1]
     snake_case_   			=   atom_mask[aatype, ...].unsqueeze(-1   )

     # [*, N, 14, 3]
     snake_case_   			=   lit_positions[aatype, ...]
     snake_case_   			=   t_atoms_to_global.apply(UpperCAmelCase   )
     snake_case_   			=   pred_positions * atom_mask

     return pred_positions

