

"""simple docstring"""




import warnings

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import pairwise_distances

warnings.filterwarnings('ignore')

__A	:       Optional[Any]        =						'K-MEANS-CLUST/ '
def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__     ,				lowercase__=None ):
					"""simple docstring"""
					if seed is not None:  # useful for obtaining consistent results
										np.random.seed(lowercase__ )
					A     =   data.shape[0]  # number of data points

					# Pick K indices from range [0, N).
					A     =   np.random.randint(0     ,				lowercase__     ,				lowercase__ )

					# Keep centroids as dense format, as many entries will be nonzero due to averaging.
					# As long as at least one document in a cluster contains a word,
					# it will carry a nonzero weight in the TF-IDF vector of the centroid.
					A     =   data[rand_indices, :]

					return centroids
def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__ ):
					"""simple docstring"""
					return pairwise_distances(lowercase__     ,				lowercase__     ,				metric="euclidean" )
def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__ ):
					"""simple docstring"""
					# Compute distances between each data point and the set of centroids:
					# Fill in the blank (RHS only)
					A     =   centroid_pairwise_dist(lowercase__     ,				lowercase__ )

					# Compute cluster assignments for each data point:
					# Fill in the blank (RHS only)
					A     =   np.argmin(lowercase__     ,				axis=1 )

					return cluster_assignment
def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__     ,				lowercase__ ):
					"""simple docstring"""
					A     =   []
					for i in range(lowercase__ ):
										# Select all data points that belong to cluster i. Fill in the blank (RHS only)
										A     =   data[cluster_assignment == i]
										# Compute the mean of the data points. Fill in the blank (RHS only)
										A     =   member_data_points.mean(axis=0 )
										new_centroids.append(lowercase__ )
					A     =   np.array(lowercase__ )

					return new_centroids
def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__     ,				lowercase__     ,				lowercase__ ):
					"""simple docstring"""
					A     =   0.0
					for i in range(lowercase__ ):
										# Select all data points that belong to cluster i. Fill in the blank (RHS only)
										A     =   data[cluster_assignment == i, :]

										if member_data_points.shape[0] > 0:  # check if i-th cluster is non-empty
															# Compute distances from centroid to data points (RHS only)
															A     =   pairwise_distances(
															    lowercase__     ,				[centroids[i]]     ,				metric="euclidean" )
															A     =   distances**2
															heterogeneity += np.sum(lowercase__ )

					return heterogeneity
def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__ ):
					"""simple docstring"""
					plt.figure(figsize=(7, 4) )
					plt.plot(lowercase__     ,				linewidth=4 )
					plt.xlabel("# Iterations" )
					plt.ylabel("Heterogeneity" )
					plt.title(F"""Heterogeneity of clustering over time, K={k:d}""" )
					plt.rcParams.update({"font.size": 16} )
					plt.show()
def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__     ,				lowercase__     ,				lowercase__=500     ,				lowercase__=None     ,				lowercase__=False ):
					"""simple docstring"""
					A     =   initial_centroids[:]
					A     =   None

					for itr in range(lowercase__ ):
										if verbose:
															print(lowercase__     ,				end="" )

										# 1. Make cluster assignments using nearest centroids
										A     =   assign_clusters(lowercase__     ,				lowercase__ )

										# 2. Compute a new centroid for each of the k clusters, averaging all data
										#    points assigned to that cluster.
										A     =   revise_centroids(lowercase__     ,				lowercase__     ,				lowercase__ )

										# Check for convergence: if none of the assignments changed, stop
										if (
										    prev_cluster_assignment is not None
										    and (prev_cluster_assignment == cluster_assignment).all()
										):
															break

										# Print number of new assignments
										if prev_cluster_assignment is not None:
															A     =   np.sum(prev_cluster_assignment != cluster_assignment )
															if verbose:
																				print(
																				    F"""    {num_changed:5d} elements changed their cluster assignment.""" )

        # Record heterogeneity convergence metric
										if record_heterogeneity is not None:
															# YOUR CODE HERE
															A     =   compute_heterogeneity(lowercase__     ,				lowercase__     ,				lowercase__     ,				lowercase__ )
															record_heterogeneity.append(lowercase__ )

										A     =   cluster_assignment[:]

					return centroids, cluster_assignment


# Mock test below
if False:  # change to true to run this test case.
	from sklearn import datasets as ds

	__A	:       Union[str, Any]        =						ds.load_iris()
	__A	:       List[Any]        =						3
	__A	:       Optional[Any]        =						[]
	__A	:       Optional[int]        =						get_initial_centroids(dataset['data'], k, seed=0)
	__A    ,       __A	:       Dict        =						kmeans(
	    dataset['data'],
	    k,
	    initial_centroids,
	    maxiter=4_00,
	    record_heterogeneity=heterogeneity,
	    verbose=True,
	)
	plot_heterogeneity(heterogeneity, k)
def   __SCREAMING_SNAKE_CASE		(		lowercase__     ,				lowercase__     ,				lowercase__=None ):
					"""simple docstring"""
					# Fill missing values with given rules
					if fill_missing_report:
										A     =   df.fillna(value=lowercase__ )
					A     =   1
					A     =   df.select_dtypes(np.number ).columns
					A     =   (
					    df.groupby(["Cluster"] )[  # construct report dataframe
					        numeric_cols
					    ]  # group by cluster number
					    .agg(
					        [
					            ("sum", np.sum),
					            ("mean_with_zeros", lambda lowercase__ : np.mean(np.nan_to_num(lowercase__ ) )),
					            ("mean_without_zeros", lambda lowercase__ : x.replace(0     ,				np.NaN ).mean()),
					            (
					                "mean_25-75",
					                lambda lowercase__ : np.mean(
					                    np.nan_to_num(
					                        sorted(lowercase__ )[
					                            round(len(lowercase__ ) * 25 / 100 ) : round(len(lowercase__ ) * 75 / 100 )
					                        ] ) ),
					            ),
					            ("mean_with_na", np.mean),
					            ("min", lambda lowercase__ : x.min()),
					            ("5%", lambda lowercase__ : x.quantile(0.05 )),
					            ("25%", lambda lowercase__ : x.quantile(0.25 )),
					            ("50%", lambda lowercase__ : x.quantile(0.50 )),
					            ("75%", lambda lowercase__ : x.quantile(0.75 )),
					            ("95%", lambda lowercase__ : x.quantile(0.95 )),
					            ("max", lambda lowercase__ : x.max()),
					            ("count", lambda lowercase__ : x.count()),
					            ("stdev", lambda lowercase__ : x.std()),
					            ("mode", lambda lowercase__ : x.mode()[0]),
					            ("median", lambda lowercase__ : x.median()),
					            ("# > 0", lambda lowercase__ : (x > 0).sum()),
					        ] )
					    .T.reset_index()
					    .rename(index=lowercase__     ,				columns={"level_0": "Features", "level_1": "Type"} )
					)  # rename columns
					# calculate the size of cluster(count of clientID's)
					A     =   report[
					    (report["Features"] == "dummy") & (report["Type"] == "count")
					].copy()  # avoid SettingWithCopyWarning
					A     =   (
					    "ClusterSize"  # rename created cluster df to match report column names
					)
					A     =   "# of Customers"
					A     =   pd.DataFrame(
					    clustersize.iloc[:, 2:].values
					    / clustersize.iloc[:, 2:].values.sum() )
					A     =   "% of Customers"  # rename created cluster df to match report column names
					A     =   "ClusterProportion"
					A     =   clusterproportion.columns.tolist()
					A     =   cols[-2:] + cols[:-2]
					A     =   clusterproportion[cols]  # rearrange columns to match report
					A     =   report.columns
					A     =   pd.DataFrame(
					    abs(
					        report[report["Type"] == "count"].iloc[:, 2:].values
					        - clustersize.iloc[:, 2:].values ) )  # generating df with count of nan values
					A     =   0
					A     =   "# of nan"
					A     =   report[
					    report["Type"] == "count"
					].Features.tolist()  # filling values in order to match report
					A     =   a.columns.tolist()
					A     =   cols[-2:] + cols[:-2]
					A     =   a[cols]  # rearrange columns to match report
					A     =   report.columns  # rename columns to match report
					A     =   report.drop(
					    report[report.Type == "count"].index )  # drop count values except cluster size
					A     =   pd.concat(
					    [report, a, clustersize, clusterproportion]     ,				axis=0 )  # concat report with clustert size and nan values
					A     =   report["Features"].isin(lowercase__ )
					A     =   report.columns.tolist()
					A     =   cols[0:2] + cols[-1:] + cols[2:-1]
					A     =   report[cols]
					A     =   {
					    "ClusterSize": 9,
					    "ClusterProportion": 8,
					    "mean_with_zeros": 7,
					    "mean_with_na": 6,
					    "max": 5,
					    "50%": 4,
					    "min": 3,
					    "25%": 2,
					    "75%": 1,
					    "# of nan": 0,
					    "# > 0": -1,
					    "sum_with_na": -2,
					}
					A     =   (
					    report.assign(
					        Sortera=lambda lowercase__ : x.Type.map(lowercase__ )     ,				Sortera=lambda lowercase__ : list(reversed(range(len(lowercase__ ) ) ) )     ,				)
					    .sort_values(["Sorter1", "Mark", "Sorter2"]     ,				ascending=lowercase__ )
					    .drop(["Sorter1", "Sorter2"]     ,				axis=1 )
					)
					A     =   ""
					A     =   report.reset_index()
					A     =   report.drop(columns=["index"] )
					return report


if __name__ == "__main__":
	import doctest

	doctest.testmod()










import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation

_lowercase				:					Any     					=logging.get_logger(__name__)

_lowercase				:					Tuple     					={"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

_lowercase				:					int     					={
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

_lowercase				:					Tuple     					={
    """microsoft/deberta-base""": 5_12,
    """microsoft/deberta-large""": 5_12,
    """microsoft/deberta-xlarge""": 5_12,
    """microsoft/deberta-base-mnli""": 5_12,
    """microsoft/deberta-large-mnli""": 5_12,
    """microsoft/deberta-xlarge-mnli""": 5_12,
}

_lowercase				:					str     					={
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}




def    UpperCAmelCase					(  ):



			'''simple docstring'''





			a__					=  (
			    list(range(ord("""!"""	)				,					ord("""~"""	) + 1	)	) + list(range(ord("""¡"""	)				,					ord("""¬"""	) + 1	)	) + list(range(ord("""®"""	)				,					ord("""ÿ"""	) + 1	)	)
			)
			a__					=  bs[:]
			a__					=  0
			for b in range(2**8	):
						if b not in bs:
									bs.append(lowercase__	)
									cs.append(2**8 + n	)
									n += 1
			a__					=  [chr(lowercase__	) for n in cs]
			return dict(zip(lowercase__				,					lowercase__	)	)




def    UpperCAmelCase					(  lowercase__						:					Optional[Any]	):



			'''simple docstring'''





			a__					=  set()
			a__					=  word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)	)
						a__					=  char
			return pairs





class lowerCAmelCase_       (   A_			):







	'''simple docstring'''




	A_       :     List[str]									=		VOCAB_FILES_NAMES
	A_       :     str									=		PRETRAINED_VOCAB_FILES_MAP
	A_       :     List[str]									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A_       :     Optional[int]									=		['input_ids', 'attention_mask', 'token_type_ids']

	def __init__(	self   ,       lowerCamelCase   ,       lowerCamelCase   ,       lowerCamelCase="replace"   ,       lowerCamelCase="[CLS]"   ,       lowerCamelCase="[SEP]"   ,       lowerCamelCase="[SEP]"   ,       lowerCamelCase="[CLS]"   ,       lowerCamelCase="[UNK]"   ,       lowerCamelCase="[PAD]"   ,       lowerCamelCase="[MASK]"   ,       lowerCamelCase=False   ,       lowerCamelCase=False   ,       **lowerCamelCase   ,       ):



				'''simple docstring'''


				a__					=  AddedToken(lowerCamelCase   ,       lstrip=lowerCamelCase   ,       rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase   ,       lowerCamelCase       ) else bos_token
				a__					=  AddedToken(lowerCamelCase   ,       lstrip=lowerCamelCase   ,       rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase   ,       lowerCamelCase       ) else eos_token
				a__					=  AddedToken(lowerCamelCase   ,       lstrip=lowerCamelCase   ,       rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase   ,       lowerCamelCase       ) else sep_token
				a__					=  AddedToken(lowerCamelCase   ,       lstrip=lowerCamelCase   ,       rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase   ,       lowerCamelCase       ) else cls_token
				a__					=  AddedToken(lowerCamelCase   ,       lstrip=lowerCamelCase   ,       rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase   ,       lowerCamelCase       ) else unk_token
				a__					=  AddedToken(lowerCamelCase   ,       lstrip=lowerCamelCase   ,       rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase   ,       lowerCamelCase       ) else pad_token

				# Mask token behave like a normal word, i.e. include the space before it
				a__					=  AddedToken(lowerCamelCase   ,       lstrip=lowerCamelCase   ,       rstrip=lowerCamelCase       ) if isinstance(lowerCamelCase   ,       lowerCamelCase       ) else mask_token

				super().__init__(
				    errors=lowerCamelCase   ,       bos_token=lowerCamelCase   ,       eos_token=lowerCamelCase   ,       unk_token=lowerCamelCase   ,       sep_token=lowerCamelCase   ,       cls_token=lowerCamelCase   ,       pad_token=lowerCamelCase   ,       mask_token=lowerCamelCase   ,       add_prefix_space=lowerCamelCase   ,       add_bos_token=lowerCamelCase   ,       **lowerCamelCase   ,       )
				a__					=  add_bos_token

				with open(lowerCamelCase   ,       encoding="""utf-8"""       ) as vocab_handle:
							a__					=  json.load(lowerCamelCase       )
				a__					=  {v: k for k, v in self.encoder.items()}
				a__					=  errors  # how to handle errors in decoding
				a__					=  bytes_to_unicode()
				a__					=  {v: k for k, v in self.byte_encoder.items()}
				with open(lowerCamelCase   ,       encoding="""utf-8"""       ) as merges_handle:
							a__					=  merges_handle.read().split("""\n"""       )[1:-1]
				a__					=  [tuple(merge.split()       ) for merge in bpe_merges]
				a__					=  dict(zip(lowerCamelCase   ,       range(len(lowerCamelCase       )       )       )       )
				a__					=  {}
				a__					=  add_prefix_space

				# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
				a__					=  re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""       )

	@property
	# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
	def 				_A			(	self       ):



				'''simple docstring'''


				return len(self.encoder       )

	def 				_A			(	self       ):



				'''simple docstring'''


				return dict(self.encoder   ,       **self.added_tokens_encoder       )

	def 				_A			(	self   ,       lowerCamelCase       ):



				'''simple docstring'''


				if token in self.cache:
							return self.cache[token]
				a__					=  tuple(lowerCamelCase       )
				a__					=  get_pairs(lowerCamelCase       )

				if not pairs:
							return token

				while True:
							a__					=  min(lowerCamelCase   ,       key=lambda lowerCamelCase       : self.bpe_ranks.get(lowerCamelCase   ,       float("""inf"""       )       )       )
							if bigram not in self.bpe_ranks:
										break
							a__				,       a__					=  bigram
							a__					=  []
							a__					=  0
							while i < len(lowerCamelCase       ):
										try:
													a__					=  word.index(lowerCamelCase   ,       lowerCamelCase       )
										except ValueError:
													new_word.extend(word[i:]       )
													break
										else:
													new_word.extend(word[i:j]       )
													a__					=  j

										if word[i] == first and i < len(lowerCamelCase       ) - 1 and word[i + 1] == second:
													new_word.append(first + second       )
													i += 2
										else:
													new_word.append(word[i]       )
													i += 1
							a__					=  tuple(lowerCamelCase       )
							a__					=  new_word
							if len(lowerCamelCase       ) == 1:
										break
							else:
										a__					=  get_pairs(lowerCamelCase       )
				a__					=  """ """.join(lowerCamelCase       )
				a__					=  word
				return word

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase = None       ):



				'''simple docstring'''


				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				a__					=  [self.cls_token_id]
				a__					=  [self.sep_token_id]
				return cls + token_ids_a + sep + token_ids_a + sep

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase = None   ,       lowerCamelCase = False       ):



				'''simple docstring'''


				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowerCamelCase   ,       token_ids_a=lowerCamelCase   ,       already_has_special_tokens=lowerCamelCase       )

				if token_ids_a is None:
							return [1] + ([0] * len(lowerCamelCase       )) + [1]
				return [1] + ([0] * len(lowerCamelCase       )) + [1] + ([0] * len(lowerCamelCase       )) + [1]

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase = None       ):



				'''simple docstring'''


				a__					=  [self.sep_token_id]
				a__					=  [self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep       ) * [0]
				return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]

	def 				_A			(	self   ,       lowerCamelCase       ):



				'''simple docstring'''


				a__					=  []
				for token in re.findall(self.pat   ,       lowerCamelCase       ):
							a__					=  """""".join(
							    self.byte_encoder[b] for b in token.encode("""utf-8"""       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase       ).split(""" """       )       )
				return bpe_tokens

	def 				_A			(	self   ,       lowerCamelCase       ):



				'''simple docstring'''


				return self.encoder.get(lowerCamelCase   ,       self.encoder.get(self.unk_token       )       )

	def 				_A			(	self   ,       lowerCamelCase       ):



				'''simple docstring'''


				return self.decoder.get(lowerCamelCase       )

	def 				_A			(	self   ,       lowerCamelCase       ):



				'''simple docstring'''


				a__					=  """""".join(lowerCamelCase       )
				a__					=  bytearray([self.byte_decoder[c] for c in text]       ).decode("""utf-8"""   ,       errors=self.errors       )
				return text

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase = None       ):



				'''simple docstring'''


				if not os.path.isdir(lowerCamelCase       ):
							logger.error(f'Vocabulary path ({save_directory}) should be a directory'       )
							return
				a__					=  os.path.join(
				    lowerCamelCase   ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
				a__					=  os.path.join(
				    lowerCamelCase   ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

				with open(lowerCamelCase   ,       """w"""   ,       encoding="""utf-8"""       ) as f:
							f.write(json.dumps(self.encoder   ,       indent=2   ,       sort_keys=lowerCamelCase   ,       ensure_ascii=lowerCamelCase       ) + """\n"""       )

				a__					=  0
				with open(lowerCamelCase   ,       """w"""   ,       encoding="""utf-8"""       ) as writer:
							writer.write("""#version: 0.2\n"""       )
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,       key=lambda lowerCamelCase       : kv[1]       ):
										if index != token_index:
													logger.warning(
													    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
													    """ Please check that the tokenizer is not corrupted!"""       )
													a__					=  token_index
										writer.write(""" """.join(lowerCamelCase       ) + """\n"""       )
										index += 1

				return vocab_file, merge_file

	def 				_A			(	self   ,       lowerCamelCase   ,       lowerCamelCase=False   ,       **lowerCamelCase       ):



				'''simple docstring'''


				a__					=  kwargs.pop("""add_prefix_space"""   ,       self.add_prefix_space       )
				if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase       ) > 0 and not text[0].isspace()):
							a__					=  """ """ + text
				return (text, kwargs)







	def 				_A			(	self   ,       lowerCamelCase       ):



				'''simple docstring'''


				a__					=  []
				for is_user, text in conversation.iter_texts():
							input_ids.extend(self.encode(lowerCamelCase   ,       add_special_tokens=lowerCamelCase       ) + [self.eos_token_id]       )
				if len(lowerCamelCase       ) > self.model_max_length:
							a__					=  input_ids[-self.model_max_length :]
				return input_ids

