






# Logistic Regression from scratch

# In[62]:

# In[63]:

# importing all the required libraries

import numpy as np
from matplotlib import pyplot as plt
from sklearn import datasets

def    SCREAMING_SNAKE_CASE_							(							__magic_name__					:  Dict					)		->			Any:






	"""simple docstring"""



	return 1 / (1 + np.exp(-z					))

def    SCREAMING_SNAKE_CASE_							(							__magic_name__					:  Optional[Any]	,			__magic_name__					:  Dict					)		->			str:






	"""simple docstring"""



	return (-y * np.log(__magic_name__					) - (1 - y) * np.log(1 - h					)).mean()

def    SCREAMING_SNAKE_CASE_							(							__magic_name__					:  Tuple	,			__magic_name__					:  List[Any]	,			__magic_name__					:  List[Any]					)		->			Union[str, Any]:






	"""simple docstring"""



	UpperCamelCase     :Tuple          =	np.dot(__magic_name__	,			__magic_name__					)
	return np.sum(y * scores - np.log(1 + np.exp(__magic_name__					)					)					)

def    SCREAMING_SNAKE_CASE_							(							__magic_name__					:  str	,			__magic_name__					:  List[str]	,			__magic_name__					:  Optional[int]	,			__magic_name__					:  Optional[Any]=7_0000					)		->			Optional[Any]:






	"""simple docstring"""



	UpperCamelCase     :Tuple          =	np.zeros(x.shape[1]					)

	for iterations in range(__magic_name__					):
		UpperCamelCase     :str          =	np.dot(__magic_name__	,			__magic_name__					)
		UpperCamelCase     :str          =	sigmoid_function(__magic_name__					)
		UpperCamelCase     :int          =	np.dot(x.T	,			h - y					) / y.size
		UpperCamelCase     :int          =	theta - alpha * gradient  # updating the weights
		UpperCamelCase     :List[str]          =	np.dot(__magic_name__	,			__magic_name__					)
		UpperCamelCase     :str          =	sigmoid_function(__magic_name__					)
		UpperCamelCase     :List[Any]          =	cost_function(__magic_name__	,			__magic_name__					)
		if iterations % 100 == 0:
			print(f"""loss: {j} \t"""					)  # printing the loss after every 100 iterations
	return theta


# In[68]:

if __name__ == "__main__":
	UpperCAmelCase_ :						int      =      datasets.load_iris()
	UpperCAmelCase_ :						Dict      =      iris.data[:, :2]
	UpperCAmelCase_ :						Dict      =      (iris.target != 0) * 1

	UpperCAmelCase_ :						Optional[int]      =      0.1
	UpperCAmelCase_ :						List[Any]      =      logistic_reg(alpha, x, y, max_iterations=7_00_00)
	print('''theta: ''', theta)  # printing the theta i.e our weights vector

	def    SCREAMING_SNAKE_CASE_							(							__magic_name__					:  Optional[int]					)		->			List[str]:






		"""simple docstring"""



		return sigmoid_function(
		    np.dot(__magic_name__	,			__magic_name__					)					)  # predicting the value of probability from the logistic regression algorithm

	plt.figure(figsize=(10, 6))
	plt.scatter(x[y == 0][:, 0], x[y == 0][:, 1], color='''b''', label='''0''')
	plt.scatter(x[y == 1][:, 0], x[y == 1][:, 1], color='''r''', label='''1''')
	((UpperCAmelCase_)  ,				(UpperCAmelCase_)) :						Union[str, Any]      =      (x[:, 0].min(), x[:, 0].max())
	((UpperCAmelCase_)  ,				(UpperCAmelCase_)) :						List[str]      =      (x[:, 1].min(), x[:, 1].max())
	((UpperCAmelCase_)  ,				(UpperCAmelCase_)) :						List[str]      =      np.meshgrid(np.linspace(xa_min, xa_max), np.linspace(xa_min, xa_max))
	UpperCAmelCase_ :						List[Any]      =      np.c_[xxa.ravel(), xxa.ravel()]
	UpperCAmelCase_ :						Tuple      =      predict_prob(grid).reshape(xxa.shape)
	plt.contour(xxa, xxa, probs, [0.5], linewidths=1, colors='''black''')

	plt.legend()

	plt.show()


'''simple docstring'''







import tensorflow as tf

from ...tf_utils import shape_list


class 						__UpperCAmelCase      (					tf.keras.layers.Layer						):


							def __init__(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_=1    ,   lowerCAmelCase_=False    ,   **lowerCAmelCase_  ):





								"""simple docstring"""






								super().__init__(**lowerCAmelCase_  )

								_snake_case   						=		vocab_size
								_snake_case   						=		d_embed
								_snake_case   						=		d_proj

								_snake_case   						=		cutoffs + [vocab_size]
								_snake_case   						=		[0] + self.cutoffs
								_snake_case   						=		div_val

								_snake_case   						=		self.cutoffs[0]
								_snake_case   						=		len(self.cutoffs  ) - 1
								_snake_case   						=		self.shortlist_size + self.n_clusters
								_snake_case   						=		keep_order

								_snake_case   						=		[]
								_snake_case   						=		[]


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								if self.n_clusters > 0:
									_snake_case   						=		self.add_weight(
									    shape=(self.n_clusters, self.d_embed)    ,   initializer='zeros'    ,   trainable=lowerCAmelCase_    ,   name='cluster_weight'  )
									_snake_case   						=		self.add_weight(
									    shape=(self.n_clusters,)    ,   initializer='zeros'    ,   trainable=lowerCAmelCase_    ,   name='cluster_bias'  )

								if self.div_val == 1:
									for i in range(len(self.cutoffs  )  ):
										if self.d_proj != self.d_embed:
											_snake_case   						=		self.add_weight(
											    shape=(self.d_embed, self.d_proj)    ,   initializer='zeros'    ,   trainable=lowerCAmelCase_    ,   name=F'out_projs_._{i}'    ,   )
											self.out_projs.append(lowerCAmelCase_  )
										else:
											self.out_projs.append(lowerCAmelCase_  )
										_snake_case   						=		self.add_weight(
										    shape=(self.vocab_size, self.d_embed)    ,   initializer='zeros'    ,   trainable=lowerCAmelCase_    ,   name=F'out_layers_._{i}_._weight'    ,   )
										_snake_case   						=		self.add_weight(
										    shape=(self.vocab_size,)    ,   initializer='zeros'    ,   trainable=lowerCAmelCase_    ,   name=F'out_layers_._{i}_._bias'    ,   )
										self.out_layers.append((weight, bias)  )
								else:
									for i in range(len(self.cutoffs  )  ):
										_snake_case						, _snake_case   						=		self.cutoff_ends[i], self.cutoff_ends[i + 1]
										_snake_case   						=		self.d_embed // (self.div_val**i)

										_snake_case   						=		self.add_weight(
										    shape=(d_emb_i, self.d_proj)    ,   initializer='zeros'    ,   trainable=lowerCAmelCase_    ,   name=F'out_projs_._{i}'  )
										self.out_projs.append(lowerCAmelCase_  )
										_snake_case   						=		self.add_weight(
										    shape=(r_idx - l_idx, d_emb_i)    ,   initializer='zeros'    ,   trainable=lowerCAmelCase_    ,   name=F'out_layers_._{i}_._weight'    ,   )
										_snake_case   						=		self.add_weight(
										    shape=(r_idx - l_idx,)    ,   initializer='zeros'    ,   trainable=lowerCAmelCase_    ,   name=F'out_layers_._{i}_._bias'    ,   )
										self.out_layers.append((weight, bias)  )
								super().build(lowerCAmelCase_  )


							@staticmethod
							def  lowerCamelCase				(       lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_=None  ):





								"""simple docstring"""






								_snake_case   						=		x
								if proj is not None:
									_snake_case   						=		tf.einsum('ibd,ed->ibe'    ,   lowerCAmelCase_    ,   lowerCAmelCase_  )
								return tf.einsum('ibd,nd->ibn'    ,   lowerCAmelCase_    ,   lowerCAmelCase_  ) + b


							@staticmethod
							def  lowerCamelCase				(       lowerCAmelCase_    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		shape_list(lowerCAmelCase_  )
								_snake_case   						=		tf.range(lp_size[0]    ,   dtype=target.dtype  )
								_snake_case   						=		tf.stack([r, target]    ,   1  )
								return tf.gather_nd(lowerCAmelCase_    ,   lowerCAmelCase_  )




							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=False  ):





								"""simple docstring"""






								_snake_case   						=		0
								if self.n_clusters == 0:
									_snake_case   						=		self._logit(lowerCAmelCase_    ,   self.out_layers[0][0]    ,   self.out_layers[0][1]    ,   self.out_projs[0]  )
									if target is not None:
										_snake_case   						=		tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lowerCAmelCase_    ,   logits=lowerCAmelCase_  )
									_snake_case   						=		tf.nn.log_softmax(lowerCAmelCase_    ,   axis=-1  )
								else:
									_snake_case   						=		shape_list(lowerCAmelCase_  )
									_snake_case   						=		[]
									_snake_case   						=		tf.zeros(hidden_sizes[:2]  )
									for i in range(len(self.cutoffs  )  ):
										_snake_case						, _snake_case   						=		self.cutoff_ends[i], self.cutoff_ends[i + 1]
										if target is not None:
											_snake_case   						=		(target >= l_idx) & (target < r_idx)
											_snake_case   						=		tf.where(lowerCAmelCase_  )
											_snake_case   						=		tf.boolean_mask(lowerCAmelCase_    ,   lowerCAmelCase_  ) - l_idx

										if self.div_val == 1:
											_snake_case   						=		self.out_layers[0][0][l_idx:r_idx]
											_snake_case   						=		self.out_layers[0][1][l_idx:r_idx]
										else:
											_snake_case   						=		self.out_layers[i][0]
											_snake_case   						=		self.out_layers[i][1]

										if i == 0:
											_snake_case   						=		tf.concat([cur_W, self.cluster_weight]    ,   0  )
											_snake_case   						=		tf.concat([cur_b, self.cluster_bias]    ,   0  )

											_snake_case   						=		self._logit(lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   self.out_projs[0]  )
											_snake_case   						=		tf.nn.log_softmax(lowerCAmelCase_  )
											out.append(head_logprob[..., : self.cutoffs[0]]  )
											if target is not None:
												_snake_case   						=		tf.boolean_mask(lowerCAmelCase_    ,   lowerCAmelCase_  )
												_snake_case   						=		self._gather_logprob(lowerCAmelCase_    ,   lowerCAmelCase_  )
										else:
											_snake_case   						=		self._logit(lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   self.out_projs[i]  )
											_snake_case   						=		tf.nn.log_softmax(lowerCAmelCase_  )
											_snake_case   						=		self.cutoffs[0] + i - 1  # No probability for the head cluster
											_snake_case   						=		head_logprob[..., cluster_prob_idx, None] + tail_logprob
											out.append(lowerCAmelCase_  )
											if target is not None:
												_snake_case   						=		tf.boolean_mask(lowerCAmelCase_    ,   lowerCAmelCase_  )
												_snake_case   						=		tf.boolean_mask(lowerCAmelCase_    ,   lowerCAmelCase_  )
												_snake_case   						=		self._gather_logprob(lowerCAmelCase_    ,   lowerCAmelCase_  )
												cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
										if target is not None:
											loss += tf.scatter_nd(lowerCAmelCase_    ,   -cur_logprob    ,   shape_list(lowerCAmelCase_  )  )
									_snake_case   						=		tf.concat(lowerCAmelCase_    ,   axis=-1  )

								if target is not None:
									if return_mean:
										_snake_case   						=		tf.reduce_mean(lowerCAmelCase_  )
									# Add the training-time loss value to the layer using `self.add_loss()`.
									self.add_loss(lowerCAmelCase_  )

									# Log the loss as a metric (we could log arbitrary metrics,
									# including different metrics for training and inference.
									self.add_metric(lowerCAmelCase_    ,   name=self.name    ,   aggregation='mean' if return_mean else ''  )

								return out

