




"""simple docstring"""

import numpy as np
import torch
from torch.utils.data import Dataset, IterableDataset

from ..utils.generic import ModelOutput





class 				lowercase_       (     _lowerCamelCase							):



 '''simple docstring'''

 def __init__( self       :       Optional[Any]    ,							_UpperCAmelCase       :       Optional[int]    ,							_UpperCAmelCase       :       Optional[Any]    ,							_UpperCAmelCase       :       Any		):
       _A					   =   dataset
       _A					   =   process
       _A					   =   params

 def __len__( self       :       Optional[Any]		):
       return len(self.dataset		)



 def __getitem__( self       :       List[Any]    ,							_UpperCAmelCase       :       Optional[int]		):
       _A					   =   self.dataset[i]
       _A					   =   self.process(lowerCAmelCase_    ,							**self.params		)
       return processed





class 				lowercase_       (     _lowerCamelCase							):



 '''simple docstring'''

 def __init__( self       :       Any    ,							_UpperCAmelCase       :       Dict    ,							_UpperCAmelCase       :       Optional[Any]    ,							_UpperCAmelCase       :       Union[str, Any]    ,							_UpperCAmelCase       :       List[Any]=None		):
       _A					   =   loader
       _A					   =   infer
       _A					   =   params
       if loader_batch_size == 1:
             # Let's spare some time by deactivating altogether
             _A					   =   None
       _A					   =   loader_batch_size

       # Internal bookkeeping
       _A					   =   None
       _A					   =   None

 def __len__( self       :       List[Any]		):
       return len(self.loader		)

 def __iter__( self       :       List[str]		):
       _A					   =   iter(self.loader		)
       return self

 def 			lowerCAmelCase_							( self       :       Dict		):






       if isinstance(self._loader_batch_data    ,							torch.Tensor		):
             # Batch data is simple tensor, just fetch the slice
             _A					   =   self._loader_batch_data[self._loader_batch_index]
       else:
             # Batch data is assumed to be BaseModelOutput (or dict)
             _A					   =   {}
             for k, element in self._loader_batch_data.items():
                   if isinstance(lowerCAmelCase_    ,							lowerCAmelCase_		):
                         # Convert ModelOutput to tuple first
                         _A					   =   element.to_tuple()
                         if isinstance(element[0]    ,							torch.Tensor		):
                               _A					   =   tuple(el[self._loader_batch_index].unsqueeze(0		) for el in element		)
                         elif isinstance(element[0]    ,							np.ndarray		):
                               _A					   =   tuple(np.expand_dims(el[self._loader_batch_index]    ,							0		) for el in element		)
                         continue
                   if k in {"hidden_states", "past_key_values", "attentions"} and isinstance(lowerCAmelCase_    ,							lowerCAmelCase_		):
                         # Those are stored as lists of tensors so need specific unbatching.
                         if isinstance(element[0]    ,							torch.Tensor		):
                               _A					   =   tuple(el[self._loader_batch_index].unsqueeze(0		) for el in element		)
                         elif isinstance(element[0]    ,							np.ndarray		):
                               _A					   =   tuple(np.expand_dims(el[self._loader_batch_index]    ,							0		) for el in element		)
                         continue
                   if element is None:
                         # This can happen for optional data that get passed around
                         _A					   =   None
                   elif isinstance(element[self._loader_batch_index]    ,							torch.Tensor		):
                         # Take correct batch data, but make it looked like batch_size=1
                         # For compatibility with other methods within transformers

                         _A					   =   element[self._loader_batch_index].unsqueeze(0		)
                   elif isinstance(element[self._loader_batch_index]    ,							np.ndarray		):
                         # Take correct batch data, but make it looked like batch_size=1
                         # For compatibility with other methods within transformers
                         _A					   =   np.expand_dims(element[self._loader_batch_index]    ,							0		)
                   else:
                         # This is typically a list, so no need to `unsqueeze`.
                         _A					   =   element[self._loader_batch_index]
            # Recreate the element by reusing the original class to make it look
            # batch_size=1
             _A					   =   self._loader_batch_data.__class__(lowerCAmelCase_		)
       self._loader_batch_index += 1
       return result



 def 			lowerCAmelCase_							( self       :       List[Any]		):






       if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
             # We are currently unrolling a batch so we just need to return
             # the current item within a batch
             return self.loader_batch_item()

       # We're out of items within a batch
       _A					   =   next(self.iterator		)
       _A					   =   self.infer(lowerCAmelCase_    ,							**self.params		)
       # We now have a batch of "inferred things".
       if self.loader_batch_size is not None:
             # Try to infer the size of the batch
             if isinstance(lowerCAmelCase_    ,							torch.Tensor		):
                   _A					   =   processed
             else:
                   _A					   =   list(processed.keys()		)[0]
                   _A					   =   processed[key]
             if isinstance(lowerCAmelCase_    ,							lowerCAmelCase_		):
                   _A					   =   len(lowerCAmelCase_		)
             else:
                   _A					   =   first_tensor.shape[0]
             if 0 < observed_batch_size < self.loader_batch_size:
                   # could be last batch so we can't unroll as many
                   # elements.
                   _A					   =   observed_batch_size
             # Setting internal index to unwrap the batch
             _A					   =   processed
             _A					   =   0
             return self.loader_batch_item()
       else:
             # We're not unrolling batches
             return processed





class 				lowercase_       (     _lowerCamelCase							):



 '''simple docstring'''

 def __init__( self       :       List[Any]    ,							_UpperCAmelCase       :       Dict    ,							_UpperCAmelCase       :       int    ,							_UpperCAmelCase       :       Optional[Any]    ,							_UpperCAmelCase       :       Optional[Any]=None		):
       super().__init__(lowerCAmelCase_    ,							lowerCAmelCase_    ,							lowerCAmelCase_		)

 def __iter__( self       :       List[str]		):
       _A					   =   iter(self.loader		)
       _A					   =   None
       return self



 def 			lowerCAmelCase_							( self       :       List[str]		):






       if self.subiterator is None:
             _A					   =   self.infer(next(self.iterator		)    ,							**self.params		)
       try:
             # Try to return next item
             _A					   =   next(self.subiterator		)
       except StopIteration:
             # When a preprocess iterator ends, we can start lookig at the next item
             # ChunkIterator will keep feeding until ALL elements of iterator
             # all have created their subiterator and have been iterating against.
             #
             # Another way to look at it, is we're basically flattening lists of lists
             # into a single list, but with generators
             _A					   =   self.infer(next(self.iterator		)    ,							**self.params		)
             _A					   =   next(self.subiterator		)
       return processed





class 				lowercase_       (     _lowerCamelCase							):



 '''simple docstring'''

 def __iter__( self       :       Union[str, Any]		):
       _A					   =   iter(self.loader		)
       return self



 def 			lowerCAmelCase_							( self       :       str		):
       _A					   =   False
       _A					   =   []
       if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
             while self._loader_batch_index < self.loader_batch_size:
                   _A					   =   self.loader_batch_item()
                   _A					   =   item.pop('is_last'		)
                   accumulator.append(lowerCAmelCase_		)
                   if is_last:
                         return accumulator

       while not is_last:
             _A					   =   self.infer(next(self.iterator		)    ,							**self.params		)
             if self.loader_batch_size is not None:
                   if isinstance(lowerCAmelCase_    ,							torch.Tensor		):
                         _A					   =   processed
                   else:
                         _A					   =   list(processed.keys()		)[0]
                         _A					   =   processed[key]
                   if isinstance(lowerCAmelCase_    ,							lowerCAmelCase_		):
                         _A					   =   len(lowerCAmelCase_		)
                   else:
                         _A					   =   first_tensor.shape[0]
                   if 0 < observed_batch_size < self.loader_batch_size:
                         # could be last batch so we can't unroll as many
                         # elements.
                         _A					   =   observed_batch_size
                   _A					   =   processed
                   _A					   =   0
                   while self._loader_batch_index < self.loader_batch_size:
                         _A					   =   self.loader_batch_item()
                         _A					   =   item.pop('is_last'		)
                         accumulator.append(lowerCAmelCase_		)
                         if is_last:
                               return accumulator
             else:
                   _A					   =   processed
                   _A					   =   item.pop('is_last'		)
                   accumulator.append(lowerCAmelCase_		)
       return accumulator





class 				lowercase_       (     _lowerCamelCase							):



 '''simple docstring'''

 def __init__( self       :       Tuple    ,							_UpperCAmelCase       :       Optional[Any]    ,							_UpperCAmelCase       :       List[str]		):
       _A					   =   dataset
       _A					   =   key

 def __len__( self       :       Union[str, Any]		):
       return len(self.dataset		)



 def __getitem__( self       :       Any    ,							_UpperCAmelCase       :       List[str]		):
       return self.dataset[i][self.key]





class 				lowercase_       (     _lowerCamelCase							):



 '''simple docstring'''

 def __init__( self       :       Dict    ,							_UpperCAmelCase       :       Union[str, Any]    ,							_UpperCAmelCase       :       List[Any]    ,							_UpperCAmelCase       :       Union[str, Any]		):
       _A					   =   dataset
       _A					   =   keya
       _A					   =   keya

 def __len__( self       :       Union[str, Any]		):
       return len(self.dataset		)



 def __getitem__( self       :       Dict    ,							_UpperCAmelCase       :       str		):
       return {"text": self.dataset[i][self.keya], "text_pair": self.dataset[i][self.keya]}



'''simple docstring'''







import subprocess
import sys

from transformers import BertConfig, BertModel, BertTokenizer, pipeline
from transformers.testing_utils import TestCasePlus, require_torch


class 						__UpperCAmelCase      (					_lowerCamelCase						):


							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

								_snake_case   						=		'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

								_snake_case   						=		'\nimport socket\ndef offline_socket(*args, **kwargs): raise RuntimeError("Offline mode is enabled, we shouldn\'t access internet")\nsocket.socket = offline_socket\n        '

								# Force fetching the files so that we can use the cache
								_snake_case   						=		'hf-internal-testing/tiny-random-bert'
								BertConfig.from_pretrained(lowerCAmelCase_  )
								BertModel.from_pretrained(lowerCAmelCase_  )
								BertTokenizer.from_pretrained(lowerCAmelCase_  )
								pipeline(task='fill-mask'    ,   model=lowerCAmelCase_  )

								# baseline - just load from_pretrained with normal network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, run, mock]  )]

								# should succeed
								_snake_case   						=		self.get_env()
								# should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
								_snake_case   						=		'1'
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )


							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

								_snake_case   						=		'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

								_snake_case   						=		'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Faking flaky internet")\nsocket.socket = offline_socket\n        '

								# Force fetching the files so that we can use the cache
								_snake_case   						=		'hf-internal-testing/tiny-random-bert'
								BertConfig.from_pretrained(lowerCAmelCase_  )
								BertModel.from_pretrained(lowerCAmelCase_  )
								BertTokenizer.from_pretrained(lowerCAmelCase_  )
								pipeline(task='fill-mask'    ,   model=lowerCAmelCase_  )

								# baseline - just load from_pretrained with normal network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, run, mock]  )]

								# should succeed
								_snake_case   						=		self.get_env()
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )


							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import BertConfig, BertModel, BertTokenizer\n        '

								_snake_case   						=		'\nmname = "hf-internal-testing/tiny-random-bert-sharded"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nprint("success")\n        '

								_snake_case   						=		'\nimport socket\ndef offline_socket(*args, **kwargs): raise ValueError("Offline mode is enabled")\nsocket.socket = offline_socket\n        '

								# baseline - just load from_pretrained with normal network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, run]  )]

								# should succeed
								_snake_case   						=		self.get_env()
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )

								# next emulate no network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, mock, run]  )]

								# Doesn't fail anymore since the model is in the cache due to other tests, so commenting this.
								# env["TRANSFORMERS_OFFLINE"] = "0"
								# result = subprocess.run(cmd, env=env, check=False, capture_output=True)
								# self.assertEqual(result.returncode, 1, result.stderr)

								# should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
								_snake_case   						=		'1'
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )


							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import pipeline\n        '
								_snake_case   						=		'\nmname = "hf-internal-testing/tiny-random-bert"\npipe = pipeline(model=mname)\n        '

								_snake_case   						=		'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Offline mode is enabled")\nsocket.socket = offline_socket\n        '
								_snake_case   						=		self.get_env()
								_snake_case   						=		'1'
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, mock, run]  )]
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   1    ,   result.stderr  )
								self.assertIn(
								    'You cannot infer task automatically within `pipeline` when using offline mode'    ,   result.stderr.decode().replace('\n'    ,   ''  )    ,   )




							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import AutoModel\n        '
								_snake_case   						=		'\nmname = "hf-internal-testing/test_dynamic_model"\nAutoModel.from_pretrained(mname, trust_remote_code=True)\nprint("success")\n        '

								# baseline - just load from_pretrained with normal network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, run]  )]

								# should succeed
								_snake_case   						=		self.get_env()
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )

								# should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
								_snake_case   						=		'1'
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )

