Get or load model and tokenizer with minimal locking.
(self, model_key: str, loader_fn)
| 847 | logger.info("CacheManager singleton initialized") |
| 848 | |
| 849 | def get_or_load_model(self, model_key: str, loader_fn) -> Tuple[Any, Any]: |
| 850 | """Get or load model and tokenizer with minimal locking.""" |
| 851 | cached_model = cached_tokenizer = None |
| 852 | cache_hit = False |
| 853 | |
| 854 | with self._lock: |
| 855 | if model_key in self.model_cache and model_key in self.tokenizer_cache: |
| 856 | cached_model = self.model_cache[model_key] |
| 857 | cached_tokenizer = self.tokenizer_cache[model_key] |
| 858 | self.model_cache.move_to_end(model_key) |
| 859 | self.tokenizer_cache.move_to_end(model_key) |
| 860 | self.cache_stats[model_key]["hits"] += 1 |
| 861 | cache_hit = True |
| 862 | logger.debug(f"Cache hit for model: {model_key}") |
| 863 | |
| 864 | if cache_hit: |
| 865 | return cached_model, cached_tokenizer |
| 866 | |
| 867 | logger.info(f"Loading model and tokenizer: {model_key}") |
| 868 | model, tokenizer = loader_fn() |
| 869 | |
| 870 | with self._lock: |
| 871 | if model_key in self.model_cache and model_key in self.tokenizer_cache: |
| 872 | cached_model = self.model_cache[model_key] |
| 873 | cached_tokenizer = self.tokenizer_cache[model_key] |
| 874 | self.cache_stats[model_key]["hits"] += 1 |
| 875 | logger.debug(f"Using already cached model: {model_key}") |
| 876 | return cached_model, cached_tokenizer |
| 877 | |
| 878 | self.model_cache[model_key] = model |
| 879 | self.tokenizer_cache[model_key] = tokenizer |
| 880 | self.cache_stats[model_key]["misses"] += 1 |
| 881 | self.model_adapter_map[model_key] = [] # Initialize empty adapter list for new model |
| 882 | |
| 883 | self._cleanup_caches() |
| 884 | |
| 885 | logger.info(f"Successfully cached model and tokenizer: {model_key}") |
| 886 | return model, tokenizer |
| 887 | |
| 888 | def get_or_load_adapter(self, model_key: str, adapter_key: str, loader_fn): |
| 889 | """Get or load adapter with enhanced caching.""" |
no test coverage detected