()
| 46 | |
| 47 | |
| 48 | def _load_plugin_lib(): |
| 49 | on_windows = platform.system() == "Windows" |
| 50 | winmode = 0 if on_windows else None |
| 51 | handle = ctypes.CDLL(plugin_lib_path(), |
| 52 | mode=ctypes.RTLD_GLOBAL, |
| 53 | winmode=winmode) |
| 54 | try: |
| 55 | handle.initTrtLlmPlugins.argtypes = [ctypes.c_void_p, ctypes.c_char_p] |
| 56 | handle.initTrtLlmPlugins.restype = ctypes.c_bool |
| 57 | except AttributeError as err: |
| 58 | raise ImportError('TensorRT LLM Plugin is unavailable') from err |
| 59 | |
| 60 | try: |
| 61 | assert handle.initTrtLlmPlugins( |
| 62 | None, TRT_LLM_PLUGIN_NAMESPACE.encode('utf-8')) |
| 63 | except OSError as e: |
| 64 | windows_err = """ |
| 65 | The error above may be caused by an outdated Microsoft Visual C++ Redistributable Version. |
| 66 | Please install the latest MSVC from the link below and re-launch. |
| 67 | |
| 68 | https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170#latest-microsoft-visual-c-redistributable-version |
| 69 | """ |
| 70 | err_msg = dedent(windows_err if on_windows else "Unknown error") |
| 71 | raise RuntimeError(err_msg) from e |
| 72 | except Exception as e: |
| 73 | raise e |
| 74 | |
| 75 | |
| 76 | class ContextFMHAType(IntEnum): |
no test coverage detected