| 150 | |
| 151 | |
| 152 | def main(): |
| 153 | parser = argparse.ArgumentParser() |
| 154 | parser.add_argument("--ports", default="8000,8002", type=str, help="ports to the http server") |
| 155 | parser.add_argument("--num-servers", default=2, type=int, help="number of workers") |
| 156 | parser.add_argument("--metrics-ports", default="-1", type=str, help="ports for metrics server") |
| 157 | parser.add_argument("--controller-ports", default="-1", type=str, help="ports for controller server port") |
| 158 | parser.add_argument("--args", nargs=argparse.REMAINDER, help="remaining arguments are passed to api_server.py") |
| 159 | args = parser.parse_args() |
| 160 | |
| 161 | logger.info(f"Launching MultiAPIServer with command: {' '.join(sys.argv)}") |
| 162 | |
| 163 | device_count = 0 |
| 164 | if current_platform.is_cuda(): |
| 165 | if os.getenv("CUDA_VISIBLE_DEVICES") is None: |
| 166 | raise ValueError("Please manually set CUDA_VISIBLE_DEVICES when launching multi-api-server.") |
| 167 | device_count = len(os.getenv("CUDA_VISIBLE_DEVICES").split(",")) |
| 168 | elif current_platform.is_xpu(): |
| 169 | if os.getenv("XPU_VISIBLE_DEVICES") is None: |
| 170 | raise ValueError("Please manually set XPU_VISIBLE_DEVICES when launching multi-api-server.") |
| 171 | device_count = len(os.getenv("XPU_VISIBLE_DEVICES").split(",")) |
| 172 | |
| 173 | processes = start_servers( |
| 174 | server_count=args.num_servers, |
| 175 | device_count=device_count, |
| 176 | server_args=args.args, |
| 177 | ports=args.ports, |
| 178 | metrics_ports=args.metrics_ports, |
| 179 | controller_ports=args.controller_ports, |
| 180 | ) |
| 181 | |
| 182 | try: |
| 183 | while True: |
| 184 | time.sleep(1) |
| 185 | except KeyboardInterrupt: |
| 186 | for proc in processes: |
| 187 | proc.terminate() |
| 188 | for proc in processes: |
| 189 | proc.wait() |
| 190 | logger.info("All servers stopped.") |
| 191 | |
| 192 | |
| 193 | if __name__ == "__main__": |