From ea2d5feba916327c8323360b4f8a619ba7bcb8e3 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 13:39:00 +0530 Subject: [PATCH 01/29] feature: implement websocket streaming for inputs Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 49 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 67a32d80a..30c8ae1e9 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -22,29 +22,51 @@ from rich.console import Console from rich.live import Live from rich.markdown import Markdown +import asyncio +import websockets from fastchat.model.model_adapter import add_model_args from fastchat.modules.gptq import GptqConfig from fastchat.serve.inference import ChatIO, chat_loop +# async def chatbot_websocket_client(): +# uri = "ws://your-golang-websocket-server-address/ws" +# async with websockets.connect(uri) as websocket: +# while True: +# user_input = await websocket.recv() +# response = "User Input we got" + user_input +# print(response) +# print(user_input) +# # This will further relay the message to WebSocket Gin Sentinel Server. +# await websocket.send(response) + class SimpleChatIO(ChatIO): - def __init__(self, multiline: bool = False): + def __init__(self, websocket, multiline: bool = False): self._multiline = multiline + self.websocket = websocket def prompt_for_input(self, role) -> str: if not self._multiline: return input(f"{role}: ") prompt_data = [] - line = input(f"{role} [ctrl-d/z on empty line to end]: ") + # line = input(f"{role} [ctrl-d/z on empty line to end]: ") + line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") while True: prompt_data.append(line.strip()) try: - line = input() + line = self.receive_input_from_websocket() except EOFError as e: break return "\n".join(prompt_data) + + async def receive_input_from_websocket(self,prompt=None)-> str: + if prompt: + self.send_message_to_websocket(prompt) + user_input = await self.websocket.recv() # Receive input from WebSocket client + print(user_input) + return user_input def prompt_for_output(self, role: str): print(f"{role}: ", end="", flush=True) @@ -159,7 +181,22 @@ def stream_output(self, output_stream): return " ".join(output_text) -def main(args): +# async def chatbot_websocket_client(): +# # URI to Point to the Proxy WebSocket Gin Server so that it can relay the message: +# uri = "wss://VM_PUBLIC_IP:8080/ws" +# async with websockets.connect() as websocket: +# while True: +# user_input = await websocket.recv() +# response = "User Input we got" + user_input +# print(response) +# print(user_input) +# # This will further relay the message to WebSocket Gin Sentinel Server. +# await websocket.send(response) + + +async def main(args): + # First of all we retrieve the Event Loop + # asyncio.get_event_loop().run_until_complete(chatbot_websocket_client()) if args.gpus: if len(args.gpus.split(",")) < args.num_gpus: raise ValueError( @@ -169,7 +206,8 @@ def main(args): os.environ["XPU_VISIBLE_DEVICES"] = args.gpus if args.style == "simple": - chatio = SimpleChatIO(args.multiline) + websocket = await websockets.connect("ws://your-proxy-server-address") + chatio = SimpleChatIO(websocket,args.multiline) elif args.style == "rich": chatio = RichChatIO(args.multiline, args.mouse) elif args.style == "programmatic": @@ -180,6 +218,7 @@ def main(args): chatio = FileInputChatIO(root_input_file_path) else: raise ValueError(f"Invalid style for console: {args.style}") + try: chat_loop( args.model_path, From 3428ff58ba00a89b0bf732c9f044b63b2634a08a Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 13:45:03 +0530 Subject: [PATCH 02/29] feature: implement event loop for asynch execution of main() Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 30c8ae1e9..8d0db69c1 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -248,6 +248,9 @@ async def main(args): if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + loop.close() parser = argparse.ArgumentParser() add_model_args(parser) parser.add_argument( From fa2eec42249b78a7906afd104715669a6523e3b9 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 13:46:14 +0530 Subject: [PATCH 03/29] feature: update websockets connection URI Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 8d0db69c1..d5f43c331 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -206,7 +206,7 @@ async def main(args): os.environ["XPU_VISIBLE_DEVICES"] = args.gpus if args.style == "simple": - websocket = await websockets.connect("ws://your-proxy-server-address") + websocket = await websockets.connect("wss://35.209.170.184:8080/ws") chatio = SimpleChatIO(websocket,args.multiline) elif args.style == "rich": chatio = RichChatIO(args.multiline, args.mouse) From 4110e713532f8719d6497b910a8bd2c47ce76a61 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 13:49:00 +0530 Subject: [PATCH 04/29] argument parsing Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index d5f43c331..cac31f223 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -248,11 +248,11 @@ async def main(args): if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) - loop.close() parser = argparse.ArgumentParser() add_model_args(parser) + loop = asyncio.get_event_loop() + loop.run_until_complete(main(args=parser.parse_args())) + loop.close() parser.add_argument( "--conv-template", type=str, default=None, help="Conversation prompt template." ) From e3620e44a74d120c6380f60f0a408d35f8b5ee9c Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 14:14:42 +0530 Subject: [PATCH 05/29] modifications Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index cac31f223..e5e4370c5 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -30,16 +30,16 @@ from fastchat.serve.inference import ChatIO, chat_loop -# async def chatbot_websocket_client(): -# uri = "ws://your-golang-websocket-server-address/ws" -# async with websockets.connect(uri) as websocket: -# while True: -# user_input = await websocket.recv() -# response = "User Input we got" + user_input -# print(response) -# print(user_input) -# # This will further relay the message to WebSocket Gin Sentinel Server. -# await websocket.send(response) +async def chatbot_websocket_client(): + uri = "ws://your-golang-websocket-server-address/ws" + async with websockets.connect(uri) as websocket: + while True: + user_input = await websocket.recv() + response = "User Input we got" + user_input + print(response) + print(user_input) + # This will further relay the message to WebSocket Gin Sentinel Server. + await websocket.send(response) class SimpleChatIO(ChatIO): def __init__(self, websocket, multiline: bool = False): @@ -206,7 +206,7 @@ async def main(args): os.environ["XPU_VISIBLE_DEVICES"] = args.gpus if args.style == "simple": - websocket = await websockets.connect("wss://35.209.170.184:8080/ws") + websocket = await websockets.connect("ws://your-proxy-server-address") chatio = SimpleChatIO(websocket,args.multiline) elif args.style == "rich": chatio = RichChatIO(args.multiline, args.mouse) @@ -250,9 +250,6 @@ async def main(args): if __name__ == "__main__": parser = argparse.ArgumentParser() add_model_args(parser) - loop = asyncio.get_event_loop() - loop.run_until_complete(main(args=parser.parse_args())) - loop.close() parser.add_argument( "--conv-template", type=str, default=None, help="Conversation prompt template." ) From 10f7920c67d0103d0941e517516277aeffd5e921 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 14:17:01 +0530 Subject: [PATCH 06/29] async await changes Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index e5e4370c5..93037c4ab 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -194,7 +194,7 @@ def stream_output(self, output_stream): # await websocket.send(response) -async def main(args): +def main(args): # First of all we retrieve the Event Loop # asyncio.get_event_loop().run_until_complete(chatbot_websocket_client()) if args.gpus: @@ -206,7 +206,7 @@ async def main(args): os.environ["XPU_VISIBLE_DEVICES"] = args.gpus if args.style == "simple": - websocket = await websockets.connect("ws://your-proxy-server-address") + websocket = websockets.connect("ws://your-proxy-server-address") chatio = SimpleChatIO(websocket,args.multiline) elif args.style == "rich": chatio = RichChatIO(args.multiline, args.mouse) From de41feb47d39362e265f5fc56f820499d191ed62 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 14:18:09 +0530 Subject: [PATCH 07/29] modify websocket server URI Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 93037c4ab..bbc0a648e 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -206,7 +206,7 @@ def main(args): os.environ["XPU_VISIBLE_DEVICES"] = args.gpus if args.style == "simple": - websocket = websockets.connect("ws://your-proxy-server-address") + websocket = websockets.connect("wss://35.209.170.184:8080/ws") chatio = SimpleChatIO(websocket,args.multiline) elif args.style == "rich": chatio = RichChatIO(args.multiline, args.mouse) From c9d3385c64b27c220c8ca46c3af7a5d4c34dfc20 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 14:57:07 +0530 Subject: [PATCH 08/29] Modification to input streaming handler over WebSockets Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index bbc0a648e..646258e06 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -46,17 +46,32 @@ def __init__(self, websocket, multiline: bool = False): self._multiline = multiline self.websocket = websocket + async def chat_websocket_client()-> str: + # URI to Point to the Proxy WebSocket Gin Server so that it can relay the message: + uri = "wss://35.209.170.184:8080/ws" + async with websockets.connect() as websocket: + while True: + user_input = await websocket.recv() + response = "User Input we got" + user_input + print(response) + print(user_input) + # This will further relay the message to WebSocket Gin Sentinel Server. + # await websocket.send(response) + return response + + def prompt_for_input(self, role) -> str: if not self._multiline: return input(f"{role}: ") prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") - line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") + # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") + line = self.chat_websocket_client() while True: prompt_data.append(line.strip()) try: - line = self.receive_input_from_websocket() + line = self.chat_websocket_client() except EOFError as e: break return "\n".join(prompt_data) @@ -181,22 +196,9 @@ def stream_output(self, output_stream): return " ".join(output_text) -# async def chatbot_websocket_client(): -# # URI to Point to the Proxy WebSocket Gin Server so that it can relay the message: -# uri = "wss://VM_PUBLIC_IP:8080/ws" -# async with websockets.connect() as websocket: -# while True: -# user_input = await websocket.recv() -# response = "User Input we got" + user_input -# print(response) -# print(user_input) -# # This will further relay the message to WebSocket Gin Sentinel Server. -# await websocket.send(response) - - def main(args): # First of all we retrieve the Event Loop - # asyncio.get_event_loop().run_until_complete(chatbot_websocket_client()) + asyncio.get_event_loop().run_until_complete(chatbot_websocket_client()) if args.gpus: if len(args.gpus.split(",")) < args.num_gpus: raise ValueError( From 66fcb1873182a870627059bff85918fff449312f Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 15:01:26 +0530 Subject: [PATCH 09/29] embed websockets connection uri Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 646258e06..f67c726d0 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -30,16 +30,16 @@ from fastchat.serve.inference import ChatIO, chat_loop -async def chatbot_websocket_client(): - uri = "ws://your-golang-websocket-server-address/ws" - async with websockets.connect(uri) as websocket: - while True: - user_input = await websocket.recv() - response = "User Input we got" + user_input - print(response) - print(user_input) - # This will further relay the message to WebSocket Gin Sentinel Server. - await websocket.send(response) +# async def chatbot_websocket_client(): +# uri = "ws://your-golang-websocket-server-address/ws" +# async with websockets.connect(uri) as websocket: +# while True: +# user_input = await websocket.recv() +# response = "User Input we got" + user_input +# print(response) +# print(user_input) +# # This will further relay the message to WebSocket Gin Sentinel Server. +# await websocket.send(response) class SimpleChatIO(ChatIO): def __init__(self, websocket, multiline: bool = False): @@ -49,7 +49,7 @@ def __init__(self, websocket, multiline: bool = False): async def chat_websocket_client()-> str: # URI to Point to the Proxy WebSocket Gin Server so that it can relay the message: uri = "wss://35.209.170.184:8080/ws" - async with websockets.connect() as websocket: + async with websockets.connect(uri) as websocket: while True: user_input = await websocket.recv() response = "User Input we got" + user_input @@ -76,12 +76,12 @@ def prompt_for_input(self, role) -> str: break return "\n".join(prompt_data) - async def receive_input_from_websocket(self,prompt=None)-> str: - if prompt: - self.send_message_to_websocket(prompt) - user_input = await self.websocket.recv() # Receive input from WebSocket client - print(user_input) - return user_input + # async def receive_input_from_websocket(self,prompt=None)-> str: + # if prompt: + # self.send_message_to_websocket(prompt) + # user_input = await self.websocket.recv() # Receive input from WebSocket client + # print(user_input) + # return user_input def prompt_for_output(self, role: str): print(f"{role}: ", end="", flush=True) @@ -198,7 +198,7 @@ def stream_output(self, output_stream): def main(args): # First of all we retrieve the Event Loop - asyncio.get_event_loop().run_until_complete(chatbot_websocket_client()) + asyncio.get_event_loop().run_until_complete(SimpleChatIO.chat_websocket_client()) if args.gpus: if len(args.gpus.split(",")) < args.num_gpus: raise ValueError( From 994eebce9bf57d3471d2ee7b6626c5235e80ff9b Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 15:04:30 +0530 Subject: [PATCH 10/29] temporary ssl validation removal Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index f67c726d0..9687e8359 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -13,6 +13,7 @@ import os import re import sys +import ssl from prompt_toolkit import PromptSession from prompt_toolkit.auto_suggest import AutoSuggestFromHistory @@ -30,16 +31,7 @@ from fastchat.serve.inference import ChatIO, chat_loop -# async def chatbot_websocket_client(): -# uri = "ws://your-golang-websocket-server-address/ws" -# async with websockets.connect(uri) as websocket: -# while True: -# user_input = await websocket.recv() -# response = "User Input we got" + user_input -# print(response) -# print(user_input) -# # This will further relay the message to WebSocket Gin Sentinel Server. -# await websocket.send(response) + class SimpleChatIO(ChatIO): def __init__(self, websocket, multiline: bool = False): @@ -49,7 +41,7 @@ def __init__(self, websocket, multiline: bool = False): async def chat_websocket_client()-> str: # URI to Point to the Proxy WebSocket Gin Server so that it can relay the message: uri = "wss://35.209.170.184:8080/ws" - async with websockets.connect(uri) as websocket: + async with websockets.connect(uri, ssl=ssl.SSLContext()) as websocket: while True: user_input = await websocket.recv() response = "User Input we got" + user_input @@ -76,12 +68,6 @@ def prompt_for_input(self, role) -> str: break return "\n".join(prompt_data) - # async def receive_input_from_websocket(self,prompt=None)-> str: - # if prompt: - # self.send_message_to_websocket(prompt) - # user_input = await self.websocket.recv() # Receive input from WebSocket client - # print(user_input) - # return user_input def prompt_for_output(self, role: str): print(f"{role}: ", end="", flush=True) From d1fd064e4427a52b1f3d3414a7ecfc13283637cc Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 15:22:43 +0530 Subject: [PATCH 11/29] role bindings for fashchat Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 9687e8359..ccefbc26d 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -59,7 +59,7 @@ def prompt_for_input(self, role) -> str: prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") - line = self.chat_websocket_client() + line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") while True: prompt_data.append(line.strip()) try: From 933acef1db102a6da614ecaea687073a142f0c42 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 15:30:03 +0530 Subject: [PATCH 12/29] sending response back to websocket server Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index ccefbc26d..d2e196859 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -48,7 +48,7 @@ async def chat_websocket_client()-> str: print(response) print(user_input) # This will further relay the message to WebSocket Gin Sentinel Server. - # await websocket.send(response) + await websocket.send(response) return response From 5d3898dc10e9f9043fb7c138a29a91ea29b3f0e5 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 15:44:14 +0530 Subject: [PATCH 13/29] temporarily removing response return Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index d2e196859..0e095ce32 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -49,7 +49,7 @@ async def chat_websocket_client()-> str: print(user_input) # This will further relay the message to WebSocket Gin Sentinel Server. await websocket.send(response) - return response + # return response def prompt_for_input(self, role) -> str: From 6ae53ac1072d08a3ebd9ee5979f29a0531bd06c8 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 15:54:35 +0530 Subject: [PATCH 14/29] temp return statement Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 0e095ce32..528560bdc 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -66,7 +66,9 @@ def prompt_for_input(self, role) -> str: line = self.chat_websocket_client() except EOFError as e: break - return "\n".join(prompt_data) + # return "\n".join(prompt_data) + + def prompt_for_output(self, role: str): From 6426e5e9c4e05c421f4443eb5e4b624f8948bc49 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 16:01:28 +0530 Subject: [PATCH 15/29] putting back the return statement to avoid infinite loop Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 528560bdc..708fd8879 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -66,7 +66,7 @@ def prompt_for_input(self, role) -> str: line = self.chat_websocket_client() except EOFError as e: break - # return "\n".join(prompt_data) + return "\n".join(prompt_data) From 42898d47471e21a1a82b18f5693a2940d4e28682 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 16:05:00 +0530 Subject: [PATCH 16/29] indentation change Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 708fd8879..e554e6206 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -66,7 +66,7 @@ def prompt_for_input(self, role) -> str: line = self.chat_websocket_client() except EOFError as e: break - return "\n".join(prompt_data) + return "\n".join(prompt_data) From b788cec67a2caf270fdcdeb06ce65382896cd0b0 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 16:06:48 +0530 Subject: [PATCH 17/29] further changes Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index e554e6206..d2e196859 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -49,7 +49,7 @@ async def chat_websocket_client()-> str: print(user_input) # This will further relay the message to WebSocket Gin Sentinel Server. await websocket.send(response) - # return response + return response def prompt_for_input(self, role) -> str: @@ -66,9 +66,7 @@ def prompt_for_input(self, role) -> str: line = self.chat_websocket_client() except EOFError as e: break - return "\n".join(prompt_data) - - + return "\n".join(prompt_data) def prompt_for_output(self, role: str): From 1f151221009eba6e43655efa0c428ce83619c570 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 16:13:34 +0530 Subject: [PATCH 18/29] changing function signature for prompt for input method Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index d2e196859..0b06ffbf7 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -52,14 +52,15 @@ async def chat_websocket_client()-> str: return response - def prompt_for_input(self, role) -> str: + async def prompt_for_input(self, role) -> str: if not self._multiline: return input(f"{role}: ") prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") - line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") + # line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") + line = await self.websocket.recv() while True: prompt_data.append(line.strip()) try: From a3a1f0dd6ec13175dd58e8b3e38a6b05a1cce627 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 16:14:19 +0530 Subject: [PATCH 19/29] amendment to prompt for input method Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 0b06ffbf7..0fe49d073 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -64,7 +64,7 @@ async def prompt_for_input(self, role) -> str: while True: prompt_data.append(line.strip()) try: - line = self.chat_websocket_client() + line = self.websocket.recv() except EOFError as e: break return "\n".join(prompt_data) From 9bae78915ba521a22da1e96015e7bbec58d09e6c Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 18:20:18 +0530 Subject: [PATCH 20/29] changign method signature to await() method Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 0fe49d073..35149d965 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -49,7 +49,7 @@ async def chat_websocket_client()-> str: print(user_input) # This will further relay the message to WebSocket Gin Sentinel Server. await websocket.send(response) - return response + # return response async def prompt_for_input(self, role) -> str: @@ -59,12 +59,11 @@ async def prompt_for_input(self, role) -> str: prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") - # line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") - line = await self.websocket.recv() + line = await self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") while True: prompt_data.append(line.strip()) try: - line = self.websocket.recv() + line = await self.chat_websocket_client() except EOFError as e: break return "\n".join(prompt_data) From cb896e563f41378c248f6ffa8433e467ec2f7e7b Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 18:22:13 +0530 Subject: [PATCH 21/29] changes Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 35149d965..dcc212142 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -59,11 +59,11 @@ async def prompt_for_input(self, role) -> str: prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") - line = await self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") + line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") while True: prompt_data.append(line.strip()) try: - line = await self.chat_websocket_client() + line = self.websocket.recv() except EOFError as e: break return "\n".join(prompt_data) From e143d7092fb8a203f3b60c146cfa5477aaf66d7b Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 18:24:23 +0530 Subject: [PATCH 22/29] further changes Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index dcc212142..f8649109b 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -59,11 +59,12 @@ async def prompt_for_input(self, role) -> str: prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") - line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") + # line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") + line = await self.websocket.recv() while True: prompt_data.append(line.strip()) try: - line = self.websocket.recv() + line = await self.websocket.recv() except EOFError as e: break return "\n".join(prompt_data) From b078c5db7034cd97a89abc198e86958ebc2cf1c3 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 18:27:16 +0530 Subject: [PATCH 23/29] changing method signature for chat_websocket_client() Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index f8649109b..008319997 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -59,12 +59,12 @@ async def prompt_for_input(self, role) -> str: prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") - # line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") - line = await self.websocket.recv() + line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") + # line = await self.websocket.recv() while True: prompt_data.append(line.strip()) try: - line = await self.websocket.recv() + line = self.chat_websocket_client() except EOFError as e: break return "\n".join(prompt_data) @@ -73,7 +73,7 @@ async def prompt_for_input(self, role) -> str: def prompt_for_output(self, role: str): print(f"{role}: ", end="", flush=True) - def stream_output(self, output_stream): + async def stream_output(self, output_stream): pre = 0 for outputs in output_stream: output_text = outputs["text"] @@ -83,6 +83,7 @@ def stream_output(self, output_stream): print(" ".join(output_text[pre:now]), end=" ", flush=True) pre = now print(" ".join(output_text[pre:]), flush=True) + await websockets.send(" ".join(output_text[pre:])) return " ".join(output_text) From 1e65e601aac4fa410cea4c029ba7448d2fa71be0 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 18:28:50 +0530 Subject: [PATCH 24/29] removing await() Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 008319997..1865b37e4 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -83,7 +83,7 @@ async def stream_output(self, output_stream): print(" ".join(output_text[pre:now]), end=" ", flush=True) pre = now print(" ".join(output_text[pre:]), flush=True) - await websockets.send(" ".join(output_text[pre:])) + # await websockets.send(" ".join(output_text[pre:])) return " ".join(output_text) From 0a50b89d96ef429e028c5ea3b02016b512e1350e Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 18:30:50 +0530 Subject: [PATCH 25/29] reverting back changes Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 1865b37e4..d2c8cc741 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -59,12 +59,12 @@ async def prompt_for_input(self, role) -> str: prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") - line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") - # line = await self.websocket.recv() + # line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") + line = await self.websocket.recv() while True: prompt_data.append(line.strip()) try: - line = self.chat_websocket_client() + line = self.websocket.recv() except EOFError as e: break return "\n".join(prompt_data) @@ -73,7 +73,7 @@ async def prompt_for_input(self, role) -> str: def prompt_for_output(self, role: str): print(f"{role}: ", end="", flush=True) - async def stream_output(self, output_stream): + def stream_output(self, output_stream): pre = 0 for outputs in output_stream: output_text = outputs["text"] @@ -83,7 +83,6 @@ async def stream_output(self, output_stream): print(" ".join(output_text[pre:now]), end=" ", flush=True) pre = now print(" ".join(output_text[pre:]), flush=True) - # await websockets.send(" ".join(output_text[pre:])) return " ".join(output_text) From 99e78e0ae297470c7cb085d1c7860f00a7a970d4 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 12 Sep 2023 18:33:10 +0530 Subject: [PATCH 26/29] changing the method type to receive output Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index d2c8cc741..0e095ce32 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -52,19 +52,18 @@ async def chat_websocket_client()-> str: # return response - async def prompt_for_input(self, role) -> str: + def prompt_for_input(self, role) -> str: if not self._multiline: return input(f"{role}: ") prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") - # line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") - line = await self.websocket.recv() + line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") while True: prompt_data.append(line.strip()) try: - line = self.websocket.recv() + line = self.chat_websocket_client() except EOFError as e: break return "\n".join(prompt_data) From 16a4d9fda2f3c8d1673a1b3d9df06c61f966da82 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 19 Sep 2023 22:14:19 +0530 Subject: [PATCH 27/29] latest changes Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 0e095ce32..c209d4b55 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -38,29 +38,35 @@ def __init__(self, websocket, multiline: bool = False): self._multiline = multiline self.websocket = websocket - async def chat_websocket_client()-> str: + async def chat_websocket_client(self)-> str: # URI to Point to the Proxy WebSocket Gin Server so that it can relay the message: uri = "wss://35.209.170.184:8080/ws" async with websockets.connect(uri, ssl=ssl.SSLContext()) as websocket: while True: - user_input = await websocket.recv() - response = "User Input we got" + user_input - print(response) - print(user_input) - # This will further relay the message to WebSocket Gin Sentinel Server. - await websocket.send(response) - # return response + try: + user_input = await websocket.recv() + response = "User Input we got" + user_input + print(response) + print(user_input) + await websocket.send(response) + task = asyncio.create_task(self.prompt_for_input("USER")) + return user_input + except websockets.exceptions.ConnectionClosed: + print("Connection Closed") + + - def prompt_for_input(self, role) -> str: + + async def prompt_for_input(self, role) -> str: if not self._multiline: return input(f"{role}: ") prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ") # line = self.receive_input_from_websocket(role + f" [ctrl-d/z on empty line to end]: ") - line = self.chat_websocket_client(role + f" [ctrl-d/z on empty line to end]: ") - while True: + line = await self.chat_websocket_client() + while line: prompt_data.append(line.strip()) try: line = self.chat_websocket_client() @@ -184,7 +190,7 @@ def stream_output(self, output_stream): def main(args): # First of all we retrieve the Event Loop - asyncio.get_event_loop().run_until_complete(SimpleChatIO.chat_websocket_client()) + # asyncio.get_event_loop().run_until_complete(SimpleChatIO.chat_websocket_client()) if args.gpus: if len(args.gpus.split(",")) < args.num_gpus: raise ValueError( @@ -196,6 +202,8 @@ def main(args): if args.style == "simple": websocket = websockets.connect("wss://35.209.170.184:8080/ws") chatio = SimpleChatIO(websocket,args.multiline) + # asyncio.get_event_loop().run_until_complete(SimpleChatIO.chat_websocket_client()) + asyncio.run(chatio.chat_websocket_client()) elif args.style == "rich": chatio = RichChatIO(args.multiline, args.mouse) elif args.style == "programmatic": From 9e8740aae824136d46dc28afa1f92a825151821d Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 19 Sep 2023 22:16:02 +0530 Subject: [PATCH 28/29] removing the infinite loop Removing the while true infinite loop since with asyncio.run() it is able to stream in the inputs from the WebSocket Server and looks for inputs since it is constantly running in the background using the event loop Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index c209d4b55..473cf00d8 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -42,7 +42,7 @@ async def chat_websocket_client(self)-> str: # URI to Point to the Proxy WebSocket Gin Server so that it can relay the message: uri = "wss://35.209.170.184:8080/ws" async with websockets.connect(uri, ssl=ssl.SSLContext()) as websocket: - while True: + # while True: try: user_input = await websocket.recv() response = "User Input we got" + user_input From ae43ad9e774a064c6c282c34c0891d97ef4d0e24 Mon Sep 17 00:00:00 2001 From: Deepanshu Arora Date: Tue, 19 Sep 2023 22:22:25 +0530 Subject: [PATCH 29/29] removing multiline inputs Signed-off-by: Deepanshu Arora --- fastchat/serve/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 473cf00d8..d395a1521 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -59,8 +59,8 @@ async def chat_websocket_client(self)-> str: async def prompt_for_input(self, role) -> str: - if not self._multiline: - return input(f"{role}: ") + # if not self._multiline: + # return input(f"{role}: ") prompt_data = [] # line = input(f"{role} [ctrl-d/z on empty line to end]: ")