From a3ecabb00e728d2ed63ce613d6f5b5902780d45c Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Sat, 23 Sep 2023 11:42:30 +0200 Subject: Improve Vercel Provider: - Fix endless loop - Add proxy, async support - Add default model Fix HuggingChat Provider --- g4f/Provider/HuggingChat.py | 2 +- g4f/Provider/Vercel.py | 102 +++++++++++++++++++++++++----------------- g4f/Provider/base_provider.py | 3 +- g4f/__init__.py | 2 +- testing/test_providers.py | 3 +- 5 files changed, 65 insertions(+), 47 deletions(-) diff --git a/g4f/Provider/HuggingChat.py b/g4f/Provider/HuggingChat.py index 7702c9dd..b2cf9793 100644 --- a/g4f/Provider/HuggingChat.py +++ b/g4f/Provider/HuggingChat.py @@ -9,7 +9,7 @@ from .base_provider import AsyncGeneratorProvider, format_prompt, get_cookies class HuggingChat(AsyncGeneratorProvider): - url = "https://huggingface.co/chat/" + url = "https://huggingface.co/chat" needs_auth = True working = True model = "OpenAssistant/oasst-sft-6-llama-30b-xor" diff --git a/g4f/Provider/Vercel.py b/g4f/Provider/Vercel.py index b3241b70..a0bca0ce 100644 --- a/g4f/Provider/Vercel.py +++ b/g4f/Provider/Vercel.py @@ -1,65 +1,82 @@ from __future__ import annotations import json, base64, requests, execjs, random, uuid +from aiohttp import ClientSession -from ..typing import Any, TypedDict, CreateResult -from .base_provider import BaseProvider -from abc import abstractmethod +from ..typing import Any, TypedDict, AsyncGenerator +from .base_provider import AsyncGeneratorProvider -class Vercel(BaseProvider): +class Vercel(AsyncGeneratorProvider): url = 'https://sdk.vercel.ai' working = True supports_gpt_35_turbo = True supports_stream = True + _anti_bot_token = None - @staticmethod - @abstractmethod - def create_completion( + @classmethod + async def create_async_generator( + cls, model: str, messages: list[dict[str, str]], - stream: bool, **kwargs ) -> CreateResult: + stream: bool, + proxy: str = None, + **kwargs + ) -> AsyncGenerator: + if not model: + model = "gpt-3.5-turbo" + elif model not in model_info: + raise ValueError(f"Model are not supported: {model}") - headers = { - 'authority' : 'sdk.vercel.ai', - 'accept' : '*/*', - 'accept-language' : 'en,fr-FR;q=0.9,fr;q=0.8,es-ES;q=0.7,es;q=0.6,en-US;q=0.5,am;q=0.4,de;q=0.3', - 'cache-control' : 'no-cache', - 'content-type' : 'application/json', - 'custom-encoding' : AntiBotToken(), - 'origin' : 'https://sdk.vercel.ai', - 'pragma' : 'no-cache', - 'referer' : 'https://sdk.vercel.ai/', - 'sec-ch-ua' : '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"', - 'sec-ch-ua-mobile' : '?0', - 'sec-ch-ua-platform': '"macOS"', - 'sec-fetch-dest' : 'empty', - 'sec-fetch-mode' : 'cors', - 'sec-fetch-site' : 'same-origin', - 'user-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.%s.%s Safari/537.36' % ( - random.randint(99, 999), - random.randint(99, 999) - ) - } + if not cls._anti_bot_token: + cls._anti_bot_token = get_anti_bot_token(proxy) json_data = { 'model' : model_info[model]['id'], 'messages' : messages, 'playgroundId': str(uuid.uuid4()), - 'chatIndex' : 0} | model_info[model]['default_params'] + 'chatIndex' : 0 + } | model_info[model]['default_params'] + for tries in range(100): + headers = { + 'authority' : 'sdk.vercel.ai', + 'accept' : '*/*', + 'accept-language' : 'en,fr-FR;q=0.9,fr;q=0.8,es-ES;q=0.7,es;q=0.6,en-US;q=0.5,am;q=0.4,de;q=0.3', + 'cache-control' : 'no-cache', + 'content-type' : 'application/json', + 'custom-encoding' : cls._anti_bot_token, + 'origin' : 'https://sdk.vercel.ai', + 'pragma' : 'no-cache', + 'referer' : 'https://sdk.vercel.ai/', + 'sec-ch-ua' : '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"', + 'sec-ch-ua-mobile' : '?0', + 'sec-ch-ua-platform': '"macOS"', + 'sec-fetch-dest' : 'empty', + 'sec-fetch-mode' : 'cors', + 'sec-fetch-site' : 'same-origin', + 'user-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.%s.%s Safari/537.36' % ( + random.randint(99, 999), + random.randint(99, 999) + ) + } + async with ClientSession( + headers=headers + ) as session: + async with session.post(f"{cls.url}/api/generate", proxy=proxy, json=json_data) as response: + try: + response.raise_for_status() + except Exception as e: + if tries >= 99: + raise e + # Maybe the token is the reason for failing + cls._anti_bot_token = get_anti_bot_token(proxy) + continue + async for token in response.content.iter_any(): + yield token.decode() + break - server_error = True - while server_error: - response = requests.post('https://sdk.vercel.ai/api/generate', - headers=headers, json=json_data, stream=True) - - for token in response.iter_content(chunk_size=2046): - if token != b'Internal Server Error': - server_error = False - yield (token.decode()) - -def AntiBotToken() -> str: +def get_anti_bot_token(proxy: str = None) -> str: headers = { 'authority' : 'sdk.vercel.ai', 'accept' : '*/*', @@ -79,8 +96,9 @@ def AntiBotToken() -> str: ) } + # Does not work with async requests response = requests.get('https://sdk.vercel.ai/openai.jpeg', - headers=headers).text + headers=headers, proxies={"https": proxy}).text raw_data = json.loads(base64.b64decode(response, validate=True)) diff --git a/g4f/Provider/base_provider.py b/g4f/Provider/base_provider.py index 0cceb220..ea81502f 100644 --- a/g4f/Provider/base_provider.py +++ b/g4f/Provider/base_provider.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod import browser_cookie3 -from ..typing import Any, AsyncGenerator, CreateResult +from ..typing import AsyncGenerator, CreateResult class BaseProvider(ABC): @@ -25,7 +25,6 @@ class BaseProvider(ABC): stream: bool, **kwargs ) -> CreateResult: - raise NotImplementedError() diff --git a/g4f/__init__.py b/g4f/__init__.py index 8fdfe21f..f3a887f6 100644 --- a/g4f/__init__.py +++ b/g4f/__init__.py @@ -64,7 +64,7 @@ class ChatCompletion: model, provider = get_model_and_provider(model, provider, False) - if not issubclass(type(provider), AsyncProvider): + if not issubclass(provider, AsyncProvider) and not issubclass(type(provider), AsyncProvider): raise Exception(f"Provider: {provider.__name__} doesn't support create_async") return await provider.create_async(model.name, messages, **kwargs) diff --git a/testing/test_providers.py b/testing/test_providers.py index 30df147a..6096d9b5 100644 --- a/testing/test_providers.py +++ b/testing/test_providers.py @@ -40,7 +40,8 @@ def get_providers() -> list[type[BaseProvider]]: "retry_provider", "BaseProvider", "AsyncProvider", - "AsyncGeneratorProvider" + "AsyncGeneratorProvider", + "RetryProvider", ] return [ getattr(Provider, provider_name) -- cgit v1.2.3 From 1215d30bf4c84685f711a2b0afddb82875eb57d8 Mon Sep 17 00:00:00 2001 From: Tekky <98614666+xtekky@users.noreply.github.com> Date: Sat, 23 Sep 2023 10:54:38 +0100 Subject: Delete g4f/Provider/Vercel.py --- g4f/Provider/Vercel.py | 387 ------------------------------------------------- 1 file changed, 387 deletions(-) delete mode 100644 g4f/Provider/Vercel.py diff --git a/g4f/Provider/Vercel.py b/g4f/Provider/Vercel.py deleted file mode 100644 index a0bca0ce..00000000 --- a/g4f/Provider/Vercel.py +++ /dev/null @@ -1,387 +0,0 @@ -from __future__ import annotations - -import json, base64, requests, execjs, random, uuid -from aiohttp import ClientSession - -from ..typing import Any, TypedDict, AsyncGenerator -from .base_provider import AsyncGeneratorProvider - - -class Vercel(AsyncGeneratorProvider): - url = 'https://sdk.vercel.ai' - working = True - supports_gpt_35_turbo = True - supports_stream = True - _anti_bot_token = None - - @classmethod - async def create_async_generator( - cls, - model: str, - messages: list[dict[str, str]], - stream: bool, - proxy: str = None, - **kwargs - ) -> AsyncGenerator: - if not model: - model = "gpt-3.5-turbo" - elif model not in model_info: - raise ValueError(f"Model are not supported: {model}") - - if not cls._anti_bot_token: - cls._anti_bot_token = get_anti_bot_token(proxy) - - json_data = { - 'model' : model_info[model]['id'], - 'messages' : messages, - 'playgroundId': str(uuid.uuid4()), - 'chatIndex' : 0 - } | model_info[model]['default_params'] - - for tries in range(100): - headers = { - 'authority' : 'sdk.vercel.ai', - 'accept' : '*/*', - 'accept-language' : 'en,fr-FR;q=0.9,fr;q=0.8,es-ES;q=0.7,es;q=0.6,en-US;q=0.5,am;q=0.4,de;q=0.3', - 'cache-control' : 'no-cache', - 'content-type' : 'application/json', - 'custom-encoding' : cls._anti_bot_token, - 'origin' : 'https://sdk.vercel.ai', - 'pragma' : 'no-cache', - 'referer' : 'https://sdk.vercel.ai/', - 'sec-ch-ua' : '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"', - 'sec-ch-ua-mobile' : '?0', - 'sec-ch-ua-platform': '"macOS"', - 'sec-fetch-dest' : 'empty', - 'sec-fetch-mode' : 'cors', - 'sec-fetch-site' : 'same-origin', - 'user-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.%s.%s Safari/537.36' % ( - random.randint(99, 999), - random.randint(99, 999) - ) - } - async with ClientSession( - headers=headers - ) as session: - async with session.post(f"{cls.url}/api/generate", proxy=proxy, json=json_data) as response: - try: - response.raise_for_status() - except Exception as e: - if tries >= 99: - raise e - # Maybe the token is the reason for failing - cls._anti_bot_token = get_anti_bot_token(proxy) - continue - async for token in response.content.iter_any(): - yield token.decode() - break - -def get_anti_bot_token(proxy: str = None) -> str: - headers = { - 'authority' : 'sdk.vercel.ai', - 'accept' : '*/*', - 'accept-language' : 'en,fr-FR;q=0.9,fr;q=0.8,es-ES;q=0.7,es;q=0.6,en-US;q=0.5,am;q=0.4,de;q=0.3', - 'cache-control' : 'no-cache', - 'pragma' : 'no-cache', - 'referer' : 'https://sdk.vercel.ai/', - 'sec-ch-ua' : '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"', - 'sec-ch-ua-mobile' : '?0', - 'sec-ch-ua-platform': '"macOS"', - 'sec-fetch-dest' : 'empty', - 'sec-fetch-mode' : 'cors', - 'sec-fetch-site' : 'same-origin', - 'user-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.%s.%s Safari/537.36' % ( - random.randint(99, 999), - random.randint(99, 999) - ) - } - - # Does not work with async requests - response = requests.get('https://sdk.vercel.ai/openai.jpeg', - headers=headers, proxies={"https": proxy}).text - - raw_data = json.loads(base64.b64decode(response, - validate=True)) - - js_script = '''const globalThis={marker:"mark"};String.prototype.fontcolor=function(){return `${this}`}; - return (%s)(%s)''' % (raw_data['c'], raw_data['a']) - - raw_token = json.dumps({'r': execjs.compile(js_script).call(''), 't': raw_data['t']}, - separators = (",", ":")) - - return base64.b64encode(raw_token.encode('utf-16le')).decode() - -class ModelInfo(TypedDict): - id: str - default_params: dict[str, Any] - -model_info: dict[str, ModelInfo] = { - 'claude-instant-v1': { - 'id': 'anthropic:claude-instant-v1', - 'default_params': { - 'temperature': 1, - 'maximumLength': 1024, - 'topP': 1, - 'topK': 1, - 'presencePenalty': 1, - 'frequencyPenalty': 1, - 'stopSequences': ['\n\nHuman:'], - }, - }, - 'claude-v1': { - 'id': 'anthropic:claude-v1', - 'default_params': { - 'temperature': 1, - 'maximumLength': 1024, - 'topP': 1, - 'topK': 1, - 'presencePenalty': 1, - 'frequencyPenalty': 1, - 'stopSequences': ['\n\nHuman:'], - }, - }, - 'claude-v2': { - 'id': 'anthropic:claude-v2', - 'default_params': { - 'temperature': 1, - 'maximumLength': 1024, - 'topP': 1, - 'topK': 1, - 'presencePenalty': 1, - 'frequencyPenalty': 1, - 'stopSequences': ['\n\nHuman:'], - }, - }, - 'a16z-infra/llama7b-v2-chat': { - 'id': 'replicate:a16z-infra/llama7b-v2-chat', - 'default_params': { - 'temperature': 0.75, - 'maximumLength': 3000, - 'topP': 1, - 'repetitionPenalty': 1, - }, - }, - 'a16z-infra/llama13b-v2-chat': { - 'id': 'replicate:a16z-infra/llama13b-v2-chat', - 'default_params': { - 'temperature': 0.75, - 'maximumLength': 3000, - 'topP': 1, - 'repetitionPenalty': 1, - }, - }, - 'replicate/llama-2-70b-chat': { - 'id': 'replicate:replicate/llama-2-70b-chat', - 'default_params': { - 'temperature': 0.75, - 'maximumLength': 3000, - 'topP': 1, - 'repetitionPenalty': 1, - }, - }, - 'bigscience/bloom': { - 'id': 'huggingface:bigscience/bloom', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 1024, - 'topP': 0.95, - 'topK': 4, - 'repetitionPenalty': 1.03, - }, - }, - 'google/flan-t5-xxl': { - 'id': 'huggingface:google/flan-t5-xxl', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 1024, - 'topP': 0.95, - 'topK': 4, - 'repetitionPenalty': 1.03, - }, - }, - 'EleutherAI/gpt-neox-20b': { - 'id': 'huggingface:EleutherAI/gpt-neox-20b', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 1024, - 'topP': 0.95, - 'topK': 4, - 'repetitionPenalty': 1.03, - 'stopSequences': [], - }, - }, - 'OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5': { - 'id': 'huggingface:OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5', - 'default_params': { - 'maximumLength': 1024, - 'typicalP': 0.2, - 'repetitionPenalty': 1, - }, - }, - 'OpenAssistant/oasst-sft-1-pythia-12b': { - 'id': 'huggingface:OpenAssistant/oasst-sft-1-pythia-12b', - 'default_params': { - 'maximumLength': 1024, - 'typicalP': 0.2, - 'repetitionPenalty': 1, - }, - }, - 'bigcode/santacoder': { - 'id': 'huggingface:bigcode/santacoder', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 1024, - 'topP': 0.95, - 'topK': 4, - 'repetitionPenalty': 1.03, - }, - }, - 'command-light-nightly': { - 'id': 'cohere:command-light-nightly', - 'default_params': { - 'temperature': 0.9, - 'maximumLength': 1024, - 'topP': 1, - 'topK': 0, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, - 'command-nightly': { - 'id': 'cohere:command-nightly', - 'default_params': { - 'temperature': 0.9, - 'maximumLength': 1024, - 'topP': 1, - 'topK': 0, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, - 'gpt-4': { - 'id': 'openai:gpt-4', - 'default_params': { - 'temperature': 0.7, - 'maximumLength': 8192, - 'topP': 1, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, - 'gpt-4-0613': { - 'id': 'openai:gpt-4-0613', - 'default_params': { - 'temperature': 0.7, - 'maximumLength': 8192, - 'topP': 1, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, - 'code-davinci-002': { - 'id': 'openai:code-davinci-002', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 1024, - 'topP': 1, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, - 'gpt-3.5-turbo': { - 'id': 'openai:gpt-3.5-turbo', - 'default_params': { - 'temperature': 0.7, - 'maximumLength': 4096, - 'topP': 1, - 'topK': 1, - 'presencePenalty': 1, - 'frequencyPenalty': 1, - 'stopSequences': [], - }, - }, - 'gpt-3.5-turbo-16k': { - 'id': 'openai:gpt-3.5-turbo-16k', - 'default_params': { - 'temperature': 0.7, - 'maximumLength': 16280, - 'topP': 1, - 'topK': 1, - 'presencePenalty': 1, - 'frequencyPenalty': 1, - 'stopSequences': [], - }, - }, - 'gpt-3.5-turbo-16k-0613': { - 'id': 'openai:gpt-3.5-turbo-16k-0613', - 'default_params': { - 'temperature': 0.7, - 'maximumLength': 16280, - 'topP': 1, - 'topK': 1, - 'presencePenalty': 1, - 'frequencyPenalty': 1, - 'stopSequences': [], - }, - }, - 'text-ada-001': { - 'id': 'openai:text-ada-001', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 1024, - 'topP': 1, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, - 'text-babbage-001': { - 'id': 'openai:text-babbage-001', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 1024, - 'topP': 1, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, - 'text-curie-001': { - 'id': 'openai:text-curie-001', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 1024, - 'topP': 1, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, - 'text-davinci-002': { - 'id': 'openai:text-davinci-002', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 1024, - 'topP': 1, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, - 'text-davinci-003': { - 'id': 'openai:text-davinci-003', - 'default_params': { - 'temperature': 0.5, - 'maximumLength': 4097, - 'topP': 1, - 'presencePenalty': 0, - 'frequencyPenalty': 0, - 'stopSequences': [], - }, - }, -} -- cgit v1.2.3