From 3ae90b57edf5ebd2b2f7dc8ad43f0f560edded84 Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Sun, 22 Oct 2023 20:01:14 +0200 Subject: Improve get_cookies helper --- g4f/Provider/ChatgptFree.py | 16 +++++++++++----- g4f/Provider/helper.py | 23 ++++++++++++++--------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/g4f/Provider/ChatgptFree.py b/g4f/Provider/ChatgptFree.py index 8b7d04c4..806ff7cc 100644 --- a/g4f/Provider/ChatgptFree.py +++ b/g4f/Provider/ChatgptFree.py @@ -14,7 +14,7 @@ from .helper import format_prompt, get_cookies class ChatgptFree(AsyncProvider): url = "https://chatgptfree.ai" supports_gpt_35_turbo = True - working = True + working = False _post_id = None _nonce = None @@ -24,6 +24,7 @@ class ChatgptFree(AsyncProvider): model: str, messages: Messages, proxy: str = None, + timeout: int = 120, cookies: dict = None, **kwargs ) -> str: @@ -45,14 +46,19 @@ class ChatgptFree(AsyncProvider): 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', - 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', + 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', } - async with StreamSession(headers=headers, - impersonate="chrome107", proxies={"https": proxy}, timeout=10) as session: + async with StreamSession( + headers=headers, + cookies=cookies, + impersonate="chrome107", + proxies={"https": proxy}, + timeout=timeout + ) as session: if not cls._nonce: - async with session.get(f"{cls.url}/", cookies=cookies) as response: + async with session.get(f"{cls.url}/") as response: response.raise_for_status() response = await response.text() diff --git a/g4f/Provider/helper.py b/g4f/Provider/helper.py index 096293b2..20500777 100644 --- a/g4f/Provider/helper.py +++ b/g4f/Provider/helper.py @@ -3,10 +3,11 @@ from __future__ import annotations import sys import asyncio import webbrowser -import http.cookiejar from os import path from asyncio import AbstractEventLoop +from platformdirs import user_config_dir + from ..typing import Dict, Messages from browser_cookie3 import chrome, chromium, opera, opera_gx, brave, edge, vivaldi, firefox, BrowserCookieError @@ -72,16 +73,22 @@ def init_cookies(): # Load cookies for a domain from all supported browsers. # Cache the results in the "_cookies" variable. def get_cookies(domain_name=''): - cj = http.cookiejar.CookieJar() - for cookie_fn in [chrome, chromium, opera, opera_gx, brave, edge, vivaldi, firefox]: + def g4f(domain_name): + user_data_dir = user_config_dir("g4f") + cookie_file = path.join(user_data_dir, "Default", "Cookies") + if not path.exists(cookie_file): + return [] + return chrome(cookie_file, domain_name) + cookie_jar = [] + for cookie_fn in [g4f, chrome, chromium, opera, opera_gx, brave, edge, vivaldi, firefox]: try: - for cookie in cookie_fn(domain_name=domain_name): - cj.set_cookie(cookie) + cookie_jar = cookie_fn(domain_name=domain_name) + if len(cookie_jar) > 0: + break except BrowserCookieError: pass - _cookies[domain_name] = {cookie.name: cookie.value for cookie in cj} - + _cookies[domain_name] = {cookie.name: cookie.value for cookie in cookie_jar} return _cookies[domain_name] @@ -100,10 +107,8 @@ def format_prompt(messages: Messages, add_special_tokens=False) -> str: def get_browser(user_data_dir: str = None): from undetected_chromedriver import Chrome - from platformdirs import user_config_dir if not user_data_dir: user_data_dir = user_config_dir("g4f") - user_data_dir = path.join(user_data_dir, "Default") return Chrome(user_data_dir=user_data_dir) \ No newline at end of file -- cgit v1.2.3 From 598255fa26e39d57e51fb41957a381a9241cb032 Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Sun, 22 Oct 2023 23:53:18 +0200 Subject: Debug logging support Async browse access token --- README.md | 2 +- g4f/Provider/AItianhuSpace.py | 4 +-- g4f/Provider/helper.py | 19 ++++++++------ g4f/Provider/needs_auth/OpenaiChat.py | 47 ++++++++++++++++++++--------------- g4f/Provider/retry_provider.py | 8 +++--- g4f/__init__.py | 5 ++-- 6 files changed, 47 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 996d550b..7ca2a9ad 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ docker compose down ```py import g4f -g4f.logging = True # enable logging +g4f.debug.logging = True # enable logging g4f.check_version = False # Disable automatic version checking print(g4f.version) # check version print(g4f.Provider.Ails.params) # supported args diff --git a/g4f/Provider/AItianhuSpace.py b/g4f/Provider/AItianhuSpace.py index 51297bcd..46856060 100644 --- a/g4f/Provider/AItianhuSpace.py +++ b/g4f/Provider/AItianhuSpace.py @@ -1,7 +1,7 @@ from __future__ import annotations import random, json -from ..debug import logging +from .. import debug from ..typing import AsyncResult, Messages from ..requests import StreamSession from .base_provider import AsyncGeneratorProvider, format_prompt, get_cookies @@ -36,7 +36,7 @@ class AItianhuSpace(AsyncGeneratorProvider): rand = ''.join(random.choice(chars) for _ in range(6)) domain = f"{rand}.{domains[model]}" - if logging: + if debug.logging: print(f"AItianhuSpace | using domain: {domain}") if not cookies: diff --git a/g4f/Provider/helper.py b/g4f/Provider/helper.py index 20500777..1b00ace5 100644 --- a/g4f/Provider/helper.py +++ b/g4f/Provider/helper.py @@ -10,7 +10,7 @@ from platformdirs import user_config_dir from ..typing import Dict, Messages from browser_cookie3 import chrome, chromium, opera, opera_gx, brave, edge, vivaldi, firefox, BrowserCookieError - +from .. import debug # Change event loop policy on windows if sys.platform == 'win32': @@ -45,7 +45,6 @@ def get_event_loop() -> AbstractEventLoop: ) def init_cookies(): - urls = [ 'https://chat-gpt.org', 'https://www.aitianhu.com', @@ -73,22 +72,26 @@ def init_cookies(): # Load cookies for a domain from all supported browsers. # Cache the results in the "_cookies" variable. def get_cookies(domain_name=''): + if domain_name in _cookies: + return _cookies[domain_name] def g4f(domain_name): user_data_dir = user_config_dir("g4f") cookie_file = path.join(user_data_dir, "Default", "Cookies") if not path.exists(cookie_file): return [] return chrome(cookie_file, domain_name) - cookie_jar = [] + cookies = {} for cookie_fn in [g4f, chrome, chromium, opera, opera_gx, brave, edge, vivaldi, firefox]: try: cookie_jar = cookie_fn(domain_name=domain_name) - if len(cookie_jar) > 0: - break - except BrowserCookieError: + if len(cookie_jar) and debug.logging: + print(f"Read cookies from {cookie_fn.__name__} for {domain_name}") + for cookie in cookie_jar: + if cookie.name not in cookies: + cookies[cookie.name] = cookie.value + except BrowserCookieError as e: pass - - _cookies[domain_name] = {cookie.name: cookie.value for cookie in cookie_jar} + _cookies[domain_name] = cookies return _cookies[domain_name] diff --git a/g4f/Provider/needs_auth/OpenaiChat.py b/g4f/Provider/needs_auth/OpenaiChat.py index b4b4a670..14619958 100644 --- a/g4f/Provider/needs_auth/OpenaiChat.py +++ b/g4f/Provider/needs_auth/OpenaiChat.py @@ -3,7 +3,7 @@ from __future__ import annotations import uuid, json, time from ..base_provider import AsyncGeneratorProvider -from ..helper import get_browser, get_cookies, format_prompt +from ..helper import get_browser, get_cookies, format_prompt, get_event_loop from ...typing import AsyncResult, Messages from ...requests import StreamSession @@ -73,26 +73,33 @@ class OpenaiChat(AsyncGeneratorProvider): last_message = new_message @classmethod - def browse_access_token(cls) -> str: - try: - from selenium.webdriver.common.by import By - from selenium.webdriver.support.ui import WebDriverWait - from selenium.webdriver.support import expected_conditions as EC + async def browse_access_token(cls) -> str: + def browse() -> str: + try: + from selenium.webdriver.common.by import By + from selenium.webdriver.support.ui import WebDriverWait + from selenium.webdriver.support import expected_conditions as EC - driver = get_browser() - except ImportError: - return + driver = get_browser() + except ImportError: + return - driver.get(f"{cls.url}/") - try: - WebDriverWait(driver, 1200).until( - EC.presence_of_element_located((By.ID, "prompt-textarea")) - ) - javascript = "return (await (await fetch('/api/auth/session')).json())['accessToken']" - return driver.execute_script(javascript) - finally: - time.sleep(1) - driver.quit() + driver.get(f"{cls.url}/") + try: + WebDriverWait(driver, 1200).until( + EC.presence_of_element_located((By.ID, "prompt-textarea")) + ) + javascript = "return (await (await fetch('/api/auth/session')).json())['accessToken']" + return driver.execute_script(javascript) + finally: + driver.close() + time.sleep(0.1) + driver.quit() + loop = get_event_loop() + return await loop.run_in_executor( + None, + browse + ) @classmethod async def fetch_access_token(cls, cookies: dict, proxies: dict = None) -> str: @@ -110,7 +117,7 @@ class OpenaiChat(AsyncGeneratorProvider): if cookies: cls._access_token = await cls.fetch_access_token(cookies, proxies) if not cls._access_token: - cls._access_token = cls.browse_access_token() + cls._access_token = await cls.browse_access_token() if not cls._access_token: raise RuntimeError("Read access token failed") return cls._access_token diff --git a/g4f/Provider/retry_provider.py b/g4f/Provider/retry_provider.py index 39d61c35..5979e15b 100644 --- a/g4f/Provider/retry_provider.py +++ b/g4f/Provider/retry_provider.py @@ -5,13 +5,13 @@ import random from typing import List, Type, Dict from ..typing import CreateResult, Messages from .base_provider import BaseProvider, AsyncProvider +from .. import debug class RetryProvider(AsyncProvider): __name__: str = "RetryProvider" working: bool = True supports_stream: bool = True - logging: bool = False def __init__( self, @@ -39,7 +39,7 @@ class RetryProvider(AsyncProvider): started: bool = False for provider in providers: try: - if self.logging: + if debug.logging: print(f"Using {provider.__name__} provider") for token in provider.create_completion(model, messages, stream, **kwargs): @@ -51,7 +51,7 @@ class RetryProvider(AsyncProvider): except Exception as e: self.exceptions[provider.__name__] = e - if self.logging: + if debug.logging: print(f"{provider.__name__}: {e.__class__.__name__}: {e}") if started: raise e @@ -77,7 +77,7 @@ class RetryProvider(AsyncProvider): ) except Exception as e: self.exceptions[provider.__name__] = e - if self.logging: + if debug.logging: print(f"{provider.__name__}: {e.__class__.__name__}: {e}") self.raise_exceptions() diff --git a/g4f/__init__.py b/g4f/__init__.py index 20f71e0d..ef8b1cc1 100644 --- a/g4f/__init__.py +++ b/g4f/__init__.py @@ -3,7 +3,7 @@ from requests import get from .models import Model, ModelUtils, _all_models from .Provider import BaseProvider, RetryProvider from .typing import Messages, CreateResult, Union, List -from .debug import logging +from . import debug version = '0.1.7.3' version_check = True @@ -46,8 +46,7 @@ def get_model_and_provider(model : Union[Model, str], if not provider.supports_stream and stream: raise ValueError(f'{provider.__name__} does not support "stream" argument') - if logging: - RetryProvider.logging = True + if debug.logging: print(f'Using {provider.__name__} provider') return model, provider -- cgit v1.2.3