diff options
| -rw-r--r-- | README.md | 14 | ||||
| -rw-r--r-- | interference/app.py | 70 |
2 files changed, 74 insertions, 10 deletions
@@ -279,6 +279,9 @@ asyncio.run(run_async()) ### interference openai-proxy api (use with openai python package) +If you want to use the embedding function, you need to get a huggingface token. You can get one at https://huggingface.co/settings/tokens make sure your role is set to write. If you have your token, just use it instead of the OpenAI api-key. + + get requirements: ```sh @@ -294,7 +297,7 @@ python3 -m interference.app ```py import openai -openai.api_key = "" +openai.api_key = "Empty if you don't use embeddings, otherwise your hugginface token" openai.api_base = "http://localhost:1337" @@ -519,12 +522,9 @@ for message in response: print(message, flush=True, end='') ``` -## ChatGPT clone - -> We are currently implementing new features and trying to scale it, but please be patient as it may be unstable. -> https://chat.g4f.ai/chat -> This site was developed by me and includes **gpt-4/3.5**, **internet access** and **gpt-jailbreak's** like DAN -> Run locally here: https://github.com/xtekky/chatgpt-clone. +## Contributors +A list of the contributors is available [here](https://github.com/xtekky/gpt4free/graphs/contributors) +The [`Vercel.py`](https://github.com/xtekky/gpt4free/blob/main/g4f/Provider/Vercel.py) file contains code from [vercel-llm-api](https://github.com/ading2210/vercel-llm-api) by [@ading2210](https://github.com/ading2210), which is licenced under the [GNU GPL v3](https://www.gnu.org/licenses/gpl-3.0.txt) ## Copyright: diff --git a/interference/app.py b/interference/app.py index 1b1af22f..15e5bc80 100644 --- a/interference/app.py +++ b/interference/app.py @@ -3,10 +3,10 @@ import random import string import time from typing import Any - +import requests from flask import Flask, request from flask_cors import CORS - +from transformers import AutoTokenizer from g4f import ChatCompletion app = Flask(__name__) @@ -88,9 +88,73 @@ def chat_completions(): return app.response_class(streaming(), mimetype="text/event-stream") +#Get the embedding from huggingface +def get_embedding(input_text, token): + huggingface_token = token + embedding_model = "sentence-transformers/all-mpnet-base-v2" + max_token_length = 500 + + # Load the tokenizer for the "all-mpnet-base-v2" model + tokenizer = AutoTokenizer.from_pretrained(embedding_model) + # Tokenize the text and split the tokens into chunks of 500 tokens each + tokens = tokenizer.tokenize(input_text) + token_chunks = [tokens[i:i + max_token_length] for i in range(0, len(tokens), max_token_length)] + + # Initialize an empty list + embeddings = [] + + # Create embeddings for each chunk + for chunk in token_chunks: + # Convert the chunk tokens back to text + chunk_text = tokenizer.convert_tokens_to_string(chunk) + + # Use the Hugging Face API to get embeddings for the chunk + api_url = f"https://api-inference.huggingface.co/pipeline/feature-extraction/{embedding_model}" + headers = {"Authorization": f"Bearer {huggingface_token}"} + chunk_text = chunk_text.replace("\n", " ") + + # Make a POST request to get the chunk's embedding + response = requests.post(api_url, headers=headers, json={"inputs": chunk_text, "options": {"wait_for_model": True}}) + + # Parse the response and extract the embedding + chunk_embedding = response.json() + # Append the embedding to the list + embeddings.append(chunk_embedding) + + #averaging all the embeddings + #this isn't very effective + #someone a better idea? + num_embeddings = len(embeddings) + average_embedding = [sum(x) / num_embeddings for x in zip(*embeddings)] + embedding = average_embedding + return embedding + + +@app.route("/embeddings", methods=["POST"]) +def embeddings(): + input_text_list = request.get_json().get("input") + input_text = ' '.join(map(str, input_text_list)) + token = request.headers.get('Authorization').replace("Bearer ", "") + embedding = get_embedding(input_text, token) + return { + "data": [ + { + "embedding": embedding, + "index": 0, + "object": "embedding" + } + ], + "model": "text-embedding-ada-002", + "object": "list", + "usage": { + "prompt_tokens": None, + "total_tokens": None + } + } + def main(): app.run(host="0.0.0.0", port=1337, debug=True) if __name__ == "__main__": - main()
\ No newline at end of file + main() |