diff options
| -rw-r--r-- | etc/interference/app.py | 163 | ||||
| -rw-r--r-- | etc/interference/requirements.txt | 5 | ||||
| -rw-r--r-- | etc/testing/log_time.py (renamed from testing/log_time.py) | 0 | ||||
| -rw-r--r-- | etc/testing/test_async.py (renamed from testing/test_async.py) | 0 | ||||
| -rw-r--r-- | etc/testing/test_chat_completion.py (renamed from testing/test_chat_completion.py) | 0 | ||||
| -rw-r--r-- | etc/testing/test_interference.py (renamed from testing/test_interference.py) | 0 | ||||
| -rw-r--r-- | etc/testing/test_needs_auth.py (renamed from testing/test_needs_auth.py) | 0 | ||||
| -rw-r--r-- | etc/testing/test_providers.py (renamed from testing/test_providers.py) | 0 | ||||
| -rw-r--r-- | etc/tool/create_provider.py (renamed from tool/create_provider.py) | 0 | ||||
| -rw-r--r-- | etc/tool/provider_init.py (renamed from tool/provider_init.py) | 0 | ||||
| -rw-r--r-- | etc/tool/readme_table.py (renamed from tool/readme_table.py) | 0 | ||||
| -rw-r--r-- | etc/tool/vercel.py (renamed from tool/vercel.py) | 0 |
12 files changed, 168 insertions, 0 deletions
diff --git a/etc/interference/app.py b/etc/interference/app.py new file mode 100644 index 00000000..5abbcff2 --- /dev/null +++ b/etc/interference/app.py @@ -0,0 +1,163 @@ +import json +import time +import random +import string +import requests + +from typing import Any +from flask import Flask, request +from flask_cors import CORS +from transformers import AutoTokenizer +from g4f import ChatCompletion + +app = Flask(__name__) +CORS(app) + +@app.route('/chat/completions', methods=['POST']) +def chat_completions(): + model = request.get_json().get('model', 'gpt-3.5-turbo') + stream = request.get_json().get('stream', False) + messages = request.get_json().get('messages') + + response = ChatCompletion.create(model = model, + stream = stream, messages = messages) + + completion_id = ''.join(random.choices(string.ascii_letters + string.digits, k=28)) + completion_timestamp = int(time.time()) + + if not stream: + return { + 'id': f'chatcmpl-{completion_id}', + 'object': 'chat.completion', + 'created': completion_timestamp, + 'model': model, + 'choices': [ + { + 'index': 0, + 'message': { + 'role': 'assistant', + 'content': response, + }, + 'finish_reason': 'stop', + } + ], + 'usage': { + 'prompt_tokens': None, + 'completion_tokens': None, + 'total_tokens': None, + }, + } + + def streaming(): + for chunk in response: + completion_data = { + 'id': f'chatcmpl-{completion_id}', + 'object': 'chat.completion.chunk', + 'created': completion_timestamp, + 'model': model, + 'choices': [ + { + 'index': 0, + 'delta': { + 'content': chunk, + }, + 'finish_reason': None, + } + ], + } + + content = json.dumps(completion_data, separators=(',', ':')) + yield f'data: {content}\n\n' + time.sleep(0.1) + + end_completion_data: dict[str, Any] = { + 'id': f'chatcmpl-{completion_id}', + 'object': 'chat.completion.chunk', + 'created': completion_timestamp, + 'model': model, + 'choices': [ + { + 'index': 0, + 'delta': {}, + 'finish_reason': 'stop', + } + ], + } + content = json.dumps(end_completion_data, separators=(',', ':')) + yield f'data: {content}\n\n' + + return app.response_class(streaming(), mimetype='text/event-stream') + + +# Get the embedding from huggingface +def get_embedding(input_text, token): + huggingface_token = token + embedding_model = 'sentence-transformers/all-mpnet-base-v2' + max_token_length = 500 + + # Load the tokenizer for the 'all-mpnet-base-v2' model + tokenizer = AutoTokenizer.from_pretrained(embedding_model) + # Tokenize the text and split the tokens into chunks of 500 tokens each + tokens = tokenizer.tokenize(input_text) + token_chunks = [tokens[i:i + max_token_length] + for i in range(0, len(tokens), max_token_length)] + + # Initialize an empty list + embeddings = [] + + # Create embeddings for each chunk + for chunk in token_chunks: + # Convert the chunk tokens back to text + chunk_text = tokenizer.convert_tokens_to_string(chunk) + + # Use the Hugging Face API to get embeddings for the chunk + api_url = f'https://api-inference.huggingface.co/pipeline/feature-extraction/{embedding_model}' + headers = {'Authorization': f'Bearer {huggingface_token}'} + chunk_text = chunk_text.replace('\n', ' ') + + # Make a POST request to get the chunk's embedding + response = requests.post(api_url, headers=headers, json={ + 'inputs': chunk_text, 'options': {'wait_for_model': True}}) + + # Parse the response and extract the embedding + chunk_embedding = response.json() + # Append the embedding to the list + embeddings.append(chunk_embedding) + + # averaging all the embeddings + # this isn't very effective + # someone a better idea? + num_embeddings = len(embeddings) + average_embedding = [sum(x) / num_embeddings for x in zip(*embeddings)] + embedding = average_embedding + return embedding + + +@app.route('/embeddings', methods=['POST']) +def embeddings(): + input_text_list = request.get_json().get('input') + input_text = ' '.join(map(str, input_text_list)) + token = request.headers.get('Authorization').replace('Bearer ', '') + embedding = get_embedding(input_text, token) + + return { + 'data': [ + { + 'embedding': embedding, + 'index': 0, + 'object': 'embedding' + } + ], + 'model': 'text-embedding-ada-002', + 'object': 'list', + 'usage': { + 'prompt_tokens': None, + 'total_tokens': None + } + } + +def main(): + app.run(host='0.0.0.0', port=1337, debug=True) + +if __name__ == '__main__': + main()
\ No newline at end of file diff --git a/etc/interference/requirements.txt b/etc/interference/requirements.txt new file mode 100644 index 00000000..eaa3265b --- /dev/null +++ b/etc/interference/requirements.txt @@ -0,0 +1,5 @@ +flask_cors +watchdog~=3.0.0 +transformers +tensorflow +torch
\ No newline at end of file diff --git a/testing/log_time.py b/etc/testing/log_time.py index 376ab86d..376ab86d 100644 --- a/testing/log_time.py +++ b/etc/testing/log_time.py diff --git a/testing/test_async.py b/etc/testing/test_async.py index 76b109b1..76b109b1 100644 --- a/testing/test_async.py +++ b/etc/testing/test_async.py diff --git a/testing/test_chat_completion.py b/etc/testing/test_chat_completion.py index 7600e46b..7600e46b 100644 --- a/testing/test_chat_completion.py +++ b/etc/testing/test_chat_completion.py diff --git a/testing/test_interference.py b/etc/testing/test_interference.py index d8e85a6c..d8e85a6c 100644 --- a/testing/test_interference.py +++ b/etc/testing/test_interference.py diff --git a/testing/test_needs_auth.py b/etc/testing/test_needs_auth.py index 26630e23..26630e23 100644 --- a/testing/test_needs_auth.py +++ b/etc/testing/test_needs_auth.py diff --git a/testing/test_providers.py b/etc/testing/test_providers.py index ec0e0271..ec0e0271 100644 --- a/testing/test_providers.py +++ b/etc/testing/test_providers.py diff --git a/tool/create_provider.py b/etc/tool/create_provider.py index 5a1fed06..5a1fed06 100644 --- a/tool/create_provider.py +++ b/etc/tool/create_provider.py diff --git a/tool/provider_init.py b/etc/tool/provider_init.py index 22f21d4d..22f21d4d 100644 --- a/tool/provider_init.py +++ b/etc/tool/provider_init.py diff --git a/tool/readme_table.py b/etc/tool/readme_table.py index b5b64cb1..b5b64cb1 100644 --- a/tool/readme_table.py +++ b/etc/tool/readme_table.py diff --git a/tool/vercel.py b/etc/tool/vercel.py index 7b87e298..7b87e298 100644 --- a/tool/vercel.py +++ b/etc/tool/vercel.py |