From 0ade2098a66054cac3bb0ade1640eb8123302db4 Mon Sep 17 00:00:00 2001
From: Heiner Lohaus <hlohaus@users.noreply.github.com>
Date: Sat, 24 Feb 2024 20:09:38 +0100
Subject: Add count word and tokens in gui

---
 g4f/gui/client/css/style.css   | 26 +++++++++++++--
 g4f/gui/client/html/index.html | 13 ++++++--
 g4f/gui/client/js/chat.v1.js   | 73 ++++++++++++++++++++++++++++++++----------
 3 files changed, 89 insertions(+), 23 deletions(-)

diff --git a/g4f/gui/client/css/style.css b/g4f/gui/client/css/style.css
index aab7e555..bd42280d 100644
--- a/g4f/gui/client/css/style.css
+++ b/g4f/gui/client/css/style.css
@@ -129,7 +129,7 @@ body {
     flex-direction: column;
     overflow: auto;
     overflow-wrap: break-word;
-    padding-bottom: 50px;
+    padding-bottom: 20px;
 }
 
 .conversation .user-input {
@@ -291,7 +291,7 @@ body {
 .message .content {
     display: flex;
     flex-direction: column;
-    gap: 18px;
+    gap: 10px;
 }
 
 .message .content,
@@ -343,6 +343,26 @@ body {
     display: block;
 }
 
+.message .content .provider a,
+.message .content .provider {
+    font-size: 12px;
+    text-decoration: none;
+}
+
+.message .content .provider a {
+    font-weight: bold;
+}
+
+.message .content .count {
+    font-size: 12px;
+}
+
+.count_total {
+    font-size: 12px;
+    padding-left: 100px;
+    padding-top: 10px;
+}
+
 .new_convo {
     padding: 8px 12px;
     display: flex;
@@ -367,7 +387,7 @@ body {
 
 .stop_generating, .regenerate {
     position: absolute;
-    bottom: 158px;
+    bottom: 122px;
     left: 50%;
     transform: translateX(-50%);
     z-index: 1000000;
diff --git a/g4f/gui/client/html/index.html b/g4f/gui/client/html/index.html
index eaae7355..102a762e 100644
--- a/g4f/gui/client/html/index.html
+++ b/g4f/gui/client/html/index.html
@@ -29,10 +29,17 @@
             }
         };
     </script>
-    <script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
+    <script id="MathJax-script" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" async></script>
+    <script type="module" src="https://cdn.jsdelivr.net/npm/mistral-tokenizer-js" async>
+        import mistralTokenizer from 'mistral-tokenizer-js'
+    </script>
+    <script type="module" src="https://belladoreai.github.io/llama-tokenizer-js/llama-tokenizer.js" async>
+        import llamaTokenizer from 'llama-tokenizer-js'
+    </script>
+    <script src="https://unpkg.com/gpt-tokenizer/dist/cl100k_base.js" async></script>
     <script>
-        const user_image = `<img src="/assets/img/user.png" alt="your avatar">`;
-        const gpt_image = `<img src="/assets/img/gpt.png" alt="your avatar">`;
+        const user_image = '<img src="/assets/img/user.png" alt="your avatar">';
+        const gpt_image = '<img src="/assets/img/gpt.png" alt="your avatar">';
     </script>
     <style>
         .hljs {
diff --git a/g4f/gui/client/js/chat.v1.js b/g4f/gui/client/js/chat.v1.js
index 37775fbb..9585ca98 100644
--- a/g4f/gui/client/js/chat.v1.js
+++ b/g4f/gui/client/js/chat.v1.js
@@ -120,16 +120,8 @@ const remove_cancel_button = async () => {
     }, 300);
 };
 
-const ask_gpt = async () => {
-    regenerate.classList.add(`regenerate-hidden`);
-    messages = await get_messages(window.conversation_id);
-
-    // Remove history, if it is selected
-    if (document.getElementById('history')?.checked) {
-        messages = [messages[messages.length-1]]
-    }
-
-    new_messages = [];
+const filter_messages = (messages) => {
+    let new_messages = [];
     for (i in messages) {
         new_message = messages[i];
         // Remove generated images from history
@@ -143,6 +135,19 @@ const ask_gpt = async () => {
             new_messages.push(new_message)
         }
     }
+    return new_messages;
+}
+
+const ask_gpt = async () => {
+    regenerate.classList.add(`regenerate-hidden`);
+    messages = await get_messages(window.conversation_id);
+    total_messages = messages.length;
+
+    // Remove history, if it is selected
+    if (document.getElementById('history')?.checked) {
+        messages = [messages[messages.length-1]];
+    }
+    messages = filter_messages(messages);
 
     window.scrollTo(0, 0);
     window.controller = new AbortController();
@@ -159,8 +164,11 @@ const ask_gpt = async () => {
     await new Promise((r) => setTimeout(r, 500));
     window.scrollTo(0, 0);
 
+    el = message_box.querySelector('.count_total');
+    el ? el.parentElement.removeChild(el) : null;
+
     message_box.innerHTML += `
-        <div class="message" data-index="${new_messages.length}">
+        <div class="message" data-index="${total_messages}">
             <div class="assistant">
                 ${gpt_image}
                 <i class="fa-solid fa-xmark"></i>
@@ -186,7 +194,7 @@ const ask_gpt = async () => {
             web_search: document.getElementById(`switch`).checked,
             provider: provider.options[provider.selectedIndex].value,
             patch_provider: document.getElementById('patch').checked,
-            messages: new_messages
+            messages: messages
         });
         const headers = {
             accept: 'text/event-stream'
@@ -240,7 +248,7 @@ const ask_gpt = async () => {
             } else {
                 html = markdown_render(text);
                 let lastElement, lastIndex = null;
-                for (element of ['</p>', '</code></pre>', '</li>\n</ol>']) {
+                for (element of ['</p>', '</code></pre>', '</li>\n</ol>', '</li>\n</ul>']) {
                     const index = html.lastIndexOf(element)
                     if (index > lastIndex) {
                         lastElement = element;
@@ -278,8 +286,9 @@ const ask_gpt = async () => {
     let cursorDiv = document.getElementById(`cursor`);
     if (cursorDiv) cursorDiv.parentNode.removeChild(cursorDiv);
     if (text) {
-        add_message(window.conversation_id, "assistant", text, provider);
+        await add_message(window.conversation_id, "assistant", text, provider);
     }
+    await load_conversation(window.conversation_id);
     message_box.scrollTop = message_box.scrollHeight;
     await remove_cancel_button();
     await register_remove_message();
@@ -372,10 +381,16 @@ const load_conversation = async (conversation_id) => {
     let elements = "";
     for (i in messages) {
         let item = messages[i];
-        let provider = item.provider ? `
+        let next_i = parseInt(i) + 1;
+        let next_provider = item.provider ? item.provider : (messages.length > next_i ? messages[next_i].provider : null);
+        let tokens_count = next_provider?.model ? count_tokens(next_provider.model, item.content) : "";
+        let append_count = tokens_count ? `, ${tokens_count} tokens` : "";
+        let words_count = `(${count_words(item.content)} words${append_count})`
+        let provider_link = item?.provider?.name ? `<a href="${item?.provider?.url}" target="_blank">${item.provider.name}</a>` : "";
+        let provider = provider_link ? `
             <div class="provider">
-            <a href="${item.provider.url}" target="_blank">${item.provider.name}</a>
-            ${item.provider.model ? ' with ' + item.provider.model : ''}
+                ${provider_link}
+                ${item.provider.model ? ' with ' + item.provider.model : ''}
             </div>
         ` : "";
         elements += `
@@ -391,10 +406,17 @@ const load_conversation = async (conversation_id) => {
                 <div class="content">
                     ${provider}
                     <div class="content_inner">${markdown_render(item.content)}</div>
+                    <div class="count">${words_count}</div>
                 </div>
             </div>
         `;
     }
+
+    let count_total = GPTTokenizer_cl100k_base?.encodeChat(filter_messages(messages), "gpt-3.5-turbo").length
+    if (count_total > 0) {
+        elements += `<div class="count_total">(${count_total} tokens used)</div>`;
+    }
+
     message_box.innerHTML = elements;
 
     await register_remove_message();
@@ -407,6 +429,23 @@ const load_conversation = async (conversation_id) => {
     }, 500);
 };
 
+function count_words(text) {
+    var matches = text.match(/[\w\d\’\'-]+/gi);
+    return matches ? matches.length : 0;
+}
+
+function count_tokens(model, text) {
+    if (model.startsWith("gpt-3") || model.startsWith("gpt-4")) {
+        return GPTTokenizer_cl100k_base?.encode(text).length
+    }
+    if (model.startsWith("llama2") || model.startsWith("codellama")) {
+        return llamaTokenizer?.encode(text).length
+    }
+    if (model.startsWith("mistral") || model.startsWith("mixtral")) {
+        return mistralTokenizer?.encode(text).length
+    }
+}
+
 const get_conversation = async (conversation_id) => {
     let conversation = await JSON.parse(
         localStorage.getItem(`conversation:${conversation_id}`)
-- 
cgit v1.2.3