Question-Answering Bot with Confidence Scoring

# Install the necessary packages
!pip install langchain langchain-seekrflow python-dotenv

# Import the required libraries
import os
from seekrai import SeekrFlow
from langchain_seekrflow import ChatSeekrFlow
from langchain.chains import LLMChain
from langchain.prompts import PromptTemplate

# Create the SeekrFlow client
client = SeekrFlow(api_key=seekr_api_key)

# Initialize the chat model
model = ChatSeekrFlow(
    client=client,
    model_name="meta-llama/Meta-Llama-3-8B-Instruct",  # Specify the model you want to use
    temperature=0.3  # Keep temperature low for more reliable answers
)

# Create a prompt template that asks for both an answer and confidence level
template = """
Question: {question}

Please answer the question above. After providing your answer, on a new line that starts with 'Confidence:', 
rate how confident you are in your answer on a scale of 1-10, where:
1 = Complete guess
5 = Moderately confident
10 = Absolutely certain

Explain briefly why you gave this confidence rating.

Answer:
"""

prompt = PromptTemplate(
    input_variables=["question"],
    template=template
)

# Create the chain
qa_chain = LLMChain(llm=model, prompt=prompt)

# Build the parser to extract answers and confidence scores
def parse_response(response):
    """Extract the answer and confidence score from the model response."""
    
    # Default values
    answer = response
    confidence = "Not provided"
    confidence_explanation = ""
    
    # Look for confidence indicators
    if "Confidence:" in response:
        parts = response.split("Confidence:")
        answer = parts[0].strip()
        confidence_part = parts[1].strip()
        
        # Try to extract the numeric rating and explanation
        import re
        rating_match = re.search(r'(\d+)(/10)?', confidence_part)
        if rating_match:
            confidence = f"{rating_match.group(1)}/10"
            # Get explanation (everything after the number)
            explanation_text = re.sub(r'^\d+(/10)?', '', confidence_part).strip()
            if explanation_text:
                confidence_explanation = explanation_text
    
    return {
        "answer": answer,
        "confidence": confidence,
        "explanation": confidence_explanation
    }

# Create a user interface that can process questions and return confident answers    
 def ask_question(question):
    """Process a user question and return answer with confidence."""
    
    # Get raw response from the model
    raw_response = qa_chain.run(question=question)
    
    # Parse the response
    parsed = parse_response(raw_response)
    
    # Display the results
    print(f"Question: {question}\n")
    print(f"Answer: {parsed['answer']}\n")
    print(f"Confidence: {parsed['confidence']}\n")
    
    # Suggest follow-up if confidence is low
    if any(low_conf in parsed['confidence'].lower() for low_conf in ["1/10", "2/10", "3/10", "low"]):
        print("This answer has low confidence. Consider rephrasing your question or asking for more specific information.")
        
    return parsed

# Test with a few questions. Swap out examples with your own!
questions = [
    "What is the capital of France?",
    "How do quantum computers work?",
    "When was the obscure book 'Trilby' by George du Maurier published?"
]

for q in questions:
    result = ask_question(q)
    print("-" * 50)
    
from langchain.memory import ConversationBufferMemory

from langchain.memory import ConversationBufferMemory

# Create a memory instance
memory = ConversationBufferMemory(input_key="question", memory_key="chat_history")

# Update the chain with memory
qa_chain_with_memory = LLMChain(
    llm=model,
    prompt=prompt,
    memory=memory
)

# Function to chat with memory
def chat_with_memory(question):
    # Get response from chain with memory
    raw_response = qa_chain_with_memory.run(question=question)
    
    # Parse and display as before
    parsed = parse_response(raw_response)
    print(f"Question: {question}\n")
    print(f"Answer: {parsed['answer']}\n")
    print(f"Confidence: {parsed['confidence']}")
    
    if parsed['explanation']:
        print(f"Explanation: {parsed['explanation']}\n")
    
    # Suggest follow-up if confidence is low
    if any(low_conf in parsed['confidence'].lower() for low_conf in ["1/10", "2/10", "3/10"]):
        print("\nThis answer has low confidence. Consider rephrasing your question or asking for more specific information.")
    
    return parsed

# Try a conversation
print("Starting conversation with memory...")
chat_with_memory("What is machine learning?")
chat_with_memory("What are some common algorithms used in it?")
chat_with_memory("Which one would be best for image recognition?")