MPT No response

[lalit34-ai]

I am trying to generate responses based on the input from mpt-30B model and create a API using flask but I am having trouble as it not giving response i.e. the response is empty for each input I am asking. I am using Standard F32s v2 (32 vcpus, 64 GiB memory) with remote access to server.

from flask import Flask, request, Response
from dataclasses import dataclass, asdict
from ctransformers import AutoModelForCausalLM, AutoConfig
import os
import time

app = Flask(name)

@DataClass
class GenerationConfig:
temperature: float
top_k: int
top_p: float
repetition_penalty: float
max_new_tokens: int
seed: int
reset: bool
stream: bool
threads: int
stop: list

def format_prompt(system_prompt: str, user_prompt: str):
system_prompt = f"system\n{system_prompt}\n"
user_prompt = f"user\n{user_prompt}\n"
assistant_prompt = f"assistant\n"
return f"{system_prompt}{user_prompt}{assistant_prompt}"

def generate(
llm: AutoModelForCausalLM,
generation_config: GenerationConfig,
system_prompt: str,
user_input: str,
):
# return llm(
# format_prompt(
# system_prompt,
# user_prompt,
# ),
# **asdict(generation_config),
# )
model_output = llm(
format_prompt(system_prompt, user_input.strip()),
**asdict(generation_config),
)
print("Model output:", model_output)
return model_output

@app.route('/generate', methods=['GET','POST'])
def generate_response_endpoint():
#user_input = request.data.decode('utf-8')
# Load the model and configuration
if request.method == 'GET':
user_input = request.args.get('user_input', '') # Get input from query parameter
elif request.method == 'POST':
user_input = request.data.decode('utf-8')

print("Loading model...")
#config = AutoConfig.from_pretrained("mosaicml/mpt-30b-chat", context_length=8192)
llm = AutoModelForCausalLM.from_pretrained(
    "/home/azureuser/mpt-30B-inference/models/mpt-30b-chat.ggmlv0.q4_1.bin",
    model_type="mpt"
)
print("model Loaded")

system_prompt = "Reply."

generation_config = GenerationConfig(
    temperature=0.2,
    top_k=0,
    top_p=0.9,
    repetition_penalty=1.0,
    max_new_tokens=512, 
    seed=42,
    reset=False,  
    stream=False, 
    threads=int(os.cpu_count() / 2),  # adjust for your CPU
    stop=["", "|<"],
)

generator = generate(llm, generation_config, system_prompt, user_input.strip())
#time.sleep(60)

print(generator)

response = generator

print(response)
return Response(response, content_type='text/plain; charset=utf-8')

if name == "main":
app.run(host='0.0.0.0', port=3002)

iimport requests

while True:
user_input = input("You: ")
if user_input.lower() in ['exit', 'quit']:
print("Exiting...")
break

data = {'user_input': user_input+"Respond to this."}
response = requests.post('http://127.0.0.1:3002/generate', json=data)

if response.status_code == 200:
    assistant_response = response.text
#    assistant_response = assistant_response.replace("You:", "").replace("Assistant:", "").strip()
    print("Assistant:", assistant_response)
else:
    print("Error:", response.status_code)   

but the repsonse which i am getting is empty as can be seen below:-

You: 3+4
Assistant:
You:

Any idea what may be causing this issue here and what can be done to resolve this?