Large portion of time spent on sample time about llama-cpp-agent HOT 8 CLOSED

>>> from llama_cpp_agent.gbnf_grammar_generator.gbnf_grammar_from_pydantic_models import generate_gbnf_grammar_from_pydantic_models
>>> generate_gbnf_grammar_from_pydantic_models([Book])
'root ::= (" "| "\\n") grammar-models\ngrammar-models ::= book\nbook ::= "{"  ws "\\"title\\"" ": " string ","  ws "\\"author\\"" ": " string ","  ws "\\"published_year\\"" ": " number ","  ws "\\"keywords\\"" ": " book-keywords ","  ws "\\"category\\"" ": " book-category ","  ws "\\"summary\\"" ": " string ws "}"\nbook-keywords ::= "[" ws string ("," ws string)* ws "]" \nbook-category ::= "\\"Fiction\\"" | "\\"Non-Fiction\\""'

from enum import Enum

from llama_cpp import Llama
from pydantic import BaseModel, Field

from llama_cpp_agent.gbnf_grammar_generator.gbnf_grammar_from_pydantic_models import \
    generate_gbnf_grammar_and_documentation
from llama_cpp_agent.llm_prompt_template import PromptTemplate
from llama_cpp_agent.llm_settings import LlamaLLMGenerationSettings
from llama_cpp_agent.messages_formatter import MessagesFormatterType, get_predefined_messages_formatter
from llama_cpp_agent.structured_output_agent import StructuredOutputAgent

settings = LlamaLLMGenerationSettings(stream=False)
main_model = Llama(
    "../gguf-models/Meta-Llama-3-8B-Instruct.Q5_k_m_with_temp_stop_token_fix.gguf",
    n_gpu_layers=-1,
    use_mlock=False,
    embedding=False,
    n_threads=12,
    n_batch=2048,
    n_ctx=2048,
    last_n_tokens_size=1024,
    verbose=True,
    seed=42,
    stream=True
)
# Example enum for our output model
class Category(Enum):
    Fiction = "Fiction"
    NonFiction = "Non-Fiction"


# Example output model
class Book(BaseModel):
    """
    Represents an entry about a book.
    """
    title: str = Field(..., description="Title of the book.")
    author: str = Field(..., description="Author of the book.")
    published_year: int = Field(..., description="Publishing year of the book.")
    keywords: list[str] = Field(..., description="A list of keywords.")
    category: Category = Field(..., description="Category of the book.")
    summary: str = Field(..., description="Summary of the book.")


structured_output_agent = StructuredOutputAgent(main_model, llama_generation_settings=settings,
                                                messages_formatter_type=MessagesFormatterType.LLAMA_3,
                                                debug_output=False)

text = """The Feynman Lectures on Physics is a physics textbook based on some lectures by Richard Feynman, a Nobel laureate who has sometimes been called "The Great Explainer". The lectures were presented before undergraduate students at the California Institute of Technology (Caltech), during 1961–1963. The book's co-authors are Feynman, Robert B. Leighton, and Matthew Sands."""
print(structured_output_agent.create_object(Book, text))

grammar, documentation = generate_gbnf_grammar_and_documentation(
    [Book],
    model_prefix="Response Model",
    fields_prefix="Response Model Field",
)

sys_prompt_template = PromptTemplate.from_string(
    "You are an advanced AI agent. You are tasked to assist the user by creating structured output in JSON format.\n\n{documentation}"
)
creation_prompt_template = PromptTemplate.from_string(
    "Create an JSON response based on the following input.\n\nInput:\n\n{user_input}"
)
sys_prompt = sys_prompt_template.generate_prompt({"documentation": documentation})
msg = creation_prompt_template.generate_prompt({"user_input": text})

sys_msg = {"role": "system", "content": msg}
user_msg = {"role": "user", "content": msg}

msg_list = [sys_msg, user_msg]
formatter = get_predefined_messages_formatter(MessagesFormatterType.LLAMA_3)
prompt, role = formatter.format_messages(msg_list, "assistant")
settings_dic = settings.as_dict()
settings_dic["stop"] = settings_dic["stop_sequences"]
settings_dic.pop("stop_sequences")
settings_dic.pop("print_output")
main_model.reset()
print(main_model.create_completion(prompt, **settings.as_dict()))