代码

import ray
from byzerllm.utils.client import ByzerLLM,LLMRequest,InferBackend
ray.init(address="auto",namespace="default",ignore_reinit_error=True)
llm = ByzerLLM()

llm.setup_gpus_per_worker(0.4).setup_num_workers(2).setup_infer_backend(InferBackend.Transformers)
llm.deploy(
    model_path="/home/dev/model/assets/embeddings/BAAI/bge-base-zh-v1.5/",
    pretrained_model_type="custom/bge",
    udf_name="emb",
    infer_params={}
)   

Failed to look up actor with name 'emb'. This could because 1. You are trying to look up a named actor you didn't create. 2. The named actor died. 3. You did not use a namespace matching the namespace of the actor.

Traceback (most recent call last):
File "/home/dev/liuyu/project/llm-os/tests/ray_learn/模型.py", line 7, in
llm.deploy(
File "/home/dev/anaconda3/envs/vllm/lib/python3.10/site-packages/byzerllm/utils/client/init.py", line 681, in deploy
UDFBuilder.build(self.ray_context,init_model,getattr(predict_module,predict_func))
File "/home/dev/anaconda3/envs/vllm/lib/python3.10/site-packages/pyjava/udf/init.py", line 211, in build
ray.get(temp_udf_master.create_workers.remote(conf))
File "/home/dev/anaconda3/envs/vllm/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
return fn(*args, **kwargs)
File "/home/dev/anaconda3/envs/vllm/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
return func(*args, **kwargs)
File "/home/dev/anaconda3/envs/vllm/lib/python3.10/site-packages/ray/_private/worker.py", line 2626, in get
raise value
ray.exceptions.RayActorError: The actor died unexpectedly before finishing this task.
class_name: UDFMaster
actor_id: b440901c458825c8c30d666f0a000000
pid: 3395398
name: emb
namespace: default
ip: 192.168.102.19
The actor is dead because its worker process has died. Worker exit type: SYSTEM_ERROR Worker exit detail: Worker unexpectedly exits with a connection error code 2. End of file. There are some potential root causes. (1) The process is killed by SIGKILL by OOM killer due to high memory usage. (2) ray stop --force is called. (3) The worker is crashed unexpectedly due to SIGSEGV or other unexpected errors.
The actor never ran - it was cancelled before it started running.

• 现象

调用llm.stream_chat_oai接口，前后推理的重复数据较多

• 排查
  通过本地加打点debug，发现是同一条数据被get_item多次导致
  
  

data_副本.txt
使用下面的代码运行的时候出现下面的异常
`/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/langchain/init.py:29: UserWarning: Importing PromptTemplate from langchain root module is no longer supported. Please use langchain_core.prompts.PromptTemplate instead.
warnings.warn(
/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/langchain/init.py:29: UserWarning: Importing PromptTemplate from langchain root module is no longer supported. Please use langchain_core.prompts.PromptTemplate instead.
warnings.warn(
/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/langchain/init.py:29: UserWarning: Importing PromptTemplate from langchain root module is no longer supported. Please use langchain_core.prompts.PromptTemplate instead.
warnings.warn(
2024-05-07 16:49:14,487 INFO worker.py:1564 -- Connecting to existing Ray cluster at address: 127.0.0.1:6379...
2024-05-07 16:49:14,508 INFO worker.py:1740 -- Connected to Ray cluster. View the dashboard at 127.0.0.1:8265
(UDFWorker pid=84793) MODEL[qianwen_chat] Init Model,It may take a while.
(UDFWorker pid=84793) MODEL[qianwen_chat] Successful to init model, time taken:0.001135110855102539s
use_shared_disk: False file_path: /Users/xxxxxx/Downloads/效期运营数据c210dd6b5671438bbe3b82d1c6dd53b0.csv new_file_path: /Users/xxxxxx/Downloads/data_analysis_pp_1d01a0e2c64fc8e68128021122f765b9.csv
Traceback (most recent call last):
File "python/ray/_raylet.pyx", line 903, in ray._raylet.prepare_args_internal
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/_private/serialization.py", line 502, in serialize
return self._serialize_to_msgpack(value)
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/_private/serialization.py", line 480, in _serialize_to_msgpack
pickle5_serialized_object = self._serialize_to_pickle5(
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/_private/serialization.py", line 433, in _serialize_to_pickle5
raise e
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/_private/serialization.py", line 428, in _serialize_to_pickle5
inband = pickle.dumps(
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/cloudpickle/cloudpickle.py", line 1479, in dumps
cp.dump(obj)
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/cloudpickle/cloudpickle.py", line 1245, in dump
return super().dump(obj)
TypeError: cannot pickle 'generator' object

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/Users/xxxxxx/Desktop/data.py", line 34, in
data_analysis = DataAnalysis("chat4","qing","/Users/xxxxxx/Downloads/效期运营数据c210dd6b5671438bbe3b82d1c6dd53b0.csv",
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/byzerllm/apps/agent/extensions/data_analysis.py", line 81, in init
self.data_analysis_pipeline = ray.get(self.manager.get_or_create_pipeline.remote(
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/actor.py", line 202, in remote
return self._remote(args, kwargs)
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 21, in auto_init_wrapper
return fn(*args, **kwargs)
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/util/tracing/tracing_helper.py", line 426, in _start_span
return method(self, args, kwargs, *_args, **_kwargs)
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/actor.py", line 327, in _remote
return invocation(args, kwargs)
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/actor.py", line 308, in invocation
return actor._actor_method_call(
File "/Users/xxxxxx/miniforge3/lib/python3.10/site-packages/ray/actor.py", line 1441, in _actor_method_call
object_refs = worker.core_worker.submit_actor_task(
File "python/ray/_raylet.pyx", line 4178, in ray._raylet.CoreWorker.submit_actor_task
File "python/ray/_raylet.pyx", line 4183, in ray._raylet.CoreWorker.submit_actor_task
File "python/ray/_raylet.pyx", line 865, in ray._raylet.prepare_args_and_increment_put_refs
File "python/ray/_raylet.pyx", line 856, in ray._raylet.prepare_args_and_increment_put_refs
File "python/ray/_raylet.pyx", line 912, in ray._raylet.prepare_args_internal
TypeError: Could not serialize the argument <byzerllm.utils.client.byzerllm_client.ByzerLLM object at 0x10285a470> for a task or actor byzerllm.apps.agent.extensions.data_analysis_pipeline_agent.DataAnalysisPipelineManager.get_or_create_pipeline:

Checking Serializability of <byzerllm.utils.client.byzerllm_client.ByzerLLM object at 0x10285a470>

!!! FAIL serialization: cannot pickle 'generator' object
Serializing '_generate_ins' <bound method ByzerLLM._generate_ins of <byzerllm.utils.client.byzerllm_client.ByzerLLM object at 0x10285a470>>...
!!! FAIL serialization: cannot pickle 'generator' object
Serializing 'func' <function ByzerLLM._generate_ins at 0x17d20cdc0>...
WARNING: Did not find non-serializable object in <bound method ByzerLLM._generate_ins of <byzerllm.utils.client.byzerllm_client.ByzerLLM object at 0x10285a470>>. This may be an oversight.

Variable:

    FailTuple(_generate_ins [obj=<bound method ByzerLLM._generate_ins of <byzerllm.utils.client.byzerllm_client.ByzerLLM object at 0x10285a470>>, parent=<byzerllm.utils.client.byzerllm_client.ByzerLLM object at 0x10285a470>])

was found to be non-serializable. There may be multiple other undetected variables that were non-serializable.
Consider either removing the instantiation/imports of these variables or moving the instantiation into the scope of the function/class.

Check https://docs.ray.io/en/master/ray-core/objects/serialization.html#troubleshooting for more information.
If you have any suggestions on how to improve this error message, please reach out to the Ray developers on github.com/ray-project/ray/issues/

运行环境 Python 3.10.14
Name: byzerllm Version: 0.1.80
conda miniforge3
ray version 2.20.0

个人对Python不是很熟悉，在运行过程中发现，最新的代码仓库中agent下的多个文件的引用路径不正确，当时无法运行，手动修改部分文件的路径，可以运行，但是出现这个异常。

qianfan.errors.InvalidArgumentError: The provided model ERNIE-3.5-128K is not in the list of supported models. If this is a recently added model, try using the endpoint arguments and create an issue to tell us. Supported models: {'ERNIE-Bot-turbo-AI', 'ERNIE-3.5-4K-0205', 'XuanYuan-70B-Chat-4bit', 'ChatGLM2-6B-32K', 'ERNIE-Functions-8K', 'ERNIE Speed-AppBuilder', 'Qianfan-BLOOMZ-7B-compressed', 'Qianfan-Chinese-Llama-2-13B', 'Yi-34B-Chat', 'ERNIE-3.5-8K-preview', 'ERNIE-Tiny-8K', 'ERNIE-Character-8K', 'Llama-2-70B-Chat', 'Llama-2-7B-Chat', 'ERNIE-Speed-128K', 'Mixtral-8x7B-Instruct', 'ERNIE Speed', 'ERNIE-4.0-preemptible', 'Meta-Llama-3-8B', 'Qianfan-Chinese-Llama-2-7B', 'Gemma-7B-it', 'ERNIE-3.5-8K-preemptible', 'ERNIE-Bot', 'BLOOMZ-7B', 'EB-turbo-AppBuilder', 'ERNIE-3.5-8K', 'ERNIE 3.5', 'ERNIE-Lite-8K', 'AquilaChat-7B', 'ERNIE-Speed', 'ERNIE-4.0-8K', 'Llama-2-13B-Chat', 'ERNIE-Speed-8K', 'ERNIE-Bot-8K', 'ERNIE-Lite-8K-0308', 'ERNIE-Bot-4', 'ERNIE-4.0-preview', 'ERNIE-3.5-8K-0205', 'ERNIE-Bot-turbo', 'ERNIE-Lite-8K-0922', 'ChatLaw', 'Meta-Llama-3-70B', 'ERNIE-3.5-8K-1222'}

2024年5月16日发布 ERNIE-3.5-128K，提升上下文窗口长度至128K

endpoint: /chat/ernie-3.5-128k

详细信息如下：

"ERNIE-3.5-128K": QfLLMInfo(
                endpoint="/chat/ernie-3.5-128k",
                required_keys={"messages"},
                optional_keys={
                    "stream",
                    "temperature",
                    "top_p",
                    "penalty_score",
                    "functions",
                    "system",
                    "user_id",
                    "user_setting",
                    "stop",
                    "disable_search",
                    "enable_citation",
                    "max_output_tokens",
                    "response_format",
                    "tool_choice",
                    "enable_trace",
                },
                max_input_chars=516096,
                max_input_tokens=126976,
                input_price_per_1k_tokens=0.012,
                output_price_per_1k_tokens=0.012,

官方文档： https://cloud.baidu.com/doc/WENXINWORKSHOP/s/dlw4ptsq7

如题所述，是不是 macOS 目前不支持，各版本如下：

(base) ➜  ~ byzerllm --version
Traceback (most recent call last):
  File "/Users/mintisan/miniconda3/bin/byzerllm", line 5, in <module>
    from byzerllm.byzerllm import main
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/byzerllm/__init__.py", line 3, in <module>
    from pyjava.api.mlsql import RayContext
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/__init__.py", line 20, in <module>
    from pyjava.api.mlsql import RayContext, PythonContext
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/api/__init__.py", line 5, in <module>
    from pyjava.api.mlsql import PythonContext
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/api/mlsql.py", line 11, in <module>
    import pyjava.utils as utils
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/utils.py", line 4, in <module>
    from pyjava.serializers import write_with_length, UTF8Deserializer, \
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/serializers.py", line 27, in <module>
    from pyjava import cloudpickle
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/cloudpickle.py", line 174, in <module>
    _cell_set_template_code = _make_cell_set_template_code()
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/cloudpickle.py", line 136, in _make_cell_set_template_code
    return types.CodeType(
           ^^^^^^^^^^^^^^^
TypeError: code() argument 13 must be str, not int
(base) ➜  ~ pip list | grep byzerllm
byzerllm                      0.1.53
(base) ➜  ~ 
(base) ➜  ~ 
(base) ➜  ~ python --verison
unknown option --verison
usage: python [option] ... [-c cmd | -m mod | file | -] [arg] ...
Try `python -h' for more information.
(base) ➜  ~ python --V      
unknown option --V
usage: python [option] ... [-c cmd | -m mod | file | -] [arg] ...
Try `python -h' for more information.
(base) ➜  ~ python -V 
Python 3.11.5
(base) ➜  ~ 
(base) ➜  ~ auto-coder 
Traceback (most recent call last):
  File "/Users/mintisan/miniconda3/bin/auto-coder", line 5, in <module>
    from autocoder.auto_coder import main
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/autocoder/auto_coder.py", line 1, in <module>
    import byzerllm
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/byzerllm/__init__.py", line 3, in <module>
    from pyjava.api.mlsql import RayContext
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/__init__.py", line 20, in <module>
    from pyjava.api.mlsql import RayContext, PythonContext
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/api/__init__.py", line 5, in <module>
    from pyjava.api.mlsql import PythonContext
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/api/mlsql.py", line 11, in <module>
    import pyjava.utils as utils
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/utils.py", line 4, in <module>
    from pyjava.serializers import write_with_length, UTF8Deserializer, \
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/serializers.py", line 27, in <module>
    from pyjava import cloudpickle
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/cloudpickle.py", line 174, in <module>
    _cell_set_template_code = _make_cell_set_template_code()
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mintisan/miniconda3/lib/python3.11/site-packages/pyjava/cloudpickle.py", line 136, in _make_cell_set_template_code
    return types.CodeType(
           ^^^^^^^^^^^^^^^
TypeError: code() argument 13 must be str, not int
(base) ➜  ~ pip list | grep auto-coder
auto-coder                    0.1.18
(base) ➜  ~ 

# https://platform.openai.com/docs/guides/function-calling
from openai import OpenAI
import json

client = OpenAI(
    base_url="http://127.0.0.1:8000/v1",
    api_key="simple"
)

# Example dummy function hard coded to return the same weather
# In production, this could be your backend API or an external API
def get_current_weather(location, unit="fahrenheit"):
    """Get the current weather in a given location"""
    if "tokyo" in location.lower():
        return json.dumps({"location": "Tokyo", "temperature": "10", "unit": unit})
    elif "san francisco" in location.lower():
        return json.dumps({"location": "San Francisco", "temperature": "72", "unit": unit})
    elif "paris" in location.lower():
        return json.dumps({"location": "Paris", "temperature": "22", "unit": unit})
    else:
        return json.dumps({"location": location, "temperature": "unknown"})

def run_conversation():
    # Step 1: send the conversation and available functions to the model
    tools = [
        {
            "type": "function",
            "function": {
                "name": "get_current_weather",
                "description": "Get the current weather in a given location",
                "parameters": {
                    "type": "object",
                    "properties": {
                        "location": {
                            "type": "string",
                            "description": "The city and state, e.g. San Francisco, CA",
                        },
                        "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
                    },
                    "required": ["location"],
                },
            },
        }
    ]
   
    messages = [{
                "role": "system", 
                "content": f"你是一个人工作智能助手，帮助回答用户的问题，必要时可以调用工具，回答用户问题。下面是你可以调用的工具:\n{tools}\n  请明确返回是否需要调用工具，并返回调用该工具所需的参数。"
            },
            {"role": "user", 
             "content": "What's the weather like in San Francisco, Tokyo, and Paris?"}
            ]
            
    
    response = client.chat.completions.create(
        model="chat",
        messages=messages,
        tools=tools,
        tool_choice="auto",  # auto is default, but we'll be explicit
    )
    response_message = response.choices[0].message
    print(response_message)
    tool_calls = response_message.tool_calls
    # Step 2: check if the model wanted to call a function
    if tool_calls:
        # Step 3: call the function
        # Note: the JSON response may not always be valid; be sure to handle errors
        available_functions = {
            "get_current_weather": get_current_weather,
        }  # only one function in this example, but you can have multiple
        messages.append(response_message)  # extend conversation with assistant's reply
        # Step 4: send the info for each function call and function response to the model
        for tool_call in tool_calls:
            function_name = tool_call.function.name
            function_to_call = available_functions[function_name]
            function_args = json.loads(tool_call.function.arguments)
            function_response = function_to_call(
                location=function_args.get("location"),
                unit=function_args.get("unit"),
            )
            messages.append(
                {
                    "tool_call_id": tool_call.id,
                    "role": "tool",
                    "name": function_name,
                    "content": function_response,
                }
            )  # extend conversation with function response
        second_response = client.chat.completions.create(
            model="chat",
            messages=messages,
        )  # get a new response from the model where it can see the function response
        return second_response
print(run_conversation())

打印返回结果如下:
ChatCompletionMessage(content="Yes, I need to call the tool to get the current weather for each location. Here are the parameters for each city:\n\n1. San Francisco, CA\n2. Tokyo, Japan\n3. Paris, France\n\nUnit will be assumed as 'celsius' for all cities unless specified otherwise.", role='assistant', function_call=None, tool_calls=None)

readme的中英文链接弄反了
| English | 中文 |