Compare commits

...

3 Commits

Author SHA1 Message Date
Novice
bb45ae381d fix: mypy issues 2025-03-14 15:08:32 +08:00
Novice
61b427a4f1 fix: use a generator to return non-streaming requests 2025-03-14 14:54:36 +08:00
Novice
d41c699513 feat: add llm blocking invoke 2025-03-13 17:10:12 +08:00

View File

@@ -3,7 +3,7 @@ from binascii import hexlify, unhexlify
from collections.abc import Generator
from core.model_manager import ModelManager
from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta
from core.model_runtime.entities.message_entities import (
PromptMessage,
SystemPromptMessage,
@@ -46,7 +46,7 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
model_parameters=payload.completion_params,
tools=payload.tools,
stop=payload.stop,
stream=payload.stream or True,
stream=True if payload.stream is None else payload.stream,
user=user_id,
)
@@ -64,7 +64,21 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
else:
if response.usage:
LLMNode.deduct_llm_quota(tenant_id=tenant.id, model_instance=model_instance, usage=response.usage)
return response
def handle_non_streaming(response: LLMResult) -> Generator[LLMResultChunk, None, None]:
yield LLMResultChunk(
model=response.model,
prompt_messages=response.prompt_messages,
system_fingerprint=response.system_fingerprint,
delta=LLMResultChunkDelta(
index=0,
message=response.message,
usage=response.usage,
finish_reason="",
),
)
return handle_non_streaming(response)
@classmethod
def invoke_text_embedding(cls, user_id: str, tenant: Tenant, payload: RequestInvokeTextEmbedding):