deal dataset vector

Co-authored-by: yyzhang <yuanyuan.zhang@haochezhu.club>

.github/ISSUE_TEMPLATE/🐛-bug-report.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41B Bug report"
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+<!--
+  Please provide a clear and concise description of what the bug is. Include
+  screenshots if needed. Please test using the latest version of the relevant
+  Dify packages to make sure your issue has not already been fixed.
+-->
+
+Dify version: Cloud | Self Host
+
+## Steps To Reproduce
+<!--
+  Your bug will get fixed much faster if we can run your code and it doesn't
+  have dependencies other than Dify. Issues without reproduction steps or
+  code examples may be immediately closed as not actionable.
+-->
+
+1.
+2.
+
+
+## The current behavior
+
+
+## The expected behavior

.github/ISSUE_TEMPLATE/🚀-feature-request.md

@@ -0,0 +1,20 @@
+---
+name: "\U0001F680 Feature request"
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/🤔-questions-and-help.md

@@ -0,0 +1,10 @@
+---
+name: "\U0001F914 Questions and Help"
+about: Ask a usage or consultation question
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

api/README.md

@@ -33,3 +33,4 @@
    flask run --host 0.0.0.0 --port=5001 --debug
    ```
 7. Setup your application by visiting http://localhost:5001/console/api/setup or other apis...
+8. If you need to debug local async processing, you can run `celery -A app.celery worker`, celery can do dataset importing and other async tasks.

api/controllers/console/app/__init__.py

@@ -17,6 +17,6 @@ def _get_app(app_id, mode=None):
         raise NotFound("App not found")
 
     if mode and app.mode != mode:
-        raise AppUnavailableError()
+        raise NotFound("The {} app not found".format(mode))
 
     return app

api/controllers/console/app/error.py

@@ -9,31 +9,33 @@ class AppNotFoundError(BaseHTTPException):
 
 class ProviderNotInitializeError(BaseHTTPException):
     error_code = 'provider_not_initialize'
-    description = "Provider Token not initialize."
+    description = "No valid model provider credentials found. " \
+                  "Please go to Settings -> Model Provider to complete your provider credentials."
     code = 400
 
 
 class ProviderQuotaExceededError(BaseHTTPException):
     error_code = 'provider_quota_exceeded'
-    description = "Provider quota exceeded."
+    description = "Your quota for Dify Hosted OpenAI has been exhausted. " \
+                  "Please go to Settings -> Model Provider to complete your own provider credentials."
     code = 400
 
 
 class ProviderModelCurrentlyNotSupportError(BaseHTTPException):
     error_code = 'model_currently_not_support'
-    description = "GPT-4 currently not support."
+    description = "Dify Hosted OpenAI trial currently not support the GPT-4 model."
     code = 400
 
 
 class ConversationCompletedError(BaseHTTPException):
     error_code = 'conversation_completed'
-    description = "Conversation was completed."
+    description = "The conversation has ended. Please start a new conversation."
     code = 400
 
 
 class AppUnavailableError(BaseHTTPException):
     error_code = 'app_unavailable'
-    description = "App unavailable."
+    description = "App unavailable, please check your app configurations."
     code = 400
 
 
@@ -45,5 +47,5 @@ class CompletionRequestError(BaseHTTPException):
 
 class AppMoreLikeThisDisabledError(BaseHTTPException):
     error_code = 'app_more_like_this_disabled'
-    description = "More like this disabled."
+    description = "The 'More like this' feature is disabled. Please refresh your page."
     code = 403

api/controllers/console/datasets/datasets_document.py

@@ -10,13 +10,14 @@ from werkzeug.exceptions import NotFound, Forbidden
 
 import services
 from controllers.console import api
-from controllers.console.app.error import ProviderNotInitializeError
+from controllers.console.app.error import ProviderNotInitializeError, ProviderQuotaExceededError, \
+    ProviderModelCurrentlyNotSupportError
 from controllers.console.datasets.error import DocumentAlreadyFinishedError, InvalidActionError, DocumentIndexingError, \
     InvalidMetadataError, ArchivedDocumentImmutableError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.indexing_runner import IndexingRunner
-from core.llm.error import ProviderTokenNotInitError
+from core.llm.error import ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from extensions.ext_redis import redis_client
 from libs.helper import TimestampField
 from extensions.ext_database import db
@@ -222,6 +223,10 @@ class DatasetDocumentListApi(Resource):
             document = DocumentService.save_document_with_dataset_id(dataset, args, current_user)
         except ProviderTokenNotInitError:
             raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
 
         return document
 
@@ -259,6 +264,10 @@ class DatasetInitApi(Resource):
             )
         except ProviderTokenNotInitError:
             raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
 
         response = {
             'dataset': dataset,

api/controllers/console/datasets/error.py

@@ -3,7 +3,7 @@ from libs.exception import BaseHTTPException
 
 class NoFileUploadedError(BaseHTTPException):
     error_code = 'no_file_uploaded'
-    description = "No file uploaded."
+    description = "Please upload your file."
     code = 400
 
 
@@ -27,25 +27,25 @@ class UnsupportedFileTypeError(BaseHTTPException):
 
 class HighQualityDatasetOnlyError(BaseHTTPException):
     error_code = 'high_quality_dataset_only'
-    description = "High quality dataset only."
+    description = "Current operation only supports 'high-quality' datasets."
     code = 400
 
 
 class DatasetNotInitializedError(BaseHTTPException):
     error_code = 'dataset_not_initialized'
-    description = "Dataset not initialized."
+    description = "The dataset is still being initialized or indexing. Please wait a moment."
     code = 400
 
 
 class ArchivedDocumentImmutableError(BaseHTTPException):
     error_code = 'archived_document_immutable'
-    description = "Cannot process an archived document."
+    description = "The archived document is not editable."
     code = 403
 
 
 class DatasetNameDuplicateError(BaseHTTPException):
     error_code = 'dataset_name_duplicate'
-    description = "Dataset name already exists."
+    description = "The dataset name already exists. Please modify your dataset name."
     code = 409
 
 
@@ -57,17 +57,17 @@ class InvalidActionError(BaseHTTPException):
 
 class DocumentAlreadyFinishedError(BaseHTTPException):
     error_code = 'document_already_finished'
-    description = "Document already finished."
+    description = "The document has been processed. Please refresh the page or go to the document details."
     code = 400
 
 
 class DocumentIndexingError(BaseHTTPException):
     error_code = 'document_indexing'
-    description = "Document indexing."
+    description = "The document is being processed and cannot be edited."
     code = 400
 
 
 class InvalidMetadataError(BaseHTTPException):
     error_code = 'invalid_metadata'
-    description = "Invalid metadata."
+    description = "The metadata content is incorrect. Please check and verify."
     code = 400

api/controllers/console/datasets/hit_testing.py

@@ -6,9 +6,12 @@ from werkzeug.exceptions import InternalServerError, NotFound, Forbidden
 
 import services
 from controllers.console import api
+from controllers.console.app.error import ProviderNotInitializeError, ProviderQuotaExceededError, \
+    ProviderModelCurrentlyNotSupportError
 from controllers.console.datasets.error import HighQualityDatasetOnlyError, DatasetNotInitializedError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
+from core.llm.error import ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from libs.helper import TimestampField
 from services.dataset_service import DatasetService
 from services.hit_testing_service import HitTestingService
@@ -92,6 +95,12 @@ class HitTestingApi(Resource):
             return {"query": response['query'], 'records': marshal(response['records'], hit_testing_record_fields)}
         except services.errors.index.IndexNotInitializedError:
             raise DatasetNotInitializedError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
         except Exception as e:
             logging.exception("Hit testing failed.")
             raise InternalServerError(str(e))

api/controllers/console/error.py

@@ -3,13 +3,14 @@ from libs.exception import BaseHTTPException
 
 class AlreadySetupError(BaseHTTPException):
     error_code = 'already_setup'
-    description = "Application already setup."
+    description = "Dify has been successfully installed. Please refresh the page or return to the dashboard homepage."
     code = 403
 
 
 class NotSetupError(BaseHTTPException):
     error_code = 'not_setup'
-    description = "Application not setup."
+    description = "Dify has not been initialized and installed yet. " \
+                  "Please proceed with the initialization and installation process first."
     code = 401
 
 

api/controllers/console/workspace/error.py

@@ -21,11 +21,11 @@ class InvalidInvitationCodeError(BaseHTTPException):
 
 class AccountAlreadyInitedError(BaseHTTPException):
     error_code = 'account_already_inited'
-    description = "Account already inited."
+    description = "The account has been initialized. Please refresh the page."
     code = 400
 
 
 class AccountNotInitializedError(BaseHTTPException):
     error_code = 'account_not_initialized'
-    description = "Account not initialized."
+    description = "The account has not been initialized yet. Please proceed with the initialization process first."
     code = 400

api/controllers/console/workspace/providers.py

@@ -90,8 +90,8 @@ class ProviderTokenApi(Resource):
                     configs=args['token']
                 )
                 token_is_valid = True
-            except ValidateFailedError:
-                token_is_valid = False
+            except ValidateFailedError as ex:
+                raise ValueError(str(ex))
 
             base64_encrypted_token = ProviderService.get_encrypted_token(
                 tenant=current_user.current_tenant,

api/controllers/service_api/app/error.py

@@ -4,43 +4,45 @@ from libs.exception import BaseHTTPException
 
 class AppUnavailableError(BaseHTTPException):
     error_code = 'app_unavailable'
-    description = "App unavailable."
+    description = "App unavailable, please check your app configurations."
     code = 400
 
 
 class NotCompletionAppError(BaseHTTPException):
     error_code = 'not_completion_app'
-    description = "Not Completion App"
+    description = "Please check if your Completion app mode matches the right API route."
     code = 400
 
 
 class NotChatAppError(BaseHTTPException):
     error_code = 'not_chat_app'
-    description = "Not Chat App"
+    description = "Please check if your Chat app mode matches the right API route."
     code = 400
 
 
 class ConversationCompletedError(BaseHTTPException):
     error_code = 'conversation_completed'
-    description = "Conversation Completed."
+    description = "The conversation has ended. Please start a new conversation."
     code = 400
 
 
 class ProviderNotInitializeError(BaseHTTPException):
     error_code = 'provider_not_initialize'
-    description = "Provider Token not initialize."
+    description = "No valid model provider credentials found. " \
+                  "Please go to Settings -> Model Provider to complete your provider credentials."
     code = 400
 
 
 class ProviderQuotaExceededError(BaseHTTPException):
     error_code = 'provider_quota_exceeded'
-    description = "Provider quota exceeded."
+    description = "Your quota for Dify Hosted OpenAI has been exhausted. " \
+                  "Please go to Settings -> Model Provider to complete your own provider credentials."
     code = 400
 
 
 class ProviderModelCurrentlyNotSupportError(BaseHTTPException):
     error_code = 'model_currently_not_support'
-    description = "GPT-4 currently not support."
+    description = "Dify Hosted OpenAI trial currently not support the GPT-4 model."
     code = 400
 
 

api/controllers/service_api/dataset/error.py

@@ -16,5 +16,5 @@ class DocumentIndexingError(BaseHTTPException):
 
 class DatasetNotInitedError(BaseHTTPException):
     error_code = 'dataset_not_inited'
-    description = "Dataset not inited."
+    description = "The dataset is still being initialized or indexing. Please wait a moment."
     code = 403

api/controllers/web/error.py

@@ -4,43 +4,45 @@ from libs.exception import BaseHTTPException
 
 class AppUnavailableError(BaseHTTPException):
     error_code = 'app_unavailable'
-    description = "App unavailable."
+    description = "App unavailable, please check your app configurations."
     code = 400
 
 
 class NotCompletionAppError(BaseHTTPException):
     error_code = 'not_completion_app'
-    description = "Not Completion App"
+    description = "Please check if your Completion app mode matches the right API route."
     code = 400
 
 
 class NotChatAppError(BaseHTTPException):
     error_code = 'not_chat_app'
-    description = "Not Chat App"
+    description = "Please check if your Chat app mode matches the right API route."
     code = 400
 
 
 class ConversationCompletedError(BaseHTTPException):
     error_code = 'conversation_completed'
-    description = "Conversation Completed."
+    description = "The conversation has ended. Please start a new conversation."
     code = 400
 
 
 class ProviderNotInitializeError(BaseHTTPException):
     error_code = 'provider_not_initialize'
-    description = "Provider Token not initialize."
+    description = "No valid model provider credentials found. " \
+                  "Please go to Settings -> Model Provider to complete your provider credentials."
     code = 400
 
 
 class ProviderQuotaExceededError(BaseHTTPException):
     error_code = 'provider_quota_exceeded'
-    description = "Provider quota exceeded."
+    description = "Your quota for Dify Hosted OpenAI has been exhausted. " \
+                  "Please go to Settings -> Model Provider to complete your own provider credentials."
     code = 400
 
 
 class ProviderModelCurrentlyNotSupportError(BaseHTTPException):
     error_code = 'model_currently_not_support'
-    description = "GPT-4 currently not support."
+    description = "Dify Hosted OpenAI trial currently not support the GPT-4 model."
     code = 400
 
 
@@ -52,11 +54,11 @@ class CompletionRequestError(BaseHTTPException):
 
 class AppMoreLikeThisDisabledError(BaseHTTPException):
     error_code = 'app_more_like_this_disabled'
-    description = "More like this disabled."
+    description = "The 'More like this' feature is disabled. Please refresh your page."
     code = 403
 
 
 class AppSuggestedQuestionsAfterAnswerDisabledError(BaseHTTPException):
     error_code = 'app_suggested_questions_after_answer_disabled'
-    description = "Function Suggested questions after answer disabled."
+    description = "The 'Suggested Questions After Answer' feature is disabled. Please refresh your page."
     code = 403

api/core/completion.py

@@ -125,13 +125,17 @@ class Completion:
         pre_prompt = PromptBuilder.process_template(pre_prompt) if pre_prompt else pre_prompt
         if mode == 'completion':
             prompt_template = OutLinePromptTemplate.from_template(
-                template=("Use the following pieces of [CONTEXT] to answer the question at the end. "
-                          "If you don't know the answer, "
-                          "just say that you don't know, don't try to make up an answer. \n"
-                          "```\n"
-                          "[CONTEXT]\n"
-                          "{context}\n"
-                          "```\n" if chain_output else "")
+                template=("""Use the following CONTEXT as your learned knowledge:
+[CONTEXT]
+{context}
+[END CONTEXT]
+
+When answer to user:
+- If you don't know, just say that you don't know.
+- If you don't know when you are not sure, ask for clarification. 
+Avoid mentioning that you obtained the information from the context.
+And answer according to the language of the user's question.
+""" if chain_output else "")
                          + (pre_prompt + "\n" if pre_prompt else "")
                          + "{query}\n"
             )
@@ -153,38 +157,46 @@ class Completion:
         else:
             messages: List[BaseMessage] = []
 
-            system_message = None
-            if pre_prompt:
-                # append pre prompt as system message
-                system_message = PromptBuilder.to_system_message(pre_prompt, inputs)
-
-            if chain_output:
-                # append context as system message, currently only use simple stuff prompt
-                context_message = PromptBuilder.to_system_message(
-                    """Use the following pieces of [CONTEXT] to answer the users question. 
-If you don't know the answer, just say that you don't know, don't try to make up an answer.
-```
-[CONTEXT]
-{context}
-```""",
-                    {'context': chain_output}
-                )
-
-                if not system_message:
-                    system_message = context_message
-                else:
-                    system_message.content = context_message.content + "\n\n" + system_message.content
-
-            if system_message:
-                messages.append(system_message)
-
             human_inputs = {
                 "query": query
             }
 
+            human_message_prompt = "{query}"
+
+            if chain_output:
+                human_inputs['context'] = chain_output
+                human_message_instruction = """Use the following CONTEXT as your learned knowledge.
+[CONTEXT]
+{context}
+[END CONTEXT]
+
+When answer to user:
+- If you don't know, just say that you don't know.
+- If you don't know when you are not sure, ask for clarification. 
+Avoid mentioning that you obtained the information from the context.
+And answer according to the language of the user's question.
+"""
+                if pre_prompt:
+                    extra_inputs = {k: inputs[k] for k in
+                                    OutLinePromptTemplate.from_template(template=pre_prompt).input_variables
+                                    if k in inputs}
+                    if extra_inputs:
+                        human_inputs.update(extra_inputs)
+                    human_message_instruction += pre_prompt + "\n"
+
+                human_message_prompt = human_message_instruction + "Q:{query}\nA:"
+            else:
+                if pre_prompt:
+                    extra_inputs = {k: inputs[k] for k in
+                                    OutLinePromptTemplate.from_template(template=pre_prompt).input_variables
+                                    if k in inputs}
+                    if extra_inputs:
+                        human_inputs.update(extra_inputs)
+                    human_message_prompt = pre_prompt + "\n" + human_message_prompt
+
             # construct main prompt
             human_message = PromptBuilder.to_human_message(
-                prompt_content="{query}",
+                prompt_content=human_message_prompt,
                 inputs=human_inputs
             )
 

api/core/conversation_message_task.py

@@ -281,6 +281,9 @@ class PubHandler:
 
     @classmethod
     def generate_channel_name(cls, user: Union[Account | EndUser], task_id: str):
+        if not user:
+            raise ValueError("user is required")
+
         user_str = 'account-' + user.id if isinstance(user, Account) else 'end-user-' + user.id
         return "generate_result:{}-{}".format(user_str, task_id)
 

api/core/index/index_builder.py

@@ -46,3 +46,15 @@ class IndexBuilder:
             prompt_helper=prompt_helper,
             embed_model=OpenAIEmbedding(**model_credentials),
         )
+
+    @classmethod
+    def get_fake_llm_service_context(cls, tenant_id: str) -> ServiceContext:
+        llm = LLMBuilder.to_llm(
+            tenant_id=tenant_id,
+            model_name='fake'
+        )
+
+        return ServiceContext.from_defaults(
+            llm_predictor=LLMPredictor(llm=llm),
+            embed_model=OpenAIEmbedding()
+        )

api/core/index/vector_index.py

@@ -83,7 +83,7 @@ class VectorIndex:
         if not self._dataset.index_struct_dict:
             return
 
-        service_context = IndexBuilder.get_default_service_context(tenant_id=self._dataset.tenant_id)
+        service_context = IndexBuilder.get_fake_llm_service_context(tenant_id=self._dataset.tenant_id)
 
         index = vector_store.get_index(
             service_context=service_context,
@@ -101,7 +101,7 @@ class VectorIndex:
         if not self._dataset.index_struct_dict:
             return
 
-        service_context = IndexBuilder.get_default_service_context(tenant_id=self._dataset.tenant_id)
+        service_context = IndexBuilder.get_fake_llm_service_context(tenant_id=self._dataset.tenant_id)
 
         index = vector_store.get_index(
             service_context=service_context,

api/core/llm/provider/azure_provider.py

@@ -1,22 +1,24 @@
 import json
+import logging
 from typing import Optional, Union
 
 import requests
 
 from core.llm.provider.base import BaseProvider
+from core.llm.provider.errors import ValidateFailedError
 from models.provider import ProviderName
 
 
 class AzureProvider(BaseProvider):
-    def get_models(self, model_id: Optional[str] = None) -> list[dict]:
-        credentials = self.get_credentials(model_id)
+    def get_models(self, model_id: Optional[str] = None, credentials: Optional[dict] = None) -> list[dict]:
+        credentials = self.get_credentials(model_id) if not credentials else credentials
         url = "{}/openai/deployments?api-version={}".format(
-            credentials.get('openai_api_base'),
-            credentials.get('openai_api_version')
+            str(credentials.get('openai_api_base')),
+            str(credentials.get('openai_api_version'))
         )
 
         headers = {
-            "api-key": credentials.get('openai_api_key'),
+            "api-key": str(credentials.get('openai_api_key')),
             "content-type": "application/json; charset=utf-8"
         }
 
@@ -29,8 +31,10 @@ class AzureProvider(BaseProvider):
                 'name': '{} ({})'.format(deployment['id'], deployment['model'])
             } for deployment in result['data'] if deployment['status'] == 'succeeded']
         else:
-            # TODO: optimize in future
-            raise Exception('Failed to get deployments from Azure OpenAI. Status code: {}'.format(response.status_code))
+            if response.status_code == 401:
+                raise AzureAuthenticationError()
+            else:
+                raise AzureRequestFailedError('Failed to request Azure OpenAI. Status code: {}'.format(response.status_code))
 
     def get_credentials(self, model_id: Optional[str] = None) -> dict:
         """
@@ -38,7 +42,7 @@ class AzureProvider(BaseProvider):
         """
         config = self.get_provider_api_key(model_id=model_id)
         config['openai_api_type'] = 'azure'
-        config['deployment_name'] = model_id.replace('.', '')
+        config['deployment_name'] = model_id.replace('.', '') if model_id else None
         return config
 
     def get_provider_name(self):
@@ -54,7 +58,7 @@ class AzureProvider(BaseProvider):
             config = {
                 'openai_api_type': 'azure',
                 'openai_api_version': '2023-03-15-preview',
-                'openai_api_base': 'https://<your-domain-prefix>.openai.azure.com/',
+                'openai_api_base': '',
                 'openai_api_key': ''
             }
 
@@ -63,7 +67,7 @@ class AzureProvider(BaseProvider):
                 config = {
                     'openai_api_type': 'azure',
                     'openai_api_version': '2023-03-15-preview',
-                    'openai_api_base': 'https://<your-domain-prefix>.openai.azure.com/',
+                    'openai_api_base': '',
                     'openai_api_key': ''
                 }
 
@@ -80,8 +84,23 @@ class AzureProvider(BaseProvider):
         """
         Validates the given config.
         """
-        # TODO: implement
-        pass
+        try:
+            if not isinstance(config, dict):
+                raise ValueError('Config must be a object.')
+
+            if 'openai_api_version' not in config:
+                config['openai_api_version'] = '2023-03-15-preview'
+
+            self.get_models(credentials=config)
+        except AzureAuthenticationError:
+            raise ValidateFailedError('Azure OpenAI Credentials validation failed, please check your API Key.')
+        except requests.ConnectionError:
+            raise ValidateFailedError('Azure OpenAI Credentials validation failed, please check your API Base Endpoint.')
+        except AzureRequestFailedError as ex:
+            raise ValidateFailedError('Azure OpenAI Credentials validation failed, error: {}.'.format(str(ex)))
+        except Exception as ex:
+            logging.exception('Azure OpenAI Credentials validation failed')
+            raise ex
 
     def get_encrypted_token(self, config: Union[dict | str]):
         """
@@ -101,3 +120,11 @@ class AzureProvider(BaseProvider):
         config = json.loads(token)
         config['openai_api_key'] = self.decrypt_token(config['openai_api_key'])
         return config
+
+
+class AzureAuthenticationError(Exception):
+    pass
+
+
+class AzureRequestFailedError(Exception):
+    pass

api/libs/helper.py

@@ -21,7 +21,7 @@ class TimestampField(fields.Raw):
 
 def email(email):
     # Define a regex pattern for email addresses
-    pattern = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$"
+    pattern = r"^[\w\.-]+@([\w-]+\.)+[\w-]{2,}$"
     # Check if the email matches the pattern
     if re.match(pattern, email) is not None:
         return email

api/services/dataset_service.py

@@ -18,6 +18,7 @@ from services.errors.account import NoPermissionError
 from services.errors.dataset import DatasetNameDuplicateError
 from services.errors.document import DocumentIndexingError
 from services.errors.file import FileNotExistsError
+from tasks.deal_dataset_vector_index_task import deal_dataset_vector_index_task
 from tasks.document_indexing_task import document_indexing_task
 
 
@@ -97,7 +98,12 @@ class DatasetService:
     def update_dataset(dataset_id, data, user):
         dataset = DatasetService.get_dataset(dataset_id)
         DatasetService.check_dataset_permission(dataset, user)
-
+        if dataset.indexing_technique != data['indexing_technique']:
+            # if update indexing_technique
+            if data['indexing_technique'] == 'economy':
+                deal_dataset_vector_index_task.delay(dataset_id, 'remove')
+            elif data['indexing_technique'] == 'high_quality':
+                deal_dataset_vector_index_task.delay(dataset_id, 'add')
         filtered_data = {k: v for k, v in data.items() if v is not None or k == 'description'}
 
         filtered_data['updated_by'] = user.id

api/tasks/deal_dataset_vector_index_task.py

@@ -0,0 +1,75 @@
+import logging
+import time
+
+import click
+from celery import shared_task
+from llama_index.data_structs.node_v2 import DocumentRelationship, Node
+from core.index.vector_index import VectorIndex
+from extensions.ext_database import db
+from models.dataset import DocumentSegment, Document, Dataset
+
+
+@shared_task
+def deal_dataset_vector_index_task(dataset_id: str, action: str):
+    """
+    Async deal dataset from index
+    :param dataset_id: dataset_id
+    :param action: action
+    Usage: deal_dataset_vector_index_task.delay(dataset_id, action)
+    """
+    logging.info(click.style('Start deal dataset vector index: {}'.format(dataset_id), fg='green'))
+    start_at = time.perf_counter()
+
+    try:
+        dataset = Dataset.query.filter_by(
+            id=dataset_id
+        ).first()
+        if not dataset:
+            raise Exception('Dataset not found')
+        documents = Document.query.filter_by(dataset_id=dataset_id).all()
+        if documents:
+            vector_index = VectorIndex(dataset=dataset)
+            for document in documents:
+                # delete from vector index
+                if action == "remove":
+                    vector_index.del_doc(document.id)
+                elif action == "add":
+                    segments = db.session.query(DocumentSegment).filter(
+                        DocumentSegment.document_id == document.id,
+                        DocumentSegment.enabled == True
+                    ) .order_by(DocumentSegment.position.asc()).all()
+
+                    nodes = []
+                    previous_node = None
+                    for segment in segments:
+                        relationships = {
+                            DocumentRelationship.SOURCE: document.id
+                        }
+
+                        if previous_node:
+                            relationships[DocumentRelationship.PREVIOUS] = previous_node.doc_id
+
+                            previous_node.relationships[DocumentRelationship.NEXT] = segment.index_node_id
+
+                        node = Node(
+                            doc_id=segment.index_node_id,
+                            doc_hash=segment.index_node_hash,
+                            text=segment.content,
+                            extra_info=None,
+                            node_info=None,
+                            relationships=relationships
+                        )
+
+                        previous_node = node
+                        nodes.append(node)
+                    # save vector index
+                    vector_index.add_nodes(
+                        nodes=nodes,
+                        duplicate_check=True
+                    )
+
+        end_at = time.perf_counter()
+        logging.info(
+            click.style('Deal dataset vector index: {} latency: {}'.format(dataset_id, end_at - start_at), fg='green'))
+    except Exception:
+        logging.exception("Deal dataset vector index failed")

web/app/(commonLayout)/_layout-client.tsx

@@ -1,5 +1,5 @@
 'use client'
-import type { FC } from 'react'
+import { FC, useRef } from 'react'
 import React, { useEffect, useState } from 'react'
 import { usePathname, useRouter, useSelectedLayoutSegments } from 'next/navigation'
 import useSWR, { SWRConfig } from 'swr'
@@ -8,7 +8,7 @@ import { fetchAppList } from '@/service/apps'
 import { fetchDatasets } from '@/service/datasets'
 import { fetchLanggeniusVersion, fetchUserProfile, logout } from '@/service/common'
 import Loading from '@/app/components/base/loading'
-import AppContext from '@/context/app-context'
+import { AppContextProvider } from '@/context/app-context'
 import DatasetsContext from '@/context/datasets-context'
 import type { LangGeniusVersionResponse, UserProfileResponse } from '@/models/common'
 
@@ -23,6 +23,7 @@ const CommonLayout: FC<ICommonLayoutProps> = ({ children }) => {
   const pattern = pathname.replace(/.*\/app\//, '')
   const [idOrMethod] = pattern.split('/')
   const isNotDetailPage = idOrMethod === 'list'
+  const pageContainerRef = useRef<HTMLDivElement>(null)
 
   const appId = isNotDetailPage ? '' : idOrMethod
 
@@ -71,14 +72,14 @@ const CommonLayout: FC<ICommonLayoutProps> = ({ children }) => {
     <SWRConfig value={{
       shouldRetryOnError: false
     }}>
-      <AppContext.Provider value={{ apps: appList.data, mutateApps, userProfile, mutateUserProfile }}>
-        <DatasetsContext.Provider value={{ datasets: datasetList?.data || [], mutateDatasets,  currentDataset }}>
-          <div className='relative flex flex-col h-full overflow-scroll bg-gray-100'>
+      <AppContextProvider value={{ apps: appList.data, mutateApps, userProfile, mutateUserProfile, pageContainerRef }}>
+        <DatasetsContext.Provider value={{ datasets: datasetList?.data || [], mutateDatasets, currentDataset }}>
+          <div ref={pageContainerRef} className='relative flex flex-col h-full overflow-auto bg-gray-100'>
             <Header isBordered={['/apps', '/datasets'].includes(pathname)} curApp={curApp as any} appItems={appList.data} userProfile={userProfile} onLogout={onLogout} langeniusVersionInfo={langeniusVersionInfo} />
             {children}
           </div>
         </DatasetsContext.Provider>
-      </AppContext.Provider>
+      </AppContextProvider>
     </SWRConfig>
   )
 }

web/app/(commonLayout)/apps/Apps.tsx

@@ -1,23 +1,50 @@
 'use client'
 
-import { useEffect } from 'react'
+import { useEffect, useRef } from 'react'
+import useSWRInfinite from 'swr/infinite'
+import { debounce } from 'lodash-es'
 import AppCard from './AppCard'
 import NewAppCard from './NewAppCard'
-import { useAppContext } from '@/context/app-context'
+import { AppListResponse } from '@/models/app'
+import { fetchAppList } from '@/service/apps'
+import { useSelector } from '@/context/app-context'
+
+const getKey = (pageIndex: number, previousPageData: AppListResponse) => {
+  if (!pageIndex || previousPageData.has_more)
+    return { url: 'apps', params: { page: pageIndex + 1, limit: 30 } }
+  return null
+}
 
 const Apps = () => {
-  const { apps, mutateApps } = useAppContext()
+  const { data, isLoading, setSize, mutate } = useSWRInfinite(getKey, fetchAppList, { revalidateFirstPage: false })
+  const loadingStateRef = useRef(false)
+  const pageContainerRef = useSelector(state => state.pageContainerRef)
+  const anchorRef = useRef<HTMLAnchorElement>(null)
 
   useEffect(() => {
-    mutateApps()
+    loadingStateRef.current = isLoading
+  }, [isLoading])
+
+  useEffect(() => {
+    const onScroll = debounce(() => {
+      if (!loadingStateRef.current) {
+        const { scrollTop, clientHeight } = pageContainerRef.current!
+        const anchorOffset = anchorRef.current!.offsetTop
+        if (anchorOffset - scrollTop - clientHeight < 100) {
+          setSize(size => size + 1)
+        }
+      }
+    }, 50)
+
+    pageContainerRef.current?.addEventListener('scroll', onScroll)
+    return () => pageContainerRef.current?.removeEventListener('scroll', onScroll)
   }, [])
 
   return (
     <nav className='grid content-start grid-cols-1 gap-4 px-12 pt-8 sm:grid-cols-2 lg:grid-cols-4 grow shrink-0'>
-      {apps.map(app => (<AppCard key={app.id} app={app} />))}
-      <NewAppCard />
+      {data?.map(({ data: apps }) => apps.map(app => (<AppCard key={app.id} app={app} />)))}
+      <NewAppCard ref={anchorRef} onSuccess={mutate} />
     </nav>
-
   )
 }
 

web/app/(commonLayout)/apps/NewAppCard.tsx

@@ -1,16 +1,20 @@
 'use client'
 
-import { useState } from 'react'
+import { forwardRef, useState } from 'react'
 import classNames from 'classnames'
 import { useTranslation } from 'react-i18next'
 import style from '../list.module.css'
 import NewAppDialog from './NewAppDialog'
 
-const CreateAppCard = () => {
+export type CreateAppCardProps = {
+  onSuccess?: () => void
+}
+
+const CreateAppCard = forwardRef<HTMLAnchorElement, CreateAppCardProps>(({ onSuccess }, ref) => {
   const { t } = useTranslation()
   const [showNewAppDialog, setShowNewAppDialog] = useState(false)
   return (
-    <a className={classNames(style.listItem, style.newItemCard)} onClick={() => setShowNewAppDialog(true)}>
+    <a ref={ref} className={classNames(style.listItem, style.newItemCard)} onClick={() => setShowNewAppDialog(true)}>
       <div className={style.listItemTitle}>
         <span className={style.newItemIcon}>
           <span className={classNames(style.newItemIconImage, style.newItemIconAdd)} />
@@ -20,9 +24,9 @@ const CreateAppCard = () => {
         </div>
       </div>
       {/* <div className='text-xs text-gray-500'>{t('app.createFromConfigFile')}</div> */}
-      <NewAppDialog show={showNewAppDialog} onClose={() => setShowNewAppDialog(false)} />
+      <NewAppDialog show={showNewAppDialog} onSuccess={onSuccess} onClose={() => setShowNewAppDialog(false)} />
     </a>
   )
-}
+})
 
 export default CreateAppCard

web/app/(commonLayout)/apps/NewAppDialog.tsx

@@ -21,10 +21,11 @@ import EmojiPicker from '@/app/components/base/emoji-picker'
 
 type NewAppDialogProps = {
   show: boolean
+  onSuccess?: () => void
   onClose?: () => void
 }
 
-const NewAppDialog = ({ show, onClose }: NewAppDialogProps) => {
+const NewAppDialog = ({ show, onSuccess, onClose }: NewAppDialogProps) => {
   const router = useRouter()
   const { notify } = useContext(ToastContext)
   const { t } = useTranslation()
@@ -79,6 +80,8 @@ const NewAppDialog = ({ show, onClose }: NewAppDialogProps) => {
         mode: isWithTemplate ? templates.data[selectedTemplateIndex].mode : newAppMode!,
         config: isWithTemplate ? templates.data[selectedTemplateIndex].model_config : undefined,
       })
+      if (onSuccess)
+        onSuccess()
       if (onClose)
         onClose()
       notify({ type: 'success', message: t('app.newApp.appCreated') })

web/app/(commonLayout)/datasets/Datasets.tsx

@@ -1,24 +1,49 @@
 'use client'
 
-import { useEffect } from 'react'
-import useSWR from 'swr'
-import { DataSet } from '@/models/datasets';
+import { useEffect, useRef } from 'react'
+import useSWRInfinite from 'swr/infinite'
+import { debounce } from 'lodash-es';
+import { DataSetListResponse } from '@/models/datasets';
 import NewDatasetCard from './NewDatasetCard'
 import DatasetCard from './DatasetCard';
 import { fetchDatasets } from '@/service/datasets';
+import { useSelector } from '@/context/app-context';
+
+const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
+  if (!pageIndex || previousPageData.has_more)
+    return { url: 'datasets', params: { page: pageIndex + 1, limit: 30 } }
+  return null
+}
 
 const Datasets = () => {
-  // const { datasets, mutateDatasets } = useAppContext()
-  const { data: datasetList, mutate: mutateDatasets } = useSWR({ url: '/datasets', params: { page: 1 } }, fetchDatasets)
+  const { data, isLoading, setSize, mutate } = useSWRInfinite(getKey, fetchDatasets, { revalidateFirstPage: false })
+  const loadingStateRef = useRef(false)
+  const pageContainerRef = useSelector(state => state.pageContainerRef)
+  const anchorRef = useRef<HTMLAnchorElement>(null)
 
   useEffect(() => {
-    mutateDatasets()
+    loadingStateRef.current = isLoading
+  }, [isLoading])
+
+  useEffect(() => {
+    const onScroll = debounce(() => {
+      if (!loadingStateRef.current) {
+        const { scrollTop, clientHeight } = pageContainerRef.current!
+        const anchorOffset = anchorRef.current!.offsetTop
+        if (anchorOffset - scrollTop - clientHeight < 100) {
+          setSize(size => size + 1)
+        }
+      }
+    }, 50)
+
+    pageContainerRef.current?.addEventListener('scroll', onScroll)
+    return () => pageContainerRef.current?.removeEventListener('scroll', onScroll)
   }, [])
 
   return (
     <nav className='grid content-start grid-cols-1 gap-4 px-12 pt-8 sm:grid-cols-2 lg:grid-cols-4 grow shrink-0'>
-      {datasetList?.data.map(dataset => (<DatasetCard key={dataset.id} dataset={dataset} />))}
-      <NewDatasetCard />
+      {data?.map(({ data: datasets }) => datasets.map(dataset => (<DatasetCard key={dataset.id} dataset={dataset} />)))}
+      <NewDatasetCard ref={anchorRef} />
     </nav>
   )
 }

web/app/(commonLayout)/datasets/NewDatasetCard.tsx

@@ -1,16 +1,16 @@
 'use client'
 
-import { useState } from 'react'
+import { forwardRef, useState } from 'react'
 import classNames from 'classnames'
 import { useTranslation } from 'react-i18next'
 import style from '../list.module.css'
 
-const CreateAppCard = () => {
+const CreateAppCard = forwardRef<HTMLAnchorElement>((_, ref) => {
   const { t } = useTranslation()
   const [showNewAppDialog, setShowNewAppDialog] = useState(false)
 
   return (
-    <a className={classNames(style.listItem, style.newItemCard)} href='/datasets/create'>
+    <a ref={ref} className={classNames(style.listItem, style.newItemCard)} href='/datasets/create'>
       <div className={style.listItemTitle}>
         <span className={style.newItemIcon}>
           <span className={classNames(style.newItemIconImage, style.newItemIconAdd)} />
@@ -23,6 +23,6 @@ const CreateAppCard = () => {
       {/* <div className='text-xs text-gray-500'>{t('app.createFromConfigFile')}</div> */}
     </a>
   )
-}
+})
 
 export default CreateAppCard

web/app/components/app/chat/index.tsx

@@ -291,74 +291,76 @@ const Answer: FC<IAnswerProps> = ({ item, feedbackDisabled = false, isHideFeedba
             </div>
           }
         </div>
-        <div className={`${s.answerWrap} ${showEdit ? 'w-full' : ''}`}>
-          <div className={`${s.answer} relative text-sm text-gray-900`}>
-            <div className={'ml-2 py-3 px-4 bg-gray-100 rounded-tr-2xl rounded-b-2xl'}>
-              {item.isOpeningStatement && (
-                <div className='flex items-center mb-1 gap-1'>
-                  <OpeningStatementIcon />
-                  <div className='text-xs text-gray-500'>{t('appDebug.openingStatement.title')}</div>
-                </div>
-              )}
-              {(isResponsing && !content) ? (
-                <div className='flex items-center justify-center w-6 h-5'>
-                  <LoadingAnim type='text' />
-                </div>
-              ) : (
-                <Markdown content={content} />
-              )}
-              {!showEdit
-                ? (annotation?.content
-                  && <>
-                    <Divider name={annotation?.account?.name || userProfile?.name} />
-                    {annotation.content}
-                  </>)
-                : <>
-                  <Divider name={annotation?.account?.name || userProfile?.name} />
-                  <AutoHeightTextarea
-                    placeholder={t('appLog.detail.operation.annotationPlaceholder') as string}
-                    value={inputValue}
-                    onChange={e => setInputValue(e.target.value)}
-                    minHeight={58}
-                    className={`${cn(s.textArea)} !py-2 resize-none block w-full !px-3 bg-gray-50 border border-gray-200 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm text-gray-700 tracking-[0.2px]`}
-                  />
-                  <div className="mt-2 flex flex-row">
-                    <Button
-                      type='primary'
-                      className='mr-2'
-                      loading={loading}
-                      onClick={async () => {
-                        if (!inputValue)
-                          return
-                        setLoading(true)
-                        const res = await onSubmitAnnotation?.(id, inputValue)
-                        if (res)
-                          setAnnotation({ ...annotation, content: inputValue } as any)
-                        setLoading(false)
-                        setShowEdit(false)
-                      }}>{t('common.operation.confirm')}</Button>
-                    <Button
-                      onClick={() => {
-                        setInputValue(annotation?.content ?? '')
-                        setShowEdit(false)
-                      }}>{t('common.operation.cancel')}</Button>
+        <div className={s.answerWrapWrap}>
+          <div className={`${s.answerWrap} ${showEdit ? 'w-full' : ''}`}>
+            <div className={`${s.answer} relative text-sm text-gray-900`}>
+              <div className={'ml-2 py-3 px-4 bg-gray-100 rounded-tr-2xl rounded-b-2xl'}>
+                {item.isOpeningStatement && (
+                  <div className='flex items-center mb-1 gap-1'>
+                    <OpeningStatementIcon />
+                    <div className='text-xs text-gray-500'>{t('appDebug.openingStatement.title')}</div>
                   </div>
-                </>
-              }
-            </div>
-            <div className='absolute top-[-14px] right-[-14px] flex flex-row justify-end gap-1'>
-              <CopyBtn
-                value={content}
-                className={cn(s.copyBtn, 'mr-1')}
-              />
-              {!feedbackDisabled && !item.feedbackDisabled && renderItemOperation(displayScene !== 'console')}
-              {/* Admin feedback is displayed only in the background. */}
-              {!feedbackDisabled && renderFeedbackRating(localAdminFeedback?.rating, false, false)}
-              {/* User feedback must be displayed */}
-              {!feedbackDisabled && renderFeedbackRating(feedback?.rating, !isHideFeedbackEdit, displayScene !== 'console')}
+                )}
+                {(isResponsing && !content) ? (
+                  <div className='flex items-center justify-center w-6 h-5'>
+                    <LoadingAnim type='text' />
+                  </div>
+                ) : (
+                  <Markdown content={content} />
+                )}
+                {!showEdit
+                  ? (annotation?.content
+                    && <>
+                      <Divider name={annotation?.account?.name || userProfile?.name} />
+                      {annotation.content}
+                    </>)
+                  : <>
+                    <Divider name={annotation?.account?.name || userProfile?.name} />
+                    <AutoHeightTextarea
+                      placeholder={t('appLog.detail.operation.annotationPlaceholder') as string}
+                      value={inputValue}
+                      onChange={e => setInputValue(e.target.value)}
+                      minHeight={58}
+                      className={`${cn(s.textArea)} !py-2 resize-none block w-full !px-3 bg-gray-50 border border-gray-200 rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm text-gray-700 tracking-[0.2px]`}
+                    />
+                    <div className="mt-2 flex flex-row">
+                      <Button
+                        type='primary'
+                        className='mr-2'
+                        loading={loading}
+                        onClick={async () => {
+                          if (!inputValue)
+                            return
+                          setLoading(true)
+                          const res = await onSubmitAnnotation?.(id, inputValue)
+                          if (res)
+                            setAnnotation({ ...annotation, content: inputValue } as any)
+                          setLoading(false)
+                          setShowEdit(false)
+                        }}>{t('common.operation.confirm')}</Button>
+                      <Button
+                        onClick={() => {
+                          setInputValue(annotation?.content ?? '')
+                          setShowEdit(false)
+                        }}>{t('common.operation.cancel')}</Button>
+                    </div>
+                  </>
+                }
+              </div>
+              <div className='absolute top-[-14px] right-[-14px] flex flex-row justify-end gap-1'>
+                <CopyBtn
+                  value={content}
+                  className={cn(s.copyBtn, 'mr-1')}
+                />
+                {!feedbackDisabled && !item.feedbackDisabled && renderItemOperation(displayScene !== 'console')}
+                {/* Admin feedback is displayed only in the background. */}
+                {!feedbackDisabled && renderFeedbackRating(localAdminFeedback?.rating, false, false)}
+                {/* User feedback must be displayed */}
+                {!feedbackDisabled && renderFeedbackRating(feedback?.rating, !isHideFeedbackEdit, displayScene !== 'console')}
+              </div>
             </div>
+            {more && <MoreInfo more={more} isQuestion={false} />}
           </div>
-          {more && <MoreInfo more={more} isQuestion={false} />}
         </div>
       </div>
     </div>
@@ -372,7 +374,7 @@ const Question: FC<IQuestionProps> = ({ id, content, more, useCurrentUserAvatar
   const userName = userProfile?.name
   return (
     <div className='flex items-start justify-end' key={id}>
-      <div>
+      <div className={s.questionWrapWrap}>
         <div className={`${s.question} relative text-sm text-gray-900`}>
           <div
             className={'mr-2 py-3 px-4 bg-blue-500 rounded-tl-2xl rounded-b-2xl'}

web/app/components/app/chat/style.module.css

@@ -42,6 +42,23 @@
   display: none;
 }
 
+.answerWrapWrap,
+.questionWrapWrap {
+  width: 0;
+  flex-grow: 1;
+}
+
+.questionWrapWrap {
+  display: flex;
+  justify-content: flex-end;
+}
+
+.answerWrap,
+.question {
+  display: inline-block;
+  max-width: 100%;
+}
+
 .answerWrap:hover .copyBtn {
   display: block;
 }

web/app/components/base/dialog/index.tsx

@@ -33,7 +33,7 @@ const CustomDialog = ({
   const close = useCallback(() => onClose?.(), [onClose])
   return (
     <Transition appear show={show} as={Fragment}>
-      <Dialog as="div" className="relative z-10" onClose={close}>
+      <Dialog as="div" className="relative z-40" onClose={close}>
         <Transition.Child
           as={Fragment}
           enter="ease-out duration-300"

web/app/components/base/emoji-picker/index.tsx

@@ -1,18 +1,18 @@
 'use client'
+import React from 'react'
+import { useState, FC, ChangeEvent } from 'react'
 import data from '@emoji-mart/data'
 import { init, SearchIndex } from 'emoji-mart'
-// import AppIcon from '@/app/components/base/app-icon'
 import cn from 'classnames'
 import Divider from '@/app/components/base/divider'
-
 import Button from '@/app/components/base/button'
 import s from './style.module.css'
-import { useState, FC, ChangeEvent } from 'react'
 import {
   MagnifyingGlassIcon
 } from '@heroicons/react/24/outline'
-import React from 'react'
+
 import Modal from '@/app/components/base/modal'
+import { useTranslation } from 'react-i18next'
 
 declare global {
   namespace JSX {
@@ -69,6 +69,7 @@ const EmojiPicker: FC<IEmojiPickerProps> = ({
   onClose
 
 }) => {
+  const { t } = useTranslation()
   const { categories } = data as any
   const [selectedEmoji, setSelectedEmoji] = useState('')
   const [selectedBackground, setSelectedBackground] = useState(backgroundColors[0])
@@ -80,6 +81,7 @@ const EmojiPicker: FC<IEmojiPickerProps> = ({
     onClose={() => { }}
     isShow
     closable={false}
+    wrapperClassName='!z-40'
     className={cn(s.container, '!w-[362px] !p-0')}
   >
     <div className='flex flex-col items-center w-full p-3'>
@@ -186,7 +188,7 @@ const EmojiPicker: FC<IEmojiPickerProps> = ({
       <Button type="default" className='w-full' onClick={() => {
         onClose && onClose()
       }}>
-        Cancel
+        {t('app.emoji.cancel')}
       </Button>
       <Button
         disabled={selectedEmoji == ''}
@@ -195,7 +197,7 @@ const EmojiPicker: FC<IEmojiPickerProps> = ({
         onClick={() => {
           onSelect && onSelect(selectedEmoji, selectedBackground)
         }}>
-        OK
+        {t('app.emoji.ok')}
       </Button>
     </div>
   </Modal> : <>

web/app/components/base/modal/index.tsx

@@ -26,7 +26,7 @@ export default function Modal({
 }: IModal) {
   return (
     <Transition appear show={isShow} as={Fragment}>
-      <Dialog as="div" className="relative z-10" onClose={onClose}>
+      <Dialog as="div" className={`relative z-10 ${wrapperClassName}`} onClose={onClose}>
         <Transition.Child
           as={Fragment}
           enter="ease-out duration-300"

web/app/components/datasets/create/file-preview/index.module.css

@@ -43,4 +43,7 @@
     background: #f9fafb center no-repeat url(../assets/Loading.svg);
     background-size: contain;
   }
+  .fileContent {
+    white-space: pre-line;
+  }
   

web/app/components/datasets/create/file-uploader/index.tsx

@@ -190,13 +190,15 @@ const FileUploader = ({ file, onFileUpdate }: IFileUploaderProps) => {
         onChange={fileChangeHandle}
       />
       <div className={s.title}>{t('datasetCreation.stepOne.uploader.title')}</div>
-      {!currentFile && !file && (
-        <div ref={dropRef} className={cn(s.uploader, dragging && s.dragging)}>
-          <span>{t('datasetCreation.stepOne.uploader.button')}</span>
-          <label className={s.browse} onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.browse')}</label>
-          {dragging && <div ref={dragRef} className={s.draggingCover}/>}
-        </div>
-      )}
+      <div ref={dropRef}>
+        {!currentFile && !file && (
+          <div className={cn(s.uploader, dragging && s.dragging)}>
+            <span>{t('datasetCreation.stepOne.uploader.button')}</span>
+            <label className={s.browse} onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.browse')}</label>
+            {dragging && <div ref={dragRef} className={s.draggingCover}/>}
+          </div>
+        )}
+      </div>
       {currentFile && (
         <div className={cn(s.file, uploading && s.uploading)}>
           {uploading && (

web/app/components/datasets/create/step-two/preview-item/index.tsx

@@ -41,7 +41,7 @@ const PreviewItem: FC<IPreviewItemProps> = ({
         </div>
       </div>
       <div className='mt-2 max-h-[120px] line-clamp-6 overflow-hidden text-sm text-gray-800'>
-        {content}
+        <div style={{ whiteSpace: 'pre-line'}}>{content}</div>
       </div>
     </div>
   )

web/app/components/datasets/documents/detail/completed/style.module.css

@@ -44,7 +44,8 @@
   @apply h-8 py-0 bg-gray-50 hover:bg-gray-100 rounded-lg shadow-none !important;
 }
 .segModalContent {
-  @apply h-96 text-gray-800 text-base overflow-y-scroll;
+  @apply h-96 text-gray-800 text-base break-all overflow-y-scroll;
+  white-space: pre-line;
 }
 .footer {
   @apply flex items-center justify-between box-border border-t-gray-200 border-t-[0.5px] pt-3 mt-4;

web/app/components/share/chat/config-scence/index.tsx

@@ -5,7 +5,7 @@ import Welcome from '../welcome'
 
 const ConfigSence: FC<IWelcomeProps> = (props) => {
   return (
-    <div className='mb-5 antialiased font-sans overflow-hidden shrink-0'>
+    <div className='mb-5 antialiased font-sans shrink-0'>
       <Welcome {...props} />
     </div>
   )

web/app/install/installForm.tsx

@@ -7,7 +7,7 @@ import { useRouter } from 'next/navigation'
 import Toast from '../components/base/toast'
 import { setup } from '@/service/common'
 
-const validEmailReg = /^([a-zA-Z0-9_-])+@([a-zA-Z0-9_-])+(\.[a-zA-Z0-9_-])+/
+const validEmailReg = /^[\w\.-]+@([\w-]+\.)+[\w-]{2,}$/
 const validPassword = /^(?=.*[a-zA-Z])(?=.*\d).{8,}$/
 
 const InstallForm = () => {

web/app/signin/normalForm.tsx

@@ -13,7 +13,7 @@ import Button from '@/app/components/base/button'
 import { login, oauth } from '@/service/common'
 import { apiPrefix } from '@/config'
 
-const validEmailReg = /^([a-zA-Z0-9_-])+@([a-zA-Z0-9_-])+(\.[a-zA-Z0-9_-])+/
+const validEmailReg = /^[\w\.-]+@([\w-]+\.)+[\w-]{2,}$/
 
 type IState = {
   formValid: boolean

web/app/styles/markdown.scss

@@ -594,6 +594,7 @@
 
 .markdown-body table th {
   font-weight: var(--base-text-weight-semibold, 600);
+  white-space: nowrap;
 }
 
 .markdown-body table th,

web/config/index.ts

@@ -75,9 +75,9 @@ export const LOCALE_COOKIE_NAME = 'locale'
 
 export const DEFAULT_VALUE_MAX_LEN = 48
 
-export const zhRegex = /^[\u4e00-\u9fa5]$/gm
-export const emojiRegex = /^[\uD800-\uDBFF][\uDC00-\uDFFF]$/gm
-export const emailRegex = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/
+export const zhRegex = /^[\u4e00-\u9fa5]$/m
+export const emojiRegex = /^[\uD800-\uDBFF][\uDC00-\uDFFF]$/m
+export const emailRegex = /^[\w\.-]+@([\w-]+\.)+[\w-]{2,}$/m
 const MAX_ZN_VAR_NAME_LENGHT = 8
 const MAX_EN_VAR_VALUE_LENGHT = 16
 export const getMaxVarNameLength = (value: string) => {

web/context/app-context.ts

@@ -1,27 +0,0 @@
-'use client'
-
-import { createContext, useContext } from 'use-context-selector'
-import type { App } from '@/types/app'
-import type { UserProfileResponse } from '@/models/common'
-
-export type AppContextValue = {
-  apps: App[]
-  mutateApps: () => void
-  userProfile: UserProfileResponse
-  mutateUserProfile: () => void
-}
-
-const AppContext = createContext<AppContextValue>({
-  apps: [],
-  mutateApps: () => { },
-  userProfile: {
-    id: '',
-    name: '',
-    email: '',
-  },
-  mutateUserProfile: () => { },
-})
-
-export const useAppContext = () => useContext(AppContext)
-
-export default AppContext

web/context/app-context.tsx

@@ -0,0 +1,45 @@
+'use client'
+
+import { createContext, useContext, useContextSelector } from 'use-context-selector'
+import type { App } from '@/types/app'
+import type { UserProfileResponse } from '@/models/common'
+import { createRef, FC, PropsWithChildren } from 'react'
+
+export const useSelector = <T extends any>(selector: (value: AppContextValue) => T): T =>
+  useContextSelector(AppContext, selector);
+
+export type AppContextValue = {
+  apps: App[]
+  mutateApps: () => void
+  userProfile: UserProfileResponse
+  mutateUserProfile: () => void
+  pageContainerRef: React.RefObject<HTMLDivElement>,
+  useSelector: typeof useSelector,
+}
+
+const AppContext = createContext<AppContextValue>({
+  apps: [],
+  mutateApps: () => { },
+  userProfile: {
+    id: '',
+    name: '',
+    email: '',
+  },
+  mutateUserProfile: () => { },
+  pageContainerRef: createRef(),
+  useSelector,
+})
+
+export type AppContextProviderProps = PropsWithChildren<{
+  value: Omit<AppContextValue, 'useSelector'>
+}>
+
+export const AppContextProvider: FC<AppContextProviderProps> = ({ value, children }) => (
+  <AppContext.Provider value={{ ...value, useSelector }}>
+    {children}
+  </AppContext.Provider>
+)
+
+export const useAppContext = () => useContext(AppContext)
+
+export default AppContext

web/i18n/lang/app.en.ts

@@ -35,6 +35,10 @@ const translation = {
     appCreated: 'App created',
     appCreateFailed: 'Failed to create app',
   },
+  emoji: {
+    ok: 'OK',
+    cancel: 'Cancel',
+  }
 }
 
 export default translation

web/i18n/lang/app.zh.ts

@@ -34,6 +34,10 @@ const translation = {
     appCreated: '应用已创建',
     appCreateFailed: '应用创建失败',
   },
+  emoji: {
+    ok: '确认',
+    cancel: '取消',
+  }
 }
 
 export default translation

web/i18n/lang/common.en.ts

@@ -149,7 +149,7 @@ const translation = {
     invalidApiKey: 'Invalid API key',
     azure: {
       apiBase: 'API Base',
-      apiBasePlaceholder: 'The API Base URL of your Azure OpenAI Resource.',
+      apiBasePlaceholder: 'The API Base URL of your Azure OpenAI Endpoint.',
       apiKey: 'API Key',
       apiKeyPlaceholder: 'Enter your API key here',
       helpTip: 'Learn Azure OpenAI Service',

web/models/app.ts

@@ -61,6 +61,10 @@ export type SiteConfig = {
 
 export type AppListResponse = {
   data: App[]
+  has_more: boolean
+  limit: number
+  page: number
+  total: number
 }
 
 export type AppDetailResponse = App

web/models/datasets.ts

@@ -29,6 +29,10 @@ export type File = {
 
 export type DataSetListResponse = {
   data: DataSet[]
+  has_more: boolean
+  limit: number
+  page: number
+  total: number
 }
 
 export type IndexingEstimateResponse = {

web/service/apps.ts

@@ -4,8 +4,8 @@ import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUse
 import type { CommonResponse } from '@/models/common'
 import type { AppMode, ModelConfig } from '@/types/app'
 
-export const fetchAppList: Fetcher<AppListResponse, { params?: Record<string, any> }> = ({ params }) => {
-  return get('apps', params) as Promise<AppListResponse>
+export const fetchAppList: Fetcher<AppListResponse, { url: string; params?: Record<string, any> }> = ({ url, params }) => {
+  return get(url, { params }) as Promise<AppListResponse>
 }
 
 export const fetchAppDetail: Fetcher<AppDetailResponse, { url: string; id: string }> = ({ url, id }) => {

web/service/base.ts

@@ -78,7 +78,7 @@ const handleStream = (response: any, onData: IOnData, onCompleted?: IOnCompleted
           if (message.startsWith('data: ')) { // check if it starts with data:
             // console.log(message);
             bufferObj = JSON.parse(message.substring(6)) // remove data: and parse as json
-            if (bufferObj.status === 400) {
+            if (bufferObj.status === 400 || !bufferObj.event) {
               onData('', false, {
                 conversationId: undefined,
                 messageId: '',