fix merge

Merge branch 'main' into feat/qa-dataset-multi-import
2025-12-07 03:45:27 +08:00 · 2023-08-15 14:31:22 +08:00 · 2023-08-15 13:34:25 +08:00 · 2023-08-15 13:29:39 +08:00 · 2023-08-12 18:39:09 +08:00 · 2023-08-12 13:51:56 +08:00
35 changed files with 1071 additions and 148 deletions
--- a/api/controllers/console/datasets/datasets.py
+++ b/api/controllers/console/datasets/datasets.py
@@ -233,6 +233,7 @@ class DatasetIndexingEstimateApi(Resource):
        parser.add_argument('info_list', type=dict, required=True, nullable=True, location='json')
        parser.add_argument('process_rule', type=dict, required=True, nullable=True, location='json')
        parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
+        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False, location='json')
        args = parser.parse_args()
        # validate args
        DocumentService.estimate_args_validate(args)
@@ -250,7 +251,7 @@ class DatasetIndexingEstimateApi(Resource):

            try:
                response = indexing_runner.file_indexing_estimate(current_user.current_tenant_id, file_details,
-                                                                  args['process_rule'], args['doc_form'])
+                                                                  args['process_rule'], args['doc_form'], args['doc_language'])
            except LLMBadRequestError:
                raise ProviderNotInitializeError(
                    f"No Embedding Model available. Please configure a valid provider "
@@ -262,7 +263,8 @@ class DatasetIndexingEstimateApi(Resource):
            try:
                response = indexing_runner.notion_indexing_estimate(current_user.current_tenant_id,
                                                                    args['info_list']['notion_info_list'],
-                                                                    args['process_rule'], args['doc_form'])
+                                                                    args['process_rule'], args['doc_form'],
+                                                                    args['doc_language'])
            except LLMBadRequestError:
                raise ProviderNotInitializeError(
                    f"No Embedding Model available. Please configure a valid provider "
--- a/api/controllers/console/datasets/datasets_document.py
+++ b/api/controllers/console/datasets/datasets_document.py
@@ -274,6 +274,7 @@ class DatasetDocumentListApi(Resource):
        parser.add_argument('duplicate', type=bool, nullable=False, location='json')
        parser.add_argument('original_document_id', type=str, required=False, location='json')
        parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
+        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False, location='json')
        args = parser.parse_args()

        if not dataset.indexing_technique and not args['indexing_technique']:
@@ -328,6 +329,7 @@ class DatasetInitApi(Resource):
        parser.add_argument('data_source', type=dict, required=True, nullable=True, location='json')
        parser.add_argument('process_rule', type=dict, required=True, nullable=True, location='json')
        parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
+        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False, location='json')
        args = parser.parse_args()

        try:
@@ -575,7 +577,8 @@ class DocumentIndexingStatusApi(DocumentResource):

        document.completed_segments = completed_segments
        document.total_segments = total_segments
-
+        if document.is_paused:
+            document.indexing_status = 'paused'
        return marshal(document, self.document_status_fields)


@@ -832,6 +835,22 @@ class DocumentStatusApi(DocumentResource):

                remove_document_from_index_task.delay(document_id)

+            return {'result': 'success'}, 200
+        elif action == "un_archive":
+            if not document.archived:
+                raise InvalidActionError('Document is not archived.')
+
+            document.archived = False
+            document.archived_at = None
+            document.archived_by = None
+            document.updated_at = datetime.utcnow()
+            db.session.commit()
+
+            # Set cache to prevent indexing the same document multiple times
+            redis_client.setex(indexing_cache_key, 600, 1)
+
+            add_document_to_index_task.delay(document_id)
+
            return {'result': 'success'}, 200
        else:
            raise InvalidActionError()
--- a/api/controllers/console/datasets/datasets_segments.py
+++ b/api/controllers/console/datasets/datasets_segments.py
@@ -1,13 +1,15 @@
 # -*- coding:utf-8 -*-
+import uuid
 from datetime import datetime

+from flask import request
 from flask_login import login_required, current_user
 from flask_restful import Resource, reqparse, fields, marshal
 from werkzeug.exceptions import NotFound, Forbidden

 import services
 from controllers.console import api
-from controllers.console.datasets.error import InvalidActionError
+from controllers.console.datasets.error import InvalidActionError, NoFileUploadedError, TooManyFilesError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
@@ -17,7 +19,9 @@ from models.dataset import DocumentSegment
 from libs.helper import TimestampField
 from services.dataset_service import DatasetService, DocumentService, SegmentService
 from tasks.enable_segment_to_index_task import enable_segment_to_index_task
-from tasks.remove_segment_from_index_task import remove_segment_from_index_task
+from tasks.disable_segment_from_index_task import disable_segment_from_index_task
+from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
+import pandas as pd

 segment_fields = {
    'id': fields.String,
@@ -197,7 +201,7 @@ class DatasetDocumentSegmentApi(Resource):
            # Set cache to prevent indexing the same segment multiple times
            redis_client.setex(indexing_cache_key, 600, 1)

-            remove_segment_from_index_task.delay(segment.id)
+            disable_segment_from_index_task.delay(segment.id)

            return {'result': 'success'}, 200
        else:
@@ -283,6 +287,104 @@ class DatasetDocumentSegmentUpdateApi(Resource):
            'doc_form': document.doc_form
        }, 200

+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, dataset_id, document_id, segment_id):
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound('Document not found.')
+        # check segment
+        segment_id = str(segment_id)
+        segment = DocumentSegment.query.filter(
+            DocumentSegment.id == str(segment_id),
+            DocumentSegment.tenant_id == current_user.current_tenant_id
+        ).first()
+        if not segment:
+            raise NotFound('Segment not found.')
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        SegmentService.delete_segment(segment, document, dataset)
+        return {'result': 'success'}, 200
+
+
+class DatasetDocumentSegmentBatchImportApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, dataset_id, document_id):
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound('Document not found.')
+        # get file from request
+        file = request.files['file']
+        # check file
+        if 'file' not in request.files:
+            raise NoFileUploadedError()
+
+        if len(request.files) > 1:
+            raise TooManyFilesError()
+        # check file type
+        if not file.filename.endswith('.csv'):
+            raise ValueError("Invalid file type. Only CSV files are allowed")
+
+        try:
+            # Skip the first row
+            df = pd.read_csv(file)
+            result = []
+            for index, row in df.iterrows():
+                data = {'content': row[0], 'answer': row[1]}
+                result.append(data)
+            if len(result) == 0:
+                raise ValueError("The CSV file is empty.")
+            # async job
+            job_id = str(uuid.uuid4())
+            indexing_cache_key = 'segment_batch_import_{}'.format(str(job_id))
+            # send batch add segments task
+            redis_client.setnx(indexing_cache_key, 'waiting')
+            batch_create_segment_to_index_task.delay(str(job_id), result, dataset_id, document_id,
+                                                     current_user.current_tenant_id, current_user.id)
+        except Exception as e:
+            return {'error': str(e)}, 500
+        return {
+            'job_id': job_id,
+            'job_status': 'waiting'
+        }, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, job_id):
+        job_id = str(job_id)
+        indexing_cache_key = 'segment_batch_import_{}'.format(job_id)
+        cache_result = redis_client.get(indexing_cache_key)
+        if cache_result is None:
+            raise ValueError("The job is not exist.")
+
+        return {
+            'job_id': job_id,
+            'job_status': cache_result.decode()
+        }, 200
+

 api.add_resource(DatasetDocumentSegmentListApi,
                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments')
@@ -292,3 +394,6 @@ api.add_resource(DatasetDocumentSegmentAddApi,
                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segment')
 api.add_resource(DatasetDocumentSegmentUpdateApi,
                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>')
+api.add_resource(DatasetDocumentSegmentBatchImportApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/batch_import',
+                 '/datasets/batch_import_status/<uuid:job_id>')
--- a/api/core/generator/llm_generator.py
+++ b/api/core/generator/llm_generator.py
@@ -175,8 +175,8 @@ class LLMGenerator:
        return rule_config

    @classmethod
-    def generate_qa_document(cls, tenant_id: str, query):
-        prompt = GENERATOR_QA_PROMPT
+    def generate_qa_document(cls, tenant_id: str, query, document_language: str):
+        prompt = GENERATOR_QA_PROMPT.format(language=document_language)

        model_instance = ModelFactory.get_text_generation_model(
            tenant_id=tenant_id,
--- a/api/core/indexing_runner.py
+++ b/api/core/indexing_runner.py
@@ -66,14 +66,6 @@ class IndexingRunner:
                    dataset_document=dataset_document,
                    processing_rule=processing_rule
                )
-                # new_documents = []
-                # for document in documents:
-                #     response = LLMGenerator.generate_qa_document(dataset.tenant_id, document.page_content)
-                #     document_qa_list = self.format_split_text(response)
-                #     for result in document_qa_list:
-                #         document = Document(page_content=result['question'], metadata={'source': result['answer']})
-                #         new_documents.append(document)
-                # build index
                self._build_index(
                    dataset=dataset,
                    dataset_document=dataset_document,
@@ -224,7 +216,7 @@ class IndexingRunner:
            db.session.commit()

    def file_indexing_estimate(self, tenant_id: str, file_details: List[UploadFile], tmp_processing_rule: dict,
-                               doc_form: str = None) -> dict:
+                               doc_form: str = None, doc_language: str = 'English') -> dict:
        """
        Estimate the indexing for the document.
        """
@@ -269,7 +261,7 @@ class IndexingRunner:
        if doc_form and doc_form == 'qa_model':
            if len(preview_texts) > 0:
                # qa model document
-                response = LLMGenerator.generate_qa_document(current_user.current_tenant_id, preview_texts[0])
+                response = LLMGenerator.generate_qa_document(current_user.current_tenant_id, preview_texts[0], doc_language)
                document_qa_list = self.format_split_text(response)
                return {
                    "total_segments": total_segments * 20,
@@ -288,7 +280,8 @@ class IndexingRunner:
            "preview": preview_texts
        }

-    def notion_indexing_estimate(self, tenant_id: str, notion_info_list: list, tmp_processing_rule: dict, doc_form: str = None) -> dict:
+    def notion_indexing_estimate(self, tenant_id: str, notion_info_list: list, tmp_processing_rule: dict,
+                                 doc_form: str = None, doc_language: str = 'English') -> dict:
        """
        Estimate the indexing for the document.
        """
@@ -350,7 +343,7 @@ class IndexingRunner:
        if doc_form and doc_form == 'qa_model':
            if len(preview_texts) > 0:
                # qa model document
-                response = LLMGenerator.generate_qa_document(current_user.current_tenant_id, preview_texts[0])
+                response = LLMGenerator.generate_qa_document(current_user.current_tenant_id, preview_texts[0], doc_language)
                document_qa_list = self.format_split_text(response)
                return {
                    "total_segments": total_segments * 20,
@@ -457,7 +450,8 @@ class IndexingRunner:
            splitter=splitter,
            processing_rule=processing_rule,
            tenant_id=dataset.tenant_id,
-            document_form=dataset_document.doc_form
+            document_form=dataset_document.doc_form,
+            document_language=dataset_document.doc_language
        )

        # save node to document segment
@@ -493,7 +487,8 @@ class IndexingRunner:
        return documents

    def _split_to_documents(self, text_docs: List[Document], splitter: TextSplitter,
-                            processing_rule: DatasetProcessRule, tenant_id: str, document_form: str) -> List[Document]:
+                            processing_rule: DatasetProcessRule, tenant_id: str,
+                            document_form: str, document_language: str) -> List[Document]:
        """
        Split the text documents into nodes.
        """
@@ -522,7 +517,8 @@ class IndexingRunner:
                sub_documents = all_documents[i:i + 10]
                for doc in sub_documents:
                    document_format_thread = threading.Thread(target=self.format_qa_document, kwargs={
-                        'tenant_id': tenant_id, 'document_node': doc, 'all_qa_documents': all_qa_documents})
+                        'tenant_id': tenant_id, 'document_node': doc, 'all_qa_documents': all_qa_documents,
+                        'document_language': document_language})
                    threads.append(document_format_thread)
                    document_format_thread.start()
                for thread in threads:
@@ -530,13 +526,13 @@ class IndexingRunner:
            return all_qa_documents
        return all_documents

-    def format_qa_document(self, tenant_id: str, document_node, all_qa_documents):
+    def format_qa_document(self, tenant_id: str, document_node, all_qa_documents, document_language):
        format_documents = []
        if document_node.page_content is None or not document_node.page_content.strip():
            return
        try:
            # qa model document
-            response = LLMGenerator.generate_qa_document(tenant_id, document_node.page_content)
+            response = LLMGenerator.generate_qa_document(tenant_id, document_node.page_content, document_language)
            document_qa_list = self.format_split_text(response)
            qa_documents = []
            for result in document_qa_list:
@@ -719,6 +715,32 @@ class IndexingRunner:
        DocumentSegment.query.filter_by(document_id=dataset_document_id).update(update_params)
        db.session.commit()

+    def batch_add_segments(self, segments: List[DocumentSegment], dataset: Dataset):
+        """
+        Batch add segments index processing
+        """
+        documents = []
+        for segment in segments:
+            document = Document(
+                page_content=segment.content,
+                metadata={
+                    "doc_id": segment.index_node_id,
+                    "doc_hash": segment.index_node_hash,
+                    "document_id": segment.document_id,
+                    "dataset_id": segment.dataset_id,
+                }
+            )
+            documents.append(document)
+        # save vector index
+        index = IndexBuilder.get_index(dataset, 'high_quality')
+        if index:
+            index.add_texts(documents, duplicate_check=True)
+
+        # save keyword index
+        index = IndexBuilder.get_index(dataset, 'economy')
+        if index:
+            index.add_texts(documents)
+

 class DocumentIsPausedException(Exception):
    pass
--- a/api/core/prompt/prompts.py
+++ b/api/core/prompt/prompts.py
@@ -44,13 +44,13 @@ SUGGESTED_QUESTIONS_AFTER_ANSWER_INSTRUCTION_PROMPT = (
 )

 GENERATOR_QA_PROMPT = (
-    "Please respond according to the language of the user's input text. If the text is in language [A], you must also reply in language [A].\n"
+    'The user will send a long text. Please think step by step.'
    'Step 1: Understand and summarize the main content of this text.\n'
    'Step 2: What key information or concepts are mentioned in this text?\n'
    'Step 3: Decompose or combine multiple pieces of information and concepts.\n'
    'Step 4: Generate 20 questions and answers based on these key information and concepts.'
    'The questions should be clear and detailed, and the answers should be detailed and complete.\n'
-    "Answer in the following format: Q1:\nA1:\nQ2:\nA2:...\n"
+    "Answer must be the language:{language} and in the following format: Q1:\nA1:\nQ2:\nA2:...\n"
 )

 RULE_CONFIG_GENERATE_TEMPLATE = """Given MY INTENDED AUDIENCES and HOPING TO SOLVE using a language model, please select \
--- a/api/migrations/versions/2c8af9671032_add_qa_document_language.py
+++ b/api/migrations/versions/2c8af9671032_add_qa_document_language.py
@@ -0,0 +1,32 @@
+"""add_qa_document_language
+
+Revision ID: 2c8af9671032
+Revises: 8d2d099ceb74
+Create Date: 2023-08-01 18:57:27.294973
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2c8af9671032'
+down_revision = '5022897aaceb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('documents', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('doc_language', sa.String(length=255), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('documents', schema=None) as batch_op:
+        batch_op.drop_column('doc_language')
+
+    # ### end Alembic commands ###
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@@ -209,6 +209,7 @@ class Document(db.Model):
    doc_metadata = db.Column(db.JSON, nullable=True)
    doc_form = db.Column(db.String(
        255), nullable=False, server_default=db.text("'text_model'::character varying"))
+    doc_language = db.Column(db.String(255), nullable=True)

    DATA_SOURCES = ['upload_file', 'notion_import']

--- a/api/requirements.txt
+++ b/api/requirements.txt
@@ -47,4 +47,5 @@ websocket-client~=1.6.1
 dashscope~=1.5.0
 huggingface_hub~=0.16.4
 transformers~=4.31.0
-stripe~=5.5.0
+stripe~=5.5.0
+pandas==1.5.3
--- a/api/services/dataset_service.py
+++ b/api/services/dataset_service.py
@@ -31,8 +31,9 @@ from tasks.document_indexing_task import document_indexing_task
 from tasks.document_indexing_update_task import document_indexing_update_task
 from tasks.create_segment_to_index_task import create_segment_to_index_task
 from tasks.update_segment_index_task import update_segment_index_task
-from tasks.update_segment_keyword_index_task\
-    import update_segment_keyword_index_task
+from tasks.recover_document_indexing_task import recover_document_indexing_task
+from tasks.update_segment_keyword_index_task import update_segment_keyword_index_task
+from tasks.delete_segment_from_index_task import delete_segment_from_index_task


 class DatasetService:
@@ -372,7 +373,7 @@ class DocumentService:
        indexing_cache_key = 'document_{}_is_paused'.format(document.id)
        redis_client.delete(indexing_cache_key)
        # trigger async task
-        document_indexing_task.delay(document.dataset_id, document.id)
+        recover_document_indexing_task.delay(document.dataset_id, document.id)

    @staticmethod
    def get_documents_position(dataset_id):
@@ -450,6 +451,7 @@ class DocumentService:
                    document = DocumentService.save_document(dataset, dataset_process_rule.id,
                                                             document_data["data_source"]["type"],
                                                             document_data["doc_form"],
+                                                             document_data["doc_language"],
                                                             data_source_info, created_from, position,
                                                             account, file_name, batch)
                    db.session.add(document)
@@ -495,20 +497,11 @@ class DocumentService:
                            document = DocumentService.save_document(dataset, dataset_process_rule.id,
                                                                     document_data["data_source"]["type"],
                                                                     document_data["doc_form"],
+                                                                     document_data["doc_language"],
                                                                     data_source_info, created_from, position,
                                                                     account, page['page_name'], batch)
-                            # if page['type'] == 'database':
-                            #     document.splitting_completed_at = datetime.datetime.utcnow()
-                            #     document.cleaning_completed_at = datetime.datetime.utcnow()
-                            #     document.parsing_completed_at = datetime.datetime.utcnow()
-                            #     document.completed_at = datetime.datetime.utcnow()
-                            #     document.indexing_status = 'completed'
-                            #     document.word_count = 0
-                            #     document.tokens = 0
-                            #     document.indexing_latency = 0
                            db.session.add(document)
                            db.session.flush()
-                            # if page['type'] != 'database':
                            document_ids.append(document.id)
                            documents.append(document)
                            position += 1
@@ -520,15 +513,15 @@ class DocumentService:
            db.session.commit()

            # trigger async task
-            #document_index_created.send(dataset.id, document_ids=document_ids)
            document_indexing_task.delay(dataset.id, document_ids)

        return documents, batch

    @staticmethod
    def save_document(dataset: Dataset, process_rule_id: str, data_source_type: str, document_form: str,
-                      data_source_info: dict, created_from: str, position: int, account: Account, name: str,
-                      batch: str):
+                      document_language: str, data_source_info: dict, created_from: str, position: int,
+                      account: Account,
+                      name: str, batch: str):
        document = Document(
            tenant_id=dataset.tenant_id,
            dataset_id=dataset.id,
@@ -540,7 +533,8 @@ class DocumentService:
            name=name,
            created_from=created_from,
            created_by=account.id,
-            doc_form=document_form
+            doc_form=document_form,
+            doc_language=document_language
        )
        return document

@@ -947,3 +941,17 @@ class SegmentService:
            redis_client.setex(indexing_cache_key, 600, 1)
            update_segment_index_task.delay(segment.id, args['keywords'])
        return segment
+
+    @classmethod
+    def delete_segment(cls, segment: DocumentSegment, document: Document, dataset: Dataset):
+        indexing_cache_key = 'segment_{}_delete_indexing'.format(segment.id)
+        cache_result = redis_client.get(indexing_cache_key)
+        if cache_result is not None:
+            raise ValueError("Segment is deleting.")
+        # send delete segment index task
+        redis_client.setex(indexing_cache_key, 600, 1)
+        # enabled segment need to delete index
+        if segment.enabled:
+            delete_segment_from_index_task.delay(segment.id, segment.index_node_id, dataset.id, document.id)
+        db.session.delete(segment)
+        db.session.commit()
--- a/api/tasks/batch_create_segment_to_index_task.py
+++ b/api/tasks/batch_create_segment_to_index_task.py
@@ -0,0 +1,94 @@
+import datetime
+import logging
+import time
+import uuid
+from typing import Optional, List
+
+import click
+from celery import shared_task
+from sqlalchemy import func
+from werkzeug.exceptions import NotFound
+
+from core.index.index import IndexBuilder
+from core.indexing_runner import IndexingRunner
+from core.model_providers.model_factory import ModelFactory
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from libs import helper
+from models.dataset import DocumentSegment, Dataset, Document
+
+
+@shared_task(queue='dataset')
+def batch_create_segment_to_index_task(job_id: str, content: List, dataset_id: str, document_id: str,
+                                       tenant_id: str, user_id: str):
+    """
+    Async batch create segment to index
+    :param job_id:
+    :param content:
+    :param dataset_id:
+    :param document_id:
+    :param tenant_id:
+    :param user_id:
+
+    Usage: batch_create_segment_to_index_task.delay(segment_id)
+    """
+    logging.info(click.style('Start batch create segment jobId: {}'.format(job_id), fg='green'))
+    start_at = time.perf_counter()
+
+    indexing_cache_key = 'segment_batch_import_{}'.format(job_id)
+
+    try:
+        dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first()
+        if not dataset:
+            raise ValueError('Dataset not exist.')
+
+        dataset_document = db.session.query(Document).filter(Document.id == document_id).first()
+        if not dataset_document:
+            raise ValueError('Document not exist.')
+
+        if not dataset_document.enabled or dataset_document.archived or dataset_document.indexing_status != 'completed':
+            raise ValueError('Document is not available.')
+        document_segments = []
+        for segment in content:
+            content = segment['content']
+            answer = segment['answer']
+            doc_id = str(uuid.uuid4())
+            segment_hash = helper.generate_text_hash(content)
+            embedding_model = ModelFactory.get_embedding_model(
+                tenant_id=dataset.tenant_id
+            )
+
+            # calc embedding use tokens
+            tokens = embedding_model.get_num_tokens(content)
+            max_position = db.session.query(func.max(DocumentSegment.position)).filter(
+                DocumentSegment.document_id == dataset_document.id
+            ).scalar()
+            segment_document = DocumentSegment(
+                tenant_id=tenant_id,
+                dataset_id=dataset_id,
+                document_id=document_id,
+                index_node_id=doc_id,
+                index_node_hash=segment_hash,
+                position=max_position + 1 if max_position else 1,
+                content=content,
+                word_count=len(content),
+                tokens=tokens,
+                created_by=user_id,
+                indexing_at=datetime.datetime.utcnow(),
+                status='completed',
+                completed_at=datetime.datetime.utcnow()
+            )
+            if dataset_document.doc_form == 'qa_model':
+                segment_document.answer = answer
+            db.session.add(segment_document)
+            document_segments.append(segment_document)
+        # add index to db
+        indexing_runner = IndexingRunner()
+        indexing_runner.batch_add_segments(document_segments, dataset)
+        db.session.commit()
+        redis_client.setex(indexing_cache_key, 600, 'completed')
+        end_at = time.perf_counter()
+        logging.info(click.style('Segment batch created job: {} latency: {}'.format(job_id, end_at - start_at), fg='green'))
+    except Exception as e:
+        logging.exception("Segments batch created index failed:{}".format(str(e)))
+        redis_client.setex(indexing_cache_key, 600, 'error')
--- a/api/tasks/delete_segment_from_index_task.py
+++ b/api/tasks/delete_segment_from_index_task.py
@@ -0,0 +1,58 @@
+import logging
+import time
+
+import click
+from celery import shared_task
+from werkzeug.exceptions import NotFound
+
+from core.index.index import IndexBuilder
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from models.dataset import DocumentSegment, Dataset, Document
+
+
+@shared_task(queue='dataset')
+def delete_segment_from_index_task(segment_id: str, index_node_id: str, dataset_id: str, document_id: str):
+    """
+    Async Remove segment from index
+    :param segment_id:
+    :param index_node_id:
+    :param dataset_id:
+    :param document_id:
+
+    Usage: delete_segment_from_index_task.delay(segment_id)
+    """
+    logging.info(click.style('Start delete segment from index: {}'.format(segment_id), fg='green'))
+    start_at = time.perf_counter()
+    indexing_cache_key = 'segment_{}_delete_indexing'.format(segment_id)
+    try:
+        dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first()
+        if not dataset:
+            logging.info(click.style('Segment {} has no dataset, pass.'.format(segment_id), fg='cyan'))
+            return
+
+        dataset_document = db.session.query(Document).filter(Document.id == document_id).first()
+        if not dataset_document:
+            logging.info(click.style('Segment {} has no document, pass.'.format(segment_id), fg='cyan'))
+            return
+
+        if not dataset_document.enabled or dataset_document.archived or dataset_document.indexing_status != 'completed':
+            logging.info(click.style('Segment {} document status is invalid, pass.'.format(segment_id), fg='cyan'))
+            return
+
+        vector_index = IndexBuilder.get_index(dataset, 'high_quality')
+        kw_index = IndexBuilder.get_index(dataset, 'economy')
+
+        # delete from vector index
+        if vector_index:
+            vector_index.delete_by_ids([index_node_id])
+
+        # delete from keyword index
+        kw_index.delete_by_ids([index_node_id])
+
+        end_at = time.perf_counter()
+        logging.info(click.style('Segment deleted from index: {} latency: {}'.format(segment_id, end_at - start_at), fg='green'))
+    except Exception:
+        logging.exception("delete segment from index failed")
+    finally:
+        redis_client.delete(indexing_cache_key)
--- a/api/tasks/disable_segment_from_index_task.py
+++ b/api/tasks/disable_segment_from_index_task.py
@@ -12,14 +12,14 @@ from models.dataset import DocumentSegment


@shared_task(queue='dataset')
-def remove_segment_from_index_task(segment_id: str):
+def disable_segment_from_index_task(segment_id: str):
    """
-    Async Remove segment from index
+    Async disable segment from index
    :param segment_id:

-    Usage: remove_segment_from_index.delay(segment_id)
+    Usage: disable_segment_from_index_task.delay(segment_id)
    """
-    logging.info(click.style('Start remove segment from index: {}'.format(segment_id), fg='green'))
+    logging.info(click.style('Start disable segment from index: {}'.format(segment_id), fg='green'))
    start_at = time.perf_counter()

    segment = db.session.query(DocumentSegment).filter(DocumentSegment.id == segment_id).first()
--- a/web/app/components/base/icons/assets/vender/line/general/dots-horizontal.svg
+++ b/web/app/components/base/icons/assets/vender/line/general/dots-horizontal.svg
@@ -1,9 +1,9 @@
-<svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
 <g id="Icon">
 <g id="Icon_2">
-<path d="M6 6.5C6.27614 6.5 6.5 6.27614 6.5 6C6.5 5.72386 6.27614 5.5 6 5.5C5.72386 5.5 5.5 5.72386 5.5 6C5.5 6.27614 5.72386 6.5 6 6.5Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M9.5 6.5C9.77614 6.5 10 6.27614 10 6C10 5.72386 9.77614 5.5 9.5 5.5C9.22386 5.5 9 5.72386 9 6C9 6.27614 9.22386 6.5 9.5 6.5Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
-<path d="M2.5 6.5C2.77614 6.5 3 6.27614 3 6C3 5.72386 2.77614 5.5 2.5 5.5C2.22386 5.5 2 5.72386 2 6C2 6.27614 2.22386 6.5 2.5 6.5Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.00008 8.66634C8.36827 8.66634 8.66675 8.36786 8.66675 7.99967C8.66675 7.63148 8.36827 7.33301 8.00008 7.33301C7.63189 7.33301 7.33341 7.63148 7.33341 7.99967C7.33341 8.36786 7.63189 8.66634 8.00008 8.66634Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M12.6667 8.66634C13.0349 8.66634 13.3334 8.36786 13.3334 7.99967C13.3334 7.63148 13.0349 7.33301 12.6667 7.33301C12.2986 7.33301 12.0001 7.63148 12.0001 7.99967C12.0001 8.36786 12.2986 8.66634 12.6667 8.66634Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M3.33341 8.66634C3.7016 8.66634 4.00008 8.36786 4.00008 7.99967C4.00008 7.63148 3.7016 7.33301 3.33341 7.33301C2.96522 7.33301 2.66675 7.63148 2.66675 7.99967C2.66675 8.36786 2.96522 8.66634 3.33341 8.66634Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
 </g>
 </g>
 </svg>
--- a/web/app/components/base/icons/src/vender/line/general/DotsHorizontal.json
+++ b/web/app/components/base/icons/src/vender/line/general/DotsHorizontal.json
@@ -4,9 +4,9 @@
 		"isRootNode": true,
 		"name": "svg",
 		"attributes": {
-			"width": "12",
-			"height": "12",
-			"viewBox": "0 0 12 12",
+			"width": "16",
+			"height": "16",
+			"viewBox": "0 0 16 16",
 			"fill": "none",
 			"xmlns": "http://www.w3.org/2000/svg"
 		},
@@ -29,7 +29,7 @@
 								"type": "element",
 								"name": "path",
 								"attributes": {
-									"d": "M6 6.5C6.27614 6.5 6.5 6.27614 6.5 6C6.5 5.72386 6.27614 5.5 6 5.5C5.72386 5.5 5.5 5.72386 5.5 6C5.5 6.27614 5.72386 6.5 6 6.5Z",
+									"d": "M8.00008 8.66634C8.36827 8.66634 8.66675 8.36786 8.66675 7.99967C8.66675 7.63148 8.36827 7.33301 8.00008 7.33301C7.63189 7.33301 7.33341 7.63148 7.33341 7.99967C7.33341 8.36786 7.63189 8.66634 8.00008 8.66634Z",
 									"stroke": "currentColor",
 									"stroke-width": "1.5",
 									"stroke-linecap": "round",
@@ -41,7 +41,7 @@
 								"type": "element",
 								"name": "path",
 								"attributes": {
-									"d": "M9.5 6.5C9.77614 6.5 10 6.27614 10 6C10 5.72386 9.77614 5.5 9.5 5.5C9.22386 5.5 9 5.72386 9 6C9 6.27614 9.22386 6.5 9.5 6.5Z",
+									"d": "M12.6667 8.66634C13.0349 8.66634 13.3334 8.36786 13.3334 7.99967C13.3334 7.63148 13.0349 7.33301 12.6667 7.33301C12.2986 7.33301 12.0001 7.63148 12.0001 7.99967C12.0001 8.36786 12.2986 8.66634 12.6667 8.66634Z",
 									"stroke": "currentColor",
 									"stroke-width": "1.5",
 									"stroke-linecap": "round",
@@ -53,7 +53,7 @@
 								"type": "element",
 								"name": "path",
 								"attributes": {
-									"d": "M2.5 6.5C2.77614 6.5 3 6.27614 3 6C3 5.72386 2.77614 5.5 2.5 5.5C2.22386 5.5 2 5.72386 2 6C2 6.27614 2.22386 6.5 2.5 6.5Z",
+									"d": "M3.33341 8.66634C3.7016 8.66634 4.00008 8.36786 4.00008 7.99967C4.00008 7.63148 3.7016 7.33301 3.33341 7.33301C2.96522 7.33301 2.66675 7.63148 2.66675 7.99967C2.66675 8.36786 2.96522 8.66634 3.33341 8.66634Z",
 									"stroke": "currentColor",
 									"stroke-width": "1.5",
 									"stroke-linecap": "round",
@@ -68,4 +68,4 @@
 		]
 	},
 	"name": "DotsHorizontal"
-}
+}
--- a/web/app/components/base/popover/index.tsx
+++ b/web/app/components/base/popover/index.tsx
@@ -9,6 +9,7 @@ type IPopover = {
  position?: 'bottom' | 'br'
  btnElement?: string | React.ReactNode
  btnClassName?: string | ((open: boolean) => string)
+  manualClose?: boolean
 }

 const timeoutDuration = 100
@@ -20,6 +21,7 @@ export default function CustomPopover({
  btnElement,
  className,
  btnClassName,
+  manualClose,
 }: IPopover) {
  const buttonRef = useRef<HTMLButtonElement>(null)
  const timeOutRef = useRef<NodeJS.Timeout | null>(null)
@@ -65,28 +67,36 @@ export default function CustomPopover({
                leaveTo="opacity-0 translate-y-1"
              >
                <Popover.Panel
-                  className={`${s.popupPanel} ${position === 'br' ? 'right-0' : 'transform -translate-x-1/2 left-1/2'} ${className}`}
+                  className={`${s.popupPanel} ${position === 'br' ? 'right-0' : 'translate-x-1/2 left-1/2'} ${className}`}
                  {...(trigger !== 'hover'
                    ? {}
                    : {
                      onMouseLeave: () => onMouseLeave(open),
                      onMouseEnter: () => onMouseEnter(open),
                    })
-                  }>
-                  <div
-                    className={s.panelContainer}
-                    {...(trigger !== 'hover'
-                      ? {}
-                      : {
-                        onMouseLeave: () => onMouseLeave(open),
-                        onMouseEnter: () => onMouseEnter(open),
-                      })
-                    }
-                  >
-                    {cloneElement(htmlContent as React.ReactElement, {
-                      onClose: () => onMouseLeave(open),
-                    })}
-                  </div>
+                  }
+                >
+                  {({ close }) => (
+                    <div
+                      className={s.panelContainer}
+                      {...(trigger !== 'hover'
+                        ? {}
+                        : {
+                          onMouseLeave: () => onMouseLeave(open),
+                          onMouseEnter: () => onMouseEnter(open),
+                        })
+                      }
+                    >
+                      {cloneElement(htmlContent as React.ReactElement, {
+                        onClose: () => onMouseLeave(open),
+                        ...(manualClose
+                          ? {
+                            onClick: close,
+                          }
+                          : {}),
+                      })}
+                    </div>
+                  )}
                </Popover.Panel>
              </Transition>
            </div>
--- a/web/app/components/datasets/create/step-two/index.tsx
+++ b/web/app/components/datasets/create/step-two/index.tsx
@@ -2,12 +2,14 @@
 'use client'
 import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import { useContext } from 'use-context-selector'
 import { useBoolean } from 'ahooks'
 import { XMarkIcon } from '@heroicons/react/20/solid'
 import cn from 'classnames'
 import Link from 'next/link'
 import { groupBy } from 'lodash-es'
 import PreviewItem, { PreviewType } from './preview-item'
+import LanguageSelect from './language-select'
 import s from './index.module.css'
 import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, NotionInfo, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
 import {
@@ -26,7 +28,9 @@ import { DataSourceType } from '@/models/datasets'
 import NotionIcon from '@/app/components/base/notion-icon'
 import Switch from '@/app/components/base/switch'
 import { MessageChatSquare } from '@/app/components/base/icons/src/public/common'
+import { XClose } from '@/app/components/base/icons/src/vender/line/general'
 import { useDatasetDetailContext } from '@/context/dataset-detail'
+import I18n from '@/context/i18n'
 import { IS_CE_EDITION } from '@/config'

 type Page = DataSourceNotionPage & { workspace_id: string }
@@ -78,6 +82,8 @@ const StepTwo = ({
  onCancel,
 }: StepTwoProps) => {
  const { t } = useTranslation()
+  const { locale } = useContext(I18n)
+
  const { mutateDatasetRes } = useDatasetDetailContext()
  const scrollRef = useRef<HTMLDivElement>(null)
  const [scrolled, setScrolled] = useState(false)
@@ -98,6 +104,8 @@ const StepTwo = ({
  const [docForm, setDocForm] = useState<DocForm | string>(
    datasetId && documentDetail ? documentDetail.doc_form : DocForm.TEXT,
  )
+  const [docLanguage, setDocLanguage] = useState<string>(locale === 'en' ? 'English' : 'Chinese')
+  const [QATipHide, setQATipHide] = useState(false)
  const [previewSwitched, setPreviewSwitched] = useState(false)
  const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
  const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
@@ -230,6 +238,7 @@ const StepTwo = ({
        indexing_technique: getIndexing_technique(),
        process_rule: getProcessRule(),
        doc_form: docForm,
+        doc_language: docLanguage,
      }
    }
    if (dataSourceType === DataSourceType.NOTION) {
@@ -241,6 +250,7 @@ const StepTwo = ({
        indexing_technique: getIndexing_technique(),
        process_rule: getProcessRule(),
        doc_form: docForm,
+        doc_language: docLanguage,
      }
    }
    return params
@@ -252,6 +262,7 @@ const StepTwo = ({
      params = {
        original_document_id: documentDetail?.id,
        doc_form: docForm,
+        doc_language: docLanguage,
        process_rule: getProcessRule(),
      } as CreateDocumentReq
    }
@@ -266,6 +277,7 @@ const StepTwo = ({
        indexing_technique: getIndexing_technique(),
        process_rule: getProcessRule(),
        doc_form: docForm,
+        doc_language: docLanguage,
      } as CreateDocumentReq
      if (dataSourceType === DataSourceType.FILE) {
        params.data_source.info_list.file_info_list = {
@@ -348,6 +360,10 @@ const StepTwo = ({
      setDocForm(DocForm.TEXT)
  }

+  const handleSelect = (language: string) => {
+    setDocLanguage(language)
+  }
+
  const changeToEconomicalType = () => {
    if (!hasSetIndexType) {
      setIndexType(IndexingType.ECONOMICAL)
@@ -573,22 +589,33 @@ const StepTwo = ({
                <Link className='text-[#155EEF]' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
              </div>
            )}
-            {IS_CE_EDITION && indexType === IndexingType.QUALIFIED && (
-              <div className='flex justify-between items-center mt-3 px-5 py-4 rounded-xl bg-gray-50 border border-gray-100'>
-                <div className='flex justify-center items-center w-8 h-8 rounded-lg bg-indigo-50'>
-                  <MessageChatSquare className='w-4 h-4' />
-                </div>
-                <div className='grow mx-3'>
-                  <div className='mb-[2px] text-md font-medium text-gray-900'>{t('datasetCreation.stepTwo.QATitle')}</div>
-                  <div className='text-[13px] leading-[18px] text-gray-500'>{t('datasetCreation.stepTwo.QATip')}</div>
-                </div>
-                <div className='shrink-0'>
-                  <Switch
-                    defaultValue={docForm === DocForm.QA}
-                    onChange={handleSwitch}
-                    size='md'
-                  />
+            {!IS_CE_EDITION && indexType === IndexingType.QUALIFIED && (
+              <div className='mt-3 rounded-xl bg-gray-50 border border-gray-100'>
+                <div className='flex justify-between items-center px-5 py-4'>
+                  <div className='flex justify-center items-center w-8 h-8 rounded-lg bg-indigo-50'>
+                    <MessageChatSquare className='w-4 h-4' />
+                  </div>
+                  <div className='grow mx-3'>
+                    <div className='mb-[2px] text-md font-medium text-gray-900'>{t('datasetCreation.stepTwo.QATitle')}</div>
+                    <div className='inline-flex items-center text-[13px] leading-[18px] text-gray-500'>
+                      <span className='pr-1'>{t('datasetCreation.stepTwo.QALanguage')}</span>
+                      <LanguageSelect currentLanguage={docLanguage} onSelect={handleSelect} />
+                    </div>
+                  </div>
+                  <div className='shrink-0'>
+                    <Switch
+                      defaultValue={docForm === DocForm.QA}
+                      onChange={handleSwitch}
+                      size='md'
+                    />
+                  </div>
                </div>
+                {docForm === DocForm.QA && !QATipHide && (
+                  <div className='flex justify-between items-center px-5 py-2 bg-orange-50 border-t border-amber-100 rounded-b-xl text-[13px] leading-[18px] text-medium text-amber-500'>
+                    {t('datasetCreation.stepTwo.QATip')}
+                    <XClose className='w-4 h-4 text-gray-500 cursor-pointer' onClick={() => setQATipHide(true)} />
+                  </div>
+                )}
              </div>
            )}
            <div className={s.source}>
--- a/web/app/components/datasets/create/step-two/language-select/index.tsx
+++ b/web/app/components/datasets/create/step-two/language-select/index.tsx
@@ -0,0 +1,38 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from 'classnames'
+import { ChevronDown } from '@/app/components/base/icons/src/vender/line/arrows'
+import Popover from '@/app/components/base/popover'
+
+export type ILanguageSelectProps = {
+  currentLanguage: string
+  onSelect: (language: string) => void
+}
+
+const LanguageSelect: FC<ILanguageSelectProps> = ({
+  currentLanguage,
+  onSelect,
+}) => {
+  return (
+    <Popover
+      manualClose
+      trigger='click'
+      htmlContent={
+        <div className='w-full py-1'>
+          <div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={() => onSelect('English')}>English</div>
+          <div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={() => onSelect('Chinese')}>简体中文</div>
+        </div>
+      }
+      btnElement={
+        <div className='inline-flex items-center'>
+          <span className='pr-[2px] text-xs leading-[18px] font-medium'>{currentLanguage === 'English' ? 'English' : '简体中文'}</span>
+          <ChevronDown className='w-3 h-3 opacity-60' />
+        </div>
+      }
+      btnClassName={open => cn('!border-0 !px-0 !py-0 !bg-inherit !hover:bg-inherit', open ? 'text-blue-600' : 'text-gray-500')}
+      className='!w-[120px] h-fit !z-20 !translate-x-0 !left-[-16px]'
+    />
+  )
+}
+export default React.memo(LanguageSelect)
--- a/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx
@@ -0,0 +1,59 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import {
+  useCSVDownloader,
+} from 'react-papaparse'
+import { useTranslation } from 'react-i18next'
+import { Download02 as DownloadIcon } from '@/app/components/base/icons/src/vender/solid/general'
+
+const CSV_TEMPLATE = [
+  ['content/question', 'answer(Only need with QA model)'],
+  ['content/question1', 'answer1'],
+  ['content/question2', 'answer2'],
+]
+
+const CSVDownload: FC = () => {
+  const { t } = useTranslation()
+  const { CSVDownloader, Type } = useCSVDownloader()
+
+  return (
+    <div className='mt-6'>
+      <div className='text-sm text-gray-900 font-medium'>{t('share.generation.csvStructureTitle')}</div>
+      <div className='mt-2 max-h-[500px] overflow-auto'>
+        <table className='table-fixed w-full border-separate border-spacing-0 border border-gray-200 rounded-lg text-xs'>
+          <thead className='text-gray-500'>
+            <tr>
+              <td className='h-9 pl-3 pr-2 border-b border-gray-200'>{t('datasetDocuments.list.batchModal.question')}</td>
+              <td className='h-9 pl-3 pr-2 border-b border-gray-200'>{t('datasetDocuments.list.batchModal.answer')}</td>
+            </tr>
+          </thead>
+          <tbody className='text-gray-700'>
+            <tr>
+              <td className='h-9 pl-3 pr-2 border-b border-gray-100 text-[13px]'>{t('datasetDocuments.list.batchModal.question')} 1</td>
+              <td className='h-9 pl-3 pr-2 border-b border-gray-100 text-[13px]'>{t('datasetDocuments.list.batchModal.answer')} 1</td>
+            </tr>
+            <tr>
+              <td className='h-9 pl-3 pr-2 text-[13px]'>{t('datasetDocuments.list.batchModal.question')} 2</td>
+              <td className='h-9 pl-3 pr-2 text-[13px]'>{t('datasetDocuments.list.batchModal.answer')} 2</td>
+            </tr>
+          </tbody>
+        </table>
+      </div>
+      <CSVDownloader
+        className="block mt-2 cursor-pointer"
+        type={Type.Link}
+        filename={'template'}
+        bom={true}
+        data={CSV_TEMPLATE}
+      >
+        <div className='flex items-center h-[18px] space-x-1 text-[#155EEF] text-xs font-medium'>
+          <DownloadIcon className='w-3 h-3' />
+          {t('datasetDocuments.list.batchModal.template')}
+        </div>
+      </CSVDownloader>
+    </div>
+
+  )
+}
+export default React.memo(CSVDownload)
--- a/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx
@@ -0,0 +1,126 @@
+'use client'
+import type { FC } from 'react'
+import React, { useEffect, useRef, useState } from 'react'
+import cn from 'classnames'
+import { useTranslation } from 'react-i18next'
+import { useContext } from 'use-context-selector'
+import { Csv as CSVIcon } from '@/app/components/base/icons/src/public/files'
+import { ToastContext } from '@/app/components/base/toast'
+import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'
+import Button from '@/app/components/base/button'
+
+export type Props = {
+  file: File | undefined
+  updateFile: (file?: File) => void
+}
+
+const CSVUploader: FC<Props> = ({
+  file,
+  updateFile,
+}) => {
+  const { t } = useTranslation()
+  const { notify } = useContext(ToastContext)
+  const [dragging, setDragging] = useState(false)
+  const dropRef = useRef<HTMLDivElement>(null)
+  const dragRef = useRef<HTMLDivElement>(null)
+  const fileUploader = useRef<HTMLInputElement>(null)
+
+  const handleDragEnter = (e: DragEvent) => {
+    e.preventDefault()
+    e.stopPropagation()
+    e.target !== dragRef.current && setDragging(true)
+  }
+  const handleDragOver = (e: DragEvent) => {
+    e.preventDefault()
+    e.stopPropagation()
+  }
+  const handleDragLeave = (e: DragEvent) => {
+    e.preventDefault()
+    e.stopPropagation()
+    e.target === dragRef.current && setDragging(false)
+  }
+  const handleDrop = (e: DragEvent) => {
+    e.preventDefault()
+    e.stopPropagation()
+    setDragging(false)
+    if (!e.dataTransfer)
+      return
+    const files = [...e.dataTransfer.files]
+    if (files.length > 1) {
+      notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.count') })
+      return
+    }
+    updateFile(files[0])
+  }
+  const selectHandle = () => {
+    if (fileUploader.current)
+      fileUploader.current.click()
+  }
+  const removeFile = () => {
+    if (fileUploader.current)
+      fileUploader.current.value = ''
+    updateFile()
+  }
+  const fileChangeHandle = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const currentFile = e.target.files?.[0]
+    updateFile(currentFile)
+  }
+
+  useEffect(() => {
+    dropRef.current?.addEventListener('dragenter', handleDragEnter)
+    dropRef.current?.addEventListener('dragover', handleDragOver)
+    dropRef.current?.addEventListener('dragleave', handleDragLeave)
+    dropRef.current?.addEventListener('drop', handleDrop)
+    return () => {
+      dropRef.current?.removeEventListener('dragenter', handleDragEnter)
+      dropRef.current?.removeEventListener('dragover', handleDragOver)
+      dropRef.current?.removeEventListener('dragleave', handleDragLeave)
+      dropRef.current?.removeEventListener('drop', handleDrop)
+    }
+  }, [])
+
+  return (
+    <div className='mt-6'>
+      <input
+        ref={fileUploader}
+        style={{ display: 'none' }}
+        type="file"
+        id="fileUploader"
+        accept='.csv'
+        onChange={fileChangeHandle}
+      />
+      <div ref={dropRef}>
+        {!file && (
+          <div className={cn('flex items-center h-20 rounded-xl bg-gray-50 border border-dashed border-gray-200 text-sm font-normal', dragging && 'bg-[#F5F8FF] border border-[#B2CCFF]')}>
+            <div className='w-full flex items-center justify-center space-x-2'>
+              <CSVIcon className="shrink-0" />
+              <div className='text-gray-500'>
+                {t('datasetDocuments.list.batchModal.csvUploadTitle')}
+                <span className='text-primary-400 cursor-pointer' onClick={selectHandle}>{t('datasetDocuments.list.batchModal.browse')}</span>
+              </div>
+            </div>
+            {dragging && <div ref={dragRef} className='absolute w-full h-full top-0 left-0'/>}
+          </div>
+        )}
+        {file && (
+          <div className={cn('flex items-center h-20 px-6 rounded-xl bg-gray-50 border border-gray-200 text-sm font-normal group', 'hover:bg-[#F5F8FF] hover:border-[#B2CCFF]')}>
+            <CSVIcon className="shrink-0" />
+            <div className='flex ml-2 w-0 grow'>
+              <span className='max-w-[calc(100%_-_30px)] text-ellipsis whitespace-nowrap overflow-hidden text-gray-800'>{file.name.replace(/.csv$/, '')}</span>
+              <span className='shrink-0 text-gray-500'>.csv</span>
+            </div>
+            <div className='hidden group-hover:flex items-center'>
+              <Button className='!h-8 !px-3 !py-[6px] bg-white !text-[13px] !leading-[18px] text-gray-700' onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.change')}</Button>
+              <div className='mx-2 w-px h-4 bg-gray-200' />
+              <div className='p-2 cursor-pointer' onClick={removeFile}>
+                <Trash03 className='w-4 h-4 text-gray-500' />
+              </div>
+            </div>
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}
+
+export default React.memo(CSVUploader)
--- a/web/app/components/datasets/documents/detail/batch-modal/index.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/index.tsx
@@ -0,0 +1,60 @@
+'use client'
+import type { FC } from 'react'
+import React, { useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import CSVUploader from './csv-uploader'
+import CSVDownloader from './csv-downloader'
+import Button from '@/app/components/base/button'
+import Modal from '@/app/components/base/modal'
+import { XClose } from '@/app/components/base/icons/src/vender/line/general'
+
+export type IBatchModalProps = {
+  isShow: boolean
+  onCancel: () => void
+  onConfirm: (file: File) => void
+}
+
+const BatchModal: FC<IBatchModalProps> = ({
+  isShow,
+  onCancel,
+  onConfirm,
+}) => {
+  const { t } = useTranslation()
+  const [currentCSV, setCurrentCSV] = useState<File>()
+  const handleFile = (file?: File) => setCurrentCSV(file)
+
+  const handleSend = () => {
+    if (!currentCSV)
+      return
+    onCancel()
+    onConfirm(currentCSV)
+  }
+
+  useEffect(() => {
+    if (!isShow)
+      setCurrentCSV(undefined)
+  }, [isShow])
+
+  return (
+    <Modal isShow={isShow} onClose={() => {}} className='px-8 py-6 !max-w-[520px] !rounded-xl'>
+      <div className='relative pb-1 text-xl font-medium leading-[30px] text-gray-900'>{t('datasetDocuments.list.batchModal.title')}</div>
+      <div className='absolute right-4 top-4 p-2 cursor-pointer' onClick={onCancel}>
+        <XClose className='w-4 h-4 text-gray-500' />
+      </div>
+      <CSVUploader
+        file={currentCSV}
+        updateFile={handleFile}
+      />
+      <CSVDownloader />
+      <div className='mt-[28px] pt-6 flex justify-end'>
+        <Button className='mr-2 text-gray-700 text-sm font-medium' onClick={onCancel}>
+          {t('datasetDocuments.list.batchModal.cancel')}
+        </Button>
+        <Button className='text-sm font-medium' type="primary" onClick={handleSend} disabled={!currentCSV}>
+          {t('datasetDocuments.list.batchModal.run')}
+        </Button>
+      </div>
+    </Modal>
+  )
+}
+export default React.memo(BatchModal)
--- a/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx
+++ b/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx
@@ -13,6 +13,7 @@ type IInfiniteVirtualListProps = {
  loadNextPage: () => Promise<any> // Callback function responsible for loading the next page of items.
  onClick: (detail: SegmentDetailModel) => void
  onChangeSwitch: (segId: string, enabled: boolean) => Promise<void>
+  onDelete: (segId: string) => Promise<void>
 }

 const InfiniteVirtualList: FC<IInfiniteVirtualListProps> = ({
@@ -22,6 +23,7 @@ const InfiniteVirtualList: FC<IInfiniteVirtualListProps> = ({
  loadNextPage,
  onClick: onClickCard,
  onChangeSwitch,
+  onDelete,
 }) => {
  // If there are more items to be loaded then add an extra row to hold a loading indicator.
  const itemCount = hasNextPage ? items.length + 1 : items.length
@@ -52,6 +54,7 @@ const InfiniteVirtualList: FC<IInfiniteVirtualListProps> = ({
          detail={segItem}
          onClick={() => onClickCard(segItem)}
          onChangeSwitch={onChangeSwitch}
+          onDelete={onDelete}
          loading={false}
        />
      ))
--- a/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx
+++ b/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx
@@ -1,5 +1,5 @@
 import type { FC } from 'react'
-import React from 'react'
+import React, { useState } from 'react'
 import cn from 'classnames'
 import { ArrowUpRightIcon } from '@heroicons/react/24/outline'
 import { useTranslation } from 'react-i18next'
@@ -7,11 +7,15 @@ import { StatusItem } from '../../list'
 import { DocumentTitle } from '../index'
 import s from './style.module.css'
 import { SegmentIndexTag } from './index'
+import Modal from '@/app/components/base/modal'
+import Button from '@/app/components/base/button'
 import Switch from '@/app/components/base/switch'
 import Divider from '@/app/components/base/divider'
 import Indicator from '@/app/components/header/indicator'
 import { formatNumber } from '@/utils/format'
 import type { SegmentDetailModel } from '@/models/datasets'
+import { AlertCircle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
+import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'

 const ProgressBar: FC<{ percent: number; loading: boolean }> = ({ percent, loading }) => {
  return (
@@ -35,6 +39,7 @@ type ISegmentCardProps = {
  score?: number
  onClick?: () => void
  onChangeSwitch?: (segId: string, enabled: boolean) => Promise<void>
+  onDelete?: (segId: string) => Promise<void>
  scene?: UsageScene
  className?: string
 }
@@ -44,6 +49,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
  score,
  onClick,
  onChangeSwitch,
+  onDelete,
  loading = true,
  scene = 'doc',
  className = '',
@@ -60,6 +66,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
    answer,
  } = detail as any
  const isDocScene = scene === 'doc'
+  const [showModal, setShowModal] = useState(false)

  const renderContent = () => {
    if (answer) {
@@ -86,7 +93,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
        s.segWrapper,
        (isDocScene && !enabled) ? 'bg-gray-25' : '',
        'group',
-        !loading ? 'pb-4' : '',
+        !loading ? 'pb-4 hover:pb-[10px]' : '',
        className,
      )}
      onClick={() => onClick?.()}
@@ -159,10 +166,16 @@ const SegmentCard: FC<ISegmentCardProps> = ({
                  <div className={cn(s.commonIcon, s.targetIcon)} />
                  <div className={s.segDataText}>{formatNumber(hit_count)}</div>
                </div>
-                <div className="flex items-center">
+                <div className="grow flex items-center">
                  <div className={cn(s.commonIcon, s.bezierCurveIcon)} />
                  <div className={s.segDataText}>{index_node_hash}</div>
                </div>
+                <div className='shrink-0 w-6 h-6 flex items-center justify-center rounded-md hover:bg-red-100 hover:text-red-600 cursor-pointer group/delete' onClick={(e) => {
+                  e.stopPropagation()
+                  setShowModal(true)
+                }}>
+                  <Trash03 className='w-[14px] h-[14px] text-gray-500 group-hover/delete:text-red-600' />
+                </div>
              </div>
            </>
            : <>
@@ -187,6 +200,26 @@ const SegmentCard: FC<ISegmentCardProps> = ({
              </div>
            </>
        )}
+      {showModal && <Modal isShow={showModal} onClose={() => setShowModal(false)} className={s.delModal} closable>
+        <div>
+          <div className={s.warningWrapper}>
+            <AlertCircle className='w-6 h-6 text-red-600' />
+          </div>
+          <div className='text-xl font-semibold text-gray-900 mb-1'>{t('datasetDocuments.segment.delete')}</div>
+          <div className='flex gap-2 justify-end'>
+            <Button onClick={() => setShowModal(false)}>{t('common.operation.cancel')}</Button>
+            <Button
+              type='warning'
+              onClick={async () => {
+                await onDelete?.(id)
+              }}
+              className='border-red-700 border-[0.5px]'
+            >
+              {t('common.operation.sure')}
+            </Button>
+          </div>
+        </div>
+      </Modal>}
    </div>
  )
 }
--- a/web/app/components/datasets/documents/detail/completed/index.tsx
+++ b/web/app/components/datasets/documents/detail/completed/index.tsx
@@ -8,6 +8,7 @@ import { debounce, isNil, omitBy } from 'lodash-es'
 import cn from 'classnames'
 import { StatusItem } from '../../list'
 import { DocumentContext } from '../index'
+import { ProcessStatus } from '../segment-add'
 import s from './style.module.css'
 import InfiniteVirtualList from './InfiniteVirtualList'
 import { formatNumber } from '@/utils/format'
@@ -18,7 +19,7 @@ import Input from '@/app/components/base/input'
 import { ToastContext } from '@/app/components/base/toast'
 import type { Item } from '@/app/components/base/select'
 import { SimpleSelect } from '@/app/components/base/select'
-import { disableSegment, enableSegment, fetchSegments, updateSegment } from '@/service/datasets'
+import { deleteSegment, disableSegment, enableSegment, fetchSegments, updateSegment } from '@/service/datasets'
 import type { SegmentDetailModel, SegmentUpdator, SegmentsQuery, SegmentsResponse } from '@/models/datasets'
 import { asyncRunSafe } from '@/utils'
 import type { CommonResponse } from '@/models/common'
@@ -186,13 +187,18 @@ export const splitArray = (arr: any[], size = 3) => {
 type ICompletedProps = {
  showNewSegmentModal: boolean
  onNewSegmentModalChange: (state: boolean) => void
+  importStatus: ProcessStatus | string | undefined
  // data: Array<{}> // all/part segments
 }
 /**
 * Embedding done, show list of all segments
 * Support search and filter
 */
-const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModalChange }) => {
+const Completed: FC<ICompletedProps> = ({
+  showNewSegmentModal,
+  onNewSegmentModalChange,
+  importStatus,
+}) => {
  const { t } = useTranslation()
  const { notify } = useContext(ToastContext)
  const { datasetId = '', documentId = '', docForm } = useContext(DocumentContext)
@@ -241,11 +247,6 @@ const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModal
    getSegments(false)
  }

-  useEffect(() => {
-    if (lastSegmentsRes !== undefined)
-      getSegments(false)
-  }, [selectedStatus, searchValue])
-
  const onClickCard = (detail: SegmentDetailModel) => {
    setCurrSegment({ segInfo: detail, showModal: true })
  }
@@ -272,6 +273,17 @@ const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModal
    }
  }

+  const onDelete = async (segId: string) => {
+    const [e] = await asyncRunSafe<CommonResponse>(deleteSegment({ datasetId, documentId, segmentId: segId }) as Promise<CommonResponse>)
+    if (!e) {
+      notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
+      resetList()
+    }
+    else {
+      notify({ type: 'error', message: t('common.actionMsg.modificationFailed') })
+    }
+  }
+
  const handleUpdateSegment = async (segmentId: string, question: string, answer: string) => {
    const params: SegmentUpdator = { content: '' }
    if (docForm === 'qa_model') {
@@ -308,6 +320,16 @@ const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModal
    setAllSegments([...allSegments])
  }

+  useEffect(() => {
+    if (lastSegmentsRes !== undefined)
+      getSegments(false)
+  }, [selectedStatus, searchValue])
+
+  useEffect(() => {
+    if (importStatus === ProcessStatus.COMPLETED)
+      resetList()
+  }, [importStatus])
+
  return (
    <>
      <div className={s.docSearchWrapper}>
@@ -330,6 +352,7 @@ const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModal
        items={allSegments}
        loadNextPage={getSegments}
        onChangeSwitch={onChangeSwitch}
+        onDelete={onDelete}
        onClick={onClickCard}
      />
      <Modal isShow={currSegment.showModal} onClose={() => {}} className='!max-w-[640px] !overflow-visible'>
--- a/web/app/components/datasets/documents/detail/completed/style.module.css
+++ b/web/app/components/datasets/documents/detail/completed/style.module.css
@@ -132,3 +132,24 @@
 .editTip {
  box-shadow: 0px 4px 6px -2px rgba(16, 24, 40, 0.03), 0px 12px 16px -4px rgba(16, 24, 40, 0.08);
 }
+
+.delModal {
+  background: linear-gradient(
+      180deg,
+      rgba(217, 45, 32, 0.05) 0%,
+      rgba(217, 45, 32, 0) 24.02%
+    ),
+    #f9fafb;
+  box-shadow: 0px 20px 24px -4px rgba(16, 24, 40, 0.08),
+    0px 8px 8px -4px rgba(16, 24, 40, 0.03);
+  @apply rounded-2xl p-8;
+}
+.warningWrapper {
+  box-shadow: 0px 20px 24px -4px rgba(16, 24, 40, 0.08),
+    0px 8px 8px -4px rgba(16, 24, 40, 0.03);
+  background: rgba(255, 255, 255, 0.9);
+  @apply h-12 w-12 border-[0.5px] border-gray-100 rounded-xl mb-3 flex items-center justify-center;
+}
+.warningIcon {
+  @apply w-[22px] h-[22px] fill-current text-red-600;
+}
--- a/web/app/components/datasets/documents/detail/index.tsx
+++ b/web/app/components/datasets/documents/detail/index.tsx
@@ -3,7 +3,7 @@ import type { FC } from 'react'
 import React, { useState } from 'react'
 import useSWR from 'swr'
 import { ArrowLeftIcon } from '@heroicons/react/24/solid'
-import { createContext } from 'use-context-selector'
+import { createContext, useContext } from 'use-context-selector'
 import { useTranslation } from 'react-i18next'
 import { useRouter } from 'next/navigation'
 import { omit } from 'lodash-es'
@@ -13,19 +13,14 @@ import s from '../style.module.css'
 import Completed from './completed'
 import Embedding from './embedding'
 import Metadata from './metadata'
+import SegmentAdd, { ProcessStatus } from './segment-add'
+import BatchModal from './batch-modal'
 import style from './style.module.css'
 import Divider from '@/app/components/base/divider'
 import Loading from '@/app/components/base/loading'
 import type { MetadataType } from '@/service/datasets'
-import { fetchDocumentDetail } from '@/service/datasets'
-
-export const BackCircleBtn: FC<{ onClick: () => void }> = ({ onClick }) => {
-  return (
-    <div onClick={onClick} className={'rounded-full w-8 h-8 flex justify-center items-center border-gray-100 cursor-pointer border hover:border-gray-300 shadow-lg'}>
-      <ArrowLeftIcon className='text-primary-600 fill-current stroke-current h-4 w-4' />
-    </div>
-  )
-}
+import { checkSegmentBatchImportProgress, fetchDocumentDetail, segmentBatchImport } from '@/service/datasets'
+import { ToastContext } from '@/app/components/base/toast'

 export const DocumentContext = createContext<{ datasetId?: string; documentId?: string; docForm: string }>({ docForm: '' })

@@ -51,10 +46,45 @@ type Props = {
 }

 const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
-  const { t } = useTranslation()
  const router = useRouter()
+  const { t } = useTranslation()
+  const { notify } = useContext(ToastContext)
  const [showMetadata, setShowMetadata] = useState(true)
-  const [showNewSegmentModal, setShowNewSegmentModal] = useState(false)
+  const [newSegmentModalVisible, setNewSegmentModalVisible] = useState(false)
+  const [batchModalVisible, setBatchModalVisible] = useState(false)
+  const [importStatus, setImportStatus] = useState<ProcessStatus | string>()
+  const showNewSegmentModal = () => setNewSegmentModalVisible(true)
+  const showBatchModal = () => setBatchModalVisible(true)
+  const hideBatchModal = () => setBatchModalVisible(false)
+  const resetProcessStatus = () => setImportStatus('')
+  const checkProcess = async (jobID: string) => {
+    try {
+      const res = await checkSegmentBatchImportProgress({ jobID })
+      setImportStatus(res.job_status)
+      if (res.job_status === ProcessStatus.WAITING || res.job_status === ProcessStatus.PROCESSING)
+        setTimeout(() => checkProcess(res.job_id), 2500)
+      if (res.job_status === ProcessStatus.ERROR)
+        notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}` })
+    }
+    catch (e: any) {
+      notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
+    }
+  }
+  const runBatch = async (csv: File) => {
+    const formData = new FormData()
+    formData.append('file', csv)
+    try {
+      const res = await segmentBatchImport({
+        url: `/datasets/${datasetId}/documents/${documentId}/segments/batch_import`,
+        body: formData,
+      })
+      setImportStatus(res.job_status)
+      checkProcess(res.job_id)
+    }
+    catch (e: any) {
+      notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
+    }
+  }

  const { data: documentDetail, error, mutate: detailMutate } = useSWR({
    action: 'fetchDocumentDetail',
@@ -91,22 +121,30 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
    <DocumentContext.Provider value={{ datasetId, documentId, docForm: documentDetail?.doc_form || '' }}>
      <div className='flex flex-col h-full'>
        <div className='flex h-16 border-b-gray-100 border-b items-center p-4'>
-          <BackCircleBtn onClick={backToPrev} />
+          <div onClick={backToPrev} className={'rounded-full w-8 h-8 flex justify-center items-center border-gray-100 cursor-pointer border hover:border-gray-300 shadow-[0px_12px_16px_-4px_rgba(16,24,40,0.08),0px_4px_6px_-2px_rgba(16,24,40,0.03)]'}>
+            <ArrowLeftIcon className='text-primary-600 fill-current stroke-current h-4 w-4' />
+          </div>
          <Divider className='!h-4' type='vertical' />
          <DocumentTitle extension={documentDetail?.data_source_info?.upload_file?.extension} name={documentDetail?.name} />
          <StatusItem status={documentDetail?.display_status || 'available'} scene='detail' />
+          <SegmentAdd
+            importStatus={importStatus}
+            clearProcessStatus={resetProcessStatus}
+            showNewSegmentModal={showNewSegmentModal}
+            showBatchModal={showBatchModal}
+          />
          <OperationAction
            scene='detail'
            detail={{
              enabled: documentDetail?.enabled || false,
              archived: documentDetail?.archived || false,
              id: documentId,
+              data_source_type: documentDetail?.data_source_type || '',
              doc_form: documentDetail?.doc_form || '',
            }}
            datasetId={datasetId}
            onUpdate={handleOperate}
            className='!w-[216px]'
-            showNewSegmentModal={() => setShowNewSegmentModal(true)}
          />
          <button
            className={cn(style.layoutRightIcon, showMetadata ? style.iconShow : style.iconClose)}
@@ -120,8 +158,9 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
              {embedding
                ? <Embedding detail={documentDetail} detailUpdate={detailMutate} />
                : <Completed
-                  showNewSegmentModal={showNewSegmentModal}
-                  onNewSegmentModalChange={setShowNewSegmentModal}
+                  showNewSegmentModal={newSegmentModalVisible}
+                  onNewSegmentModalChange={setNewSegmentModalVisible}
+                  importStatus={importStatus}
                />
              }
            </div>
@@ -132,6 +171,11 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
            onUpdate={metadataMutate}
          />}
        </div>
+        <BatchModal
+          isShow={batchModalVisible}
+          onCancel={hideBatchModal}
+          onConfirm={runBatch}
+        />
      </div>
    </DocumentContext.Provider>
  )
--- a/web/app/components/datasets/documents/detail/segment-add/index.tsx
+++ b/web/app/components/datasets/documents/detail/segment-add/index.tsx
@@ -0,0 +1,84 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from 'classnames'
+import { FilePlus02 } from '@/app/components/base/icons/src/vender/line/files'
+import { Loading02 } from '@/app/components/base/icons/src/vender/line/general'
+import { AlertCircle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
+import { CheckCircle } from '@/app/components/base/icons/src/vender/solid/general'
+import Popover from '@/app/components/base/popover'
+
+export type ISegmentAddProps = {
+  importStatus: ProcessStatus | string | undefined
+  clearProcessStatus: () => void
+  showNewSegmentModal: () => void
+  showBatchModal: () => void
+}
+
+export enum ProcessStatus {
+  WAITING = 'waiting',
+  PROCESSING = 'processing',
+  COMPLETED = 'completed',
+  ERROR = 'error',
+}
+
+const SegmentAdd: FC<ISegmentAddProps> = ({
+  importStatus,
+  clearProcessStatus,
+  showNewSegmentModal,
+  showBatchModal,
+}) => {
+  const { t } = useTranslation()
+
+  if (importStatus) {
+    return (
+      <>
+        {(importStatus === ProcessStatus.WAITING || importStatus === ProcessStatus.PROCESSING) && (
+          <div className='relative overflow-hidden inline-flex items-center mr-2 px-3 py-[6px] text-blue-700 bg-[#F5F8FF] rounded-lg border border-black/5'>
+            {importStatus === ProcessStatus.WAITING && <div className='absolute left-0 top-0 w-3/12 h-full bg-[#D1E0FF] z-0'/>}
+            {importStatus === ProcessStatus.PROCESSING && <div className='absolute left-0 top-0 w-2/3 h-full bg-[#D1E0FF] z-0'/>}
+            <Loading02 className='animate-spin mr-2 w-4 h-4' />
+            <span className='font-medium text-[13px] leading-[18px] z-10'>{t('datasetDocuments.list.batchModal.processing')}</span>
+          </div>
+        )}
+        {importStatus === ProcessStatus.COMPLETED && (
+          <div className='inline-flex items-center mr-2 px-3 py-[6px] text-gray-700 bg-[#F6FEF9] rounded-lg border border-black/5'>
+            <CheckCircle className='mr-2 w-4 h-4 text-[#039855]' />
+            <span className='font-medium text-[13px] leading-[18px]'>{t('datasetDocuments.list.batchModal.completed')}</span>
+            <span className='pl-2 font-medium text-[13px] leading-[18px] text-[#155EEF] cursor-pointer' onClick={clearProcessStatus}>{t('datasetDocuments.list.batchModal.ok')}</span>
+          </div>
+        )}
+        {importStatus === ProcessStatus.ERROR && (
+          <div className='inline-flex items-center mr-2 px-3 py-[6px] text-red-600 bg-red-100 rounded-lg border border-black/5'>
+            <AlertCircle className='mr-2 w-4 h-4 text-[#D92D20]' />
+            <span className='font-medium text-[13px] leading-[18px]'>{t('datasetDocuments.list.batchModal.error')}</span>
+            <span className='pl-2 font-medium text-[13px] leading-[18px] text-[#155EEF] cursor-pointer' onClick={clearProcessStatus}>{t('datasetDocuments.list.batchModal.ok')}</span>
+          </div>
+        )}
+      </>
+    )
+  }
+
+  return (
+    <Popover
+      manualClose
+      trigger='click'
+      htmlContent={
+        <div className='w-full py-1'>
+          <div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={showNewSegmentModal}>{t('datasetDocuments.list.action.add')}</div>
+          <div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={showBatchModal}>{t('datasetDocuments.list.action.batchAdd')}</div>
+        </div>
+      }
+      btnElement={
+        <div className='inline-flex items-center'>
+          <FilePlus02 className='w-4 h-4 text-gray-700' />
+          <span className='pl-1'>{t('datasetDocuments.list.action.addButton')}</span>
+        </div>
+      }
+      btnClassName={open => cn('mr-2 !py-[6px] !text-[13px] !leading-[18px] hover:bg-gray-50 border border-gray-200 hover:border-gray-300 hover:shadow-[0_1px_2px_rgba(16,24,40,0.05)]', open ? '!bg-gray-100 !shadow-none' : '!bg-transparent')}
+      className='!w-[132px] h-fit !z-20  !translate-x-0 !left-0'
+    />
+  )
+}
+export default React.memo(SegmentAdd)
--- a/web/app/components/datasets/documents/list.tsx
+++ b/web/app/components/datasets/documents/list.tsx
@@ -22,12 +22,12 @@ import type { IndicatorProps } from '@/app/components/header/indicator'
 import Indicator from '@/app/components/header/indicator'
 import { asyncRunSafe } from '@/utils'
 import { formatNumber } from '@/utils/format'
-import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument } from '@/service/datasets'
+import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument, unArchiveDocument } from '@/service/datasets'
 import NotionIcon from '@/app/components/base/notion-icon'
 import ProgressBar from '@/app/components/base/progress-bar'
 import { DataSourceType, type DocumentDisplayStatus, type SimpleDocumentDetail } from '@/models/datasets'
 import type { CommonResponse } from '@/models/common'
-import { FilePlus02 } from '@/app/components/base/icons/src/vender/line/files'
+import { DotsHorizontal } from '@/app/components/base/icons/src/vender/line/general'

 export const SettingsIcon: FC<{ className?: string }> = ({ className }) => {
  return <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
@@ -86,7 +86,7 @@ export const StatusItem: FC<{
  </div>
 }

-type OperationName = 'delete' | 'archive' | 'enable' | 'disable' | 'sync'
+type OperationName = 'delete' | 'archive' | 'enable' | 'disable' | 'sync' | 'un_archive'

 // operation action for list and detail
 export const OperationAction: FC<{
@@ -101,8 +101,7 @@ export const OperationAction: FC<{
  onUpdate: (operationName?: string) => void
  scene?: 'list' | 'detail'
  className?: string
-  showNewSegmentModal?: () => void
-}> = ({ datasetId, detail, onUpdate, scene = 'list', className = '', showNewSegmentModal }) => {
+}> = ({ datasetId, detail, onUpdate, scene = 'list', className = '' }) => {
  const { id, enabled = false, archived = false, data_source_type } = detail || {}
  const [showModal, setShowModal] = useState(false)
  const { notify } = useContext(ToastContext)
@@ -117,6 +116,9 @@ export const OperationAction: FC<{
      case 'archive':
        opApi = archiveDocument
        break
+      case 'un_archive':
+        opApi = unArchiveDocument
+        break
      case 'enable':
        opApi = enableDocument
        break
@@ -188,22 +190,12 @@ export const OperationAction: FC<{
                <SettingsIcon />
                <span className={s.actionName}>{t('datasetDocuments.list.action.settings')}</span>
              </div>
-              {
-                !isListScene && (
-                  <div className={s.actionItem} onClick={showNewSegmentModal}>
-                    <FilePlus02 className='w-4 h-4 text-gray-500' />
-                    <span className={s.actionName}>{t('datasetDocuments.list.action.add')}</span>
-                  </div>
-                )
-              }
-              {
-                data_source_type === 'notion_import' && (
-                  <div className={s.actionItem} onClick={() => onOperate('sync')}>
-                    <SyncIcon />
-                    <span className={s.actionName}>{t('datasetDocuments.list.action.sync')}</span>
-                  </div>
-                )
-              }
+              {data_source_type === 'notion_import' && (
+                <div className={s.actionItem} onClick={() => onOperate('sync')}>
+                  <SyncIcon />
+                  <span className={s.actionName}>{t('datasetDocuments.list.action.sync')}</span>
+                </div>
+              )}
              <Divider className='my-1' />
            </>
          )}
@@ -211,6 +203,12 @@ export const OperationAction: FC<{
            <ArchiveIcon />
            <span className={s.actionName}>{t('datasetDocuments.list.action.archive')}</span>
          </div>}
+          {archived && (
+            <div className={s.actionItem} onClick={() => onOperate('un_archive')}>
+              <ArchiveIcon />
+              <span className={s.actionName}>{t('datasetDocuments.list.action.unarchive')}</span>
+            </div>
+          )}
          <div className={cn(s.actionItem, s.deleteActionItem, 'group')} onClick={() => setShowModal(true)}>
            <TrashIcon className={'w-4 h-4 stroke-current text-gray-500 stroke-2 group-hover:text-red-500'} />
            <span className={cn(s.actionName, 'group-hover:text-red-500')}>{t('datasetDocuments.list.action.delete')}</span>
@@ -219,7 +217,11 @@ export const OperationAction: FC<{
      }
      trigger='click'
      position='br'
-      btnElement={<div className={cn(s.actionIcon, s.commonIcon)} />}
+      btnElement={
+        <div className={cn(s.commonIcon)}>
+          <DotsHorizontal className='w-4 h-4 text-gray-700' />
+        </div>
+      }
      btnClassName={open => cn(isListScene ? s.actionIconWrapperList : s.actionIconWrapperDetail, open ? '!bg-gray-100 !shadow-none' : '!bg-transparent')}
      className={`!w-[200px] h-fit !z-20 ${className}`}
    />
--- a/web/app/components/datasets/documents/style.module.css
+++ b/web/app/components/datasets/documents/style.module.css
@@ -56,10 +56,6 @@
  background-position: center center;
  background-size: contain;
 }
-.actionIcon {
-  @apply bg-gray-500;
-  mask-image: url(./assets/action.svg);
-}
 .pdfIcon {
  background-image: url(./assets/pdf.svg);
 }
--- a/web/i18n/lang/dataset-creation.en.ts
+++ b/web/i18n/lang/dataset-creation.en.ts
@@ -75,6 +75,7 @@ const translation = {
    economicalTip: 'Use offline vector engines, keyword indexes, etc. to reduce accuracy without spending tokens',
    QATitle: 'Segmenting in Question & Answer format',
    QATip: 'Enable this option will consume more tokens',
+    QALanguage: 'Segment using',
    emstimateCost: 'Estimation',
    emstimateSegment: 'Estimated segments',
    segmentCount: 'segments',
--- a/web/i18n/lang/dataset-creation.zh.ts
+++ b/web/i18n/lang/dataset-creation.zh.ts
@@ -75,6 +75,7 @@ const translation = {
    economicalTip: '使用离线的向量引擎、关键词索引等方式，降低了准确度但无需花费 Token',
    QATitle: '采用 Q&A 分段模式',
    QATip: '开启后将会消耗额外的 token',
+    QALanguage: '分段使用',
    emstimateCost: '执行嵌入预估消耗',
    emstimateSegment: '预估分段数',
    segmentCount: '段',
--- a/web/i18n/lang/dataset-documents.en.ts
+++ b/web/i18n/lang/dataset-documents.en.ts
@@ -17,8 +17,11 @@ const translation = {
    action: {
      uploadFile: 'Upload new file',
      settings: 'Segment settings',
-      add: 'Add new segment',
+      addButton: 'Add segment',
+      add: 'Add a segment',
+      batchAdd: 'Batch add',
      archive: 'Archive',
+      unarchive: 'Unarchive',
      delete: 'Delete',
      enableWarning: 'Archived file cannot be enabled',
      sync: 'Sync',
@@ -53,6 +56,22 @@ const translation = {
      title: 'Are you sure Delete?',
      content: 'If you need to resume processing later, you will continue from where you left off',
    },
+    batchModal: {
+      title: 'Batch add segments',
+      csvUploadTitle: 'Drag and drop your CSV file here, or ',
+      browse: 'browse',
+      tip: 'The CSV file must conform to the following structure:',
+      question: 'Question',
+      answer: 'Answer',
+      template: 'Download the template here',
+      cancel: 'Cancel',
+      run: 'Run Batch',
+      runError: 'Run batch failed',
+      processing: 'In batch processing',
+      completed: 'Import completed',
+      error: 'Import Error',
+      ok: 'OK',
+    },
  },
  metadata: {
    title: 'Metadata',
@@ -319,6 +338,7 @@ const translation = {
    contentEmpty: 'Content can not be empty',
    newTextSegment: 'New Text Segment',
    newQaSegment: 'New Q&A Segment',
+    delete: 'Delete this segment ?',
  },
 }

--- a/web/i18n/lang/dataset-documents.zh.ts
+++ b/web/i18n/lang/dataset-documents.zh.ts
@@ -17,8 +17,11 @@ const translation = {
    action: {
      uploadFile: '上传新文件',
      settings: '分段设置',
+      addButton: '添加分段',
      add: '添加新分段',
+      batchAdd: '批量添加',
      archive: '归档',
+      unarchive: '撤销归档',
      delete: '删除',
      enableWarning: '归档的文件无法启用',
      sync: '同步',
@@ -53,6 +56,22 @@ const translation = {
      title: '确定删除吗？',
      content: '如果您需要稍后恢复处理，您将从您离开的地方继续',
    },
+    batchModal: {
+      title: '批量添加分段',
+      csvUploadTitle: '将您的 CSV 文件拖放到此处，或',
+      browse: '选择文件',
+      tip: 'CSV 文件必须符合以下结构：',
+      question: '问题',
+      answer: '回答',
+      template: '下载模板',
+      cancel: '取消',
+      run: '导入',
+      runError: '批量导入失败',
+      processing: '批量处理中',
+      completed: '导入完成',
+      error: '导入出错',
+      ok: '确定',
+    },
  },
  metadata: {
    title: '元数据',
@@ -318,6 +337,7 @@ const translation = {
    contentEmpty: '内容不能为空',
    newTextSegment: '新文本分段',
    newQaSegment: '新问答分段',
+    delete: '删除这个分段？',
  },
 }

--- a/web/models/datasets.ts
+++ b/web/models/datasets.ts
@@ -179,6 +179,7 @@ export type CreateDocumentReq = {
  original_document_id?: string
  indexing_technique?: string
  doc_form: 'text_model' | 'qa_model'
+  doc_language: string
  data_source: DataSource
  process_rule: ProcessRule
 }
--- a/web/service/datasets.ts
+++ b/web/service/datasets.ts
@@ -118,6 +118,10 @@ export const archiveDocument: Fetcher<CommonResponse, CommonDocReq> = ({ dataset
  return patch(`/datasets/${datasetId}/documents/${documentId}/status/archive`) as Promise<CommonResponse>
 }

+export const unArchiveDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
+  return patch(`/datasets/${datasetId}/documents/${documentId}/status/un_archive`) as Promise<CommonResponse>
+}
+
 export const enableDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
  return patch(`/datasets/${datasetId}/documents/${documentId}/status/enable`) as Promise<CommonResponse>
 }
@@ -138,10 +142,6 @@ export const modifyDocMetadata: Fetcher<CommonResponse, CommonDocReq & { body: {
  return put(`/datasets/${datasetId}/documents/${documentId}/metadata`, { body }) as Promise<CommonResponse>
 }

-export const getDatasetIndexingStatus: Fetcher<{ data: IndexingStatusResponse[] }, string> = (datasetId) => {
-  return get(`/datasets/${datasetId}/indexing-status`) as Promise<{ data: IndexingStatusResponse[] }>
-}
-
 // apis for segments in a document

 export const fetchSegments: Fetcher<SegmentsResponse, CommonDocReq & { params: SegmentsQuery }> = ({ datasetId, documentId, params }) => {
@@ -164,6 +164,18 @@ export const addSegment: Fetcher<{ data: SegmentDetailModel; doc_form: string },
  return post(`/datasets/${datasetId}/documents/${documentId}/segment`, { body }) as Promise<{ data: SegmentDetailModel; doc_form: string }>
 }

+export const deleteSegment: Fetcher<CommonResponse, { datasetId: string; documentId: string; segmentId: string }> = ({ datasetId, documentId, segmentId }) => {
+  return del(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}`) as Promise<CommonResponse>
+}
+
+export const segmentBatchImport: Fetcher<{ job_id: string; job_status: string }, { url: string; body: FormData }> = ({ url, body }) => {
+  return post(url, { body }, { bodyStringify: false, deleteContentType: true }) as Promise<{ job_id: string; job_status: string }>
+}
+
+export const checkSegmentBatchImportProgress: Fetcher<{ job_id: string; job_status: string }, { jobID: string }> = ({ jobID }) => {
+  return get(`/datasets/batch_import_status/${jobID}`) as Promise<{ job_id: string; job_status: string }>
+}
+
 // hit testing
 export const hitTesting: Fetcher<HitTestingResponse, { datasetId: string; queryText: string }> = ({ datasetId, queryText }) => {
  return post(`/datasets/${datasetId}/hit-testing`, { body: { query: queryText } }) as Promise<HitTestingResponse>
Author	SHA1	Message	Date
jyong	400f692f9d	fix merge	2023-08-15 14:31:22 +08:00
jyong	f90eb9e13d	fix merge	2023-08-15 13:34:25 +08:00
jyong	ef04df29c4	Merge branch 'main' into feat/qa-dataset-multi-import # Conflicts: # api/controllers/console/datasets/datasets.py # api/core/generator/llm_generator.py # api/core/indexing_runner.py # api/requirements.txt # api/tasks/disable_segment_from_index_task.py # web/app/components/base/icons/assets/vender/line/general/dots-horizontal.svg # web/app/components/base/icons/src/vender/line/general/DotsHorizontal.json # web/app/components/datasets/create/step-two/index.tsx # web/app/components/datasets/documents/detail/index.tsx	2023-08-15 13:29:39 +08:00
JzoNg	bfb6c52337	fix: width of popover	2023-08-12 18:39:09 +08:00
JzoNg	9fb8d28195	chore: modify temple of segment import	2023-08-12 13:51:56 +08:00
JzoNg	7756f9d4ec	feat: segment batch import	2023-08-12 13:06:47 +08:00
JzoNg	76dd849d15	feat: language selection supported in segmentation	2023-08-08 22:21:09 +08:00
JzoNg	2b5bc9e035	feat: delete segment supported	2023-08-04 17:51:42 +08:00
JzoNg	340d749595	feat: unarchive document supported	2023-08-04 16:41:36 +08:00
jyong	2be29bb194	1.add batch add segment 2.support delete segment 3.support un_archive document 4. QA language	2023-08-01 20:23:07 +08:00
jyong	ceff8edbab	1.add batch add segment 2.support delete segment 3.support un_archive document 4. QA language	2023-08-01 20:22:47 +08:00
jyong	1cd5d5eb2c	add qa thread control	2023-07-31 23:02:38 +08:00