leftyer/mindformers
1
0
Fork 0
mirror of https://gitee.com/mindspore/mindformers.git synced 2025-12-06 11:29:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

!7100 【bugfix】【master】修复删除indexed_dataloader后导包错误问题

Browse Source 
Merge pull request !7100 from hsshuai/bugfix/master/delete_ut_indexed_dataset
This commit is contained in:
i-robot
2025-08-25 08:41:15 +00:00
committed by Gitee
parent 0a594dafd3 a96bca95bf
commit 9c11ae6990
3 changed files with 6 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
research/deepseek3/wikitext_to_bin.py
View File

@@ -23,7 +23,7 @@ except ImportError:
nltk_available = False
# pylint: disable=W0611
from mindformers.dataset.dataloader import indexed_dataset
from mindformers.dataset.blended_datasets.indexed_dataset import IndexedDatasetBuilder
from mindformers.models import build_tokenizer
from mindformers.models.tokenization_utils import AddedToken
from mindformers.models.llama.llama_tokenizer_fast import LlamaTokenizerFast
@@ -238,7 +238,7 @@ class Partition:
key, level)
output_idx_files[key] = "{}_{}_{}.idx".format(output_prefix,
key, level)
builders[key] = indexed_dataset.IndexedDataBuilder(
builders[key] = IndexedDatasetBuilder(
output_bin_files[key],
dtype=np.int32,
)
@@ -535,7 +535,7 @@ def main():
key, level)
output_idx_files[key] = "{}_{}_{}.idx".format(args.output_prefix,
key, level)
builders[key] = indexed_dataset.IndexedDataBuilder(
builders[key] = IndexedDatasetBuilder(
output_bin_files[key],
dtype=np.int32,
)