fix: split long memory lines
This commit is contained in:
1 parent
40345642fa
commit
4b11ebb30e
3 files changed
+33
-6
No files matched your search
@@ -0,0 +1,16 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { chunkMarkdown } from "./internal.js";
|
||||
|
||||
describe("chunkMarkdown", () => {
|
||||
it("splits overly long lines into max-sized chunks", () => {
|
||||
const chunkTokens = 400;
|
||||
const maxChars = chunkTokens * 4;
|
||||
const content = "a".repeat(maxChars * 3 + 25);
|
||||
const chunks = chunkMarkdown(content, { tokens: chunkTokens, overlap: 0 });
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.text.length).toBeLessThanOrEqual(maxChars);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user