small fix
This commit is contained in:
1 parent
558aed1888
commit
b89b6cdc1d
3 files changed
+24
-9
No files matched your search
@@ -927,10 +927,15 @@
|
||||
cleanedText = cleanedText.replace(/\\.\\s'/g, ".'");
|
||||
cleanedText = cleanedText.replace(/(?<!\\w)\['\]+(?!\\w)/g, '');
|
||||
cleanedText = cleanedText.replace(/"/g, "'");
|
||||
cleanedText = cleanedText.replace(/[^a-zA-Z0-9\s.,;:!?\-'"$À-ÿ\u0400-\u04FF\u0150\u0151\u0170\u0171\u0900-\u097F\u2018\u2019\u201C\u201D\u3001\u3002\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF\uAC00-\uD7A3\u1100-\u11FF\u3130-\u318F\uFF01\uFF0c\uFF1A\uFF1B\uFF1F]/g, '');
|
||||
cleanedText = cleanedText.replace(/[^a-zA-Z0-9\s.,;:!?\-'"$À-ÿ\u0400-\u04FF\u0150\u0151\u0170\u0171\u0900-\u097F\u2018\u2019\u201C\u201D\u2026\u3001\u3002\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF\uAC00-\uD7A3\u1100-\u11FF\u3130-\u318F\uFF01\uFF0c\uFF1A\uFF1B\uFF1F]/g, '');
|
||||
cleanedText = cleanedText.replace(/-\\s/g, '; ');
|
||||
cleanedText = cleanedText.replace(/--/g, '; ');
|
||||
let sentences = cleanedText.split(/(?<=\[.!?:\])\\s+/);
|
||||
// Split by Fullwidth/CJK punctuation regardless of if there are spaces behind them
|
||||
// Please test if it works properly. In my test they it should be
|
||||
// (?<=[.!?:])\s+ instead of
|
||||
// (?<=\[.!?:\])\\s+
|
||||
// so I don't know whats going on here
|
||||
let sentences = cleanedText.split(/(?<=\[.!?:\])\\s+|(?<=\[\u3002\uFF01\uFF1A\uFF1F\])/);
|
||||
|
||||
if (!sentences) {
|
||||
console.error("No sentences found in the input text.");
|
||||
|
||||
Reference in new issue
Block a user