explicit allowance for ć, ś, ń, ł etc
This commit is contained in:
1 parent
c0d01ee931
commit
00b93cfe1b
1 file changed
+2
-10
@@ -663,11 +663,7 @@ def output_modifier(string, state):
|
|||||||
# Replace multiple exclamation marks, question marks, or other punctuation with a single instance
|
# Replace multiple exclamation marks, question marks, or other punctuation with a single instance
|
||||||
cleaned_part = re.sub(r"([!?.])\1+", r"\1", part)
|
cleaned_part = re.sub(r"([!?.])\1+", r"\1", part)
|
||||||
# Further clean to remove any other unwanted characters
|
# Further clean to remove any other unwanted characters
|
||||||
cleaned_part = re.sub(
|
cleaned_part = re.sub(r'[^a-zA-Z0-9\s\.,;:!?\-\'"\u0100-\u017F\u0400-\u04FFÀ-ÿ\u0150\u0151\u0170\u0171\$]', '', cleaned_part)
|
||||||
r'[^a-zA-Z0-9\s\.,;:!?\-\'"\u0400-\u04FFÀ-ÿ\u0150\u0151\u0170\u0171\$]',
|
|
||||||
"",
|
|
||||||
cleaned_part,
|
|
||||||
)
|
|
||||||
# Remove all newline characters (single or multiple)
|
# Remove all newline characters (single or multiple)
|
||||||
cleaned_part = re.sub(r"\n+", " ", cleaned_part)
|
cleaned_part = re.sub(r"\n+", " ", cleaned_part)
|
||||||
|
|
||||||
@@ -694,11 +690,7 @@ def output_modifier(string, state):
|
|||||||
# Replace multiple instances of certain punctuation marks with a single instance
|
# Replace multiple instances of certain punctuation marks with a single instance
|
||||||
cleaned_part = re.sub(r"([!?.])\1+", r"\1", cleaned_part)
|
cleaned_part = re.sub(r"([!?.])\1+", r"\1", cleaned_part)
|
||||||
# Further clean to remove any other unwanted characters
|
# Further clean to remove any other unwanted characters
|
||||||
cleaned_part = re.sub(
|
cleaned_part = re.sub(r'[^a-zA-Z0-9\s\.,;:!?\-\'"\u0100-\u017F\u0400-\u04FFÀ-ÿ\u0150\u0151\u0170\u0171\$]', '', cleaned_part)
|
||||||
r'[^a-zA-Z0-9\s\.,;:!?\-\'"\u0400-\u04FFÀ-ÿ\u0150\u0151\u0170\u0171\$]',
|
|
||||||
"",
|
|
||||||
cleaned_part,
|
|
||||||
)
|
|
||||||
# Remove all newline characters (single or multiple)
|
# Remove all newline characters (single or multiple)
|
||||||
cleaned_part = re.sub(r"\n+", " ", cleaned_part)
|
cleaned_part = re.sub(r"\n+", " ", cleaned_part)
|
||||||
# Process the part and give it a non-character name if being used vai API or standalone.
|
# Process the part and give it a non-character name if being used vai API or standalone.
|
||||||
|
|||||||
Reference in new issue
Block a user