llama : fix whitespace escaping in tokenizer (#2724)

2025-11-01 09:01:57 +00:00 · 2023-08-22 23:10:42 +02:00
parent c63bb1d16a
commit 46ef5b5fcf
3 changed files with 16 additions and 21 deletions
--- a/tests/test-tokenizer-1.cpp
+++ b/tests/test-tokenizer-1.cpp
@@ -11,18 +11,11 @@
 #include <locale>

 static std::string escape_whitespace(const std::string& text) {
-    std::string result;
-    bool escaping = false;
-    result += "\xe2\x96\x81";
+    std::string result = "\xe2\x96\x81";
    for (size_t offs = 0; offs < text.length(); ++offs) {
        if (text[offs] == ' ') {
-            if (!escaping) {
-                result += "\xe2\x96\x81";
-                escaping = true;
-            }
-        }
-        else {
-            escaping = false;
+            result += "\xe2\x96\x81";
+        } else {
            result += text[offs];
        }
    }