Internal Changes

deepseek/dsv4_tokenizer.cc

@@ -18,14 +18,14 @@
 #include <stdio.h>
 
 #include <algorithm>
+#include <cstring>
 #include <fstream>
 #include <limits>
 #include <sstream>
 #include <string>
 #include <string_view>
 #include <vector>
 
-
 #include "hwy/base.h"  // HWY_ABORT
 #include "nlohmann/json.hpp"
 
@@ -177,13 +177,14 @@ Dsv4Tokenizer::Dsv4Tokenizer(const std::string& tokenizer_json_path) {
   Init(contents);
 }
 
-Dsv4Tokenizer::Dsv4Tokenizer(std::string_view json_content, bool /*is_content*/) {
+Dsv4Tokenizer::Dsv4Tokenizer(std::string_view json_content,
+                             bool /*is_content*/) {
   Init(json_content);
 }
 
 void Dsv4Tokenizer::Init(std::string_view json_content) {
   json j = json::parse(json_content.begin(), json_content.end(), /*cb=*/nullptr,
-                      /*allow_exceptions=*/false);
+                       /*allow_exceptions=*/false);
   if (j.is_discarded()) {
     HWY_ABORT("Failed to parse tokenizer JSON");
   }
@@ -274,11 +275,15 @@ std::string Dsv4Tokenizer::WrapChat(const std::string& user_msg,
   static const char kAssistant[] =
       "<\xEF\xBD\x9C"
       "Assistant\xEF\xBD\x9C>";
-  std::string out(kBos);
+  const char* think_marker = thinking ? "<think>" : "</think>";
+  std::string out;
+  out.reserve((sizeof(kBos) - 1) + (sizeof(kUser) - 1) + user_msg.size() +
+              (sizeof(kAssistant) - 1) + std::strlen(think_marker));
+  out += kBos;
   out += kUser;
   out += user_msg;
   out += kAssistant;
-  out += thinking ? "<think>" : "</think>";
+  out += think_marker;
   return out;
 }
 
@@ -494,4 +499,21 @@ void Dsv4Tokenizer::AppendDecoded(int id, std::string& out) const {
   out += id_to_bytes_[id];
 }
 
+std::string Dsv4Tokenizer::Decode(const std::vector<int>& ids) const {
+  size_t total_size = 0;
+  for (int id : ids) {
+    if (id >= 0 && static_cast<size_t>(id) < id_to_bytes_.size() &&
+        !is_special_[id]) {
+      total_size += id_to_bytes_[id].size();
+    }
+  }
+
+  std::string out;
+  out.reserve(total_size);
+  for (int id : ids) {
+    AppendDecoded(id, out);
+  }
+  return out;
+}
+
 }  // namespace gcpp

deepseek/dsv4_tokenizer.h

@@ -40,7 +40,6 @@ class Dsv4Tokenizer {
   // 'is_content' is used to distinguish from path constructor.
   Dsv4Tokenizer(std::string_view json_content, bool is_content);
 
-
   // Extracts added tokens (chat markers etc.), pre-tokenizes and BPE-encodes
   // everything in between. Equivalent to HF encode(add_special_tokens=false).
   std::vector<int> Encode(const std::string& text) const;
@@ -55,6 +54,9 @@ class Dsv4Tokenizer {
   // skip_special_tokens=true; other added tokens (e.g. </think>) are kept.
   void AppendDecoded(int id, std::string& out) const;
 
+  // Decodes a sequence of token IDs, skipping special tokens.
+  std::string Decode(const std::vector<int>& ids) const;
+
   size_t VocabSize() const { return id_to_bytes_.size(); }
 
  private:
@@ -65,7 +67,6 @@ class Dsv4Tokenizer {
 
   void Init(std::string_view json_content);
 
-
   // Splits `text` (a span with no added tokens) into pre-tokenization pieces
   // and BPE-encodes each, appending ids.
   void EncodeSegment(const char* bytes, size_t len,