Revert "adding to_lower option"

This reverts commit a0caeb1.

test/test_transforms.py

@@ -586,10 +586,8 @@ def test_clip_tokenizer_save_load_torchscript(self):
 class TestBERTTokenizer(TorchtextTestCase):
     def _load_tokenizer(self, test_scripting: bool, return_tokens: bool):
         vocab_file = "bert_base_uncased_vocab.txt"
-        to_lower = True
         tokenizer = transforms.BERTTokenizer(
             vocab_path=get_asset_path(vocab_file),
-            to_lower=to_lower,
             return_tokens=return_tokens,
         )
         if test_scripting:
@@ -639,6 +637,6 @@ def test_bert_tokenizer_save_load_torchscript(self):
         tokenizer_path = os.path.join(self.test_dir, "bert_tokenizer_torchscript.pt")
         # Call the __prepare_scriptable__() func and convert the building block to the torbhind version
         # Not expect users to use the torchbind version on eager mode but still need a CI test here.
-        torch.save(torch.jit.script(tokenizer), tokenizer_path)
+        torch.save(tokenizer.__prepare_scriptable__(), tokenizer_path)
         loaded_tokenizer = torch.load(tokenizer_path)
         self._bert_tokenizer((loaded_tokenizer))

torchtext/csrc/bert_tokenizer.cpp

@@ -86,15 +86,10 @@ static void _to_lower(UString& text) {
   }
 }
 
-BERTEncoder::BERTEncoder(const std::string& vocab_file, bool to_lower)
-    : vocab_{_load_vocab_from_file(vocab_file, 1, 1)} {
-  to_lower_ = to_lower;
-}
+BERTEncoder::BERTEncoder(const std::string& vocab_file)
+    : vocab_{_load_vocab_from_file(vocab_file, 1, 1)} {}
 
-BERTEncoder::BERTEncoder(std::vector<std::string> tokens, bool to_lower)
-    : vocab_{Vocab(tokens)} {
-  to_lower = to_lower_;
-}
+BERTEncoder::BERTEncoder(Vocab vocab) : vocab_{vocab} {}
 
 UString BERTEncoder::_clean(UString text) {
   /* This function combines:
@@ -219,8 +214,7 @@ std::vector<std::string> BERTEncoder::Tokenize(std::string text) {
   unicodes = _basic_tokenize(unicodes);
 
   // Convert text to lower-case
-  if (to_lower_)
-    _to_lower(unicodes);
+  _to_lower(unicodes);
 
   // Convert back to string from code-points
   std::string newtext = _convert_from_unicode(unicodes);
@@ -250,24 +244,37 @@ std::vector<int64_t> BERTEncoder::Encode(std::string text) {
   return indices;
 }
 
-BERTEncoderStates _serialize_bert_encoder(
+VocabStates _serialize_bert_encoder(
     const c10::intrusive_ptr<BERTEncoder>& self) {
-  auto strings = self->vocab_.itos_;
-  return std::make_tuple(self->to_lower_, std::move(strings));
+  return _serialize_vocab(c10::make_intrusive<Vocab>(self->vocab_));
 }
 
-c10::intrusive_ptr<BERTEncoder> _deserialize_bert_encoder(
-    BERTEncoderStates states) {
+c10::intrusive_ptr<BERTEncoder> _deserialize_bert_encoder(VocabStates states) {
   auto state_size = std::tuple_size<decltype(states)>::value;
   TORCH_CHECK(
-      state_size == 2,
-      "Expected deserialized BERTEncoder to have 2 states but found " +
+      state_size == 4,
+      "Expected deserialized Vocab to have 4 states but found " +
           std::to_string(state_size) + " states");
 
-  auto& to_lower = std::get<0>(states);
-  auto& strings = std::get<1>(states);
+  auto& version_str = std::get<0>(states);
+  auto& integers = std::get<1>(states);
+  auto& strings = std::get<2>(states);
+  auto& tensors = std::get<3>(states);
+
+  // check tensors are empty
+  TORCH_CHECK(tensors.size() == 0, "Expected `tensors` states to be empty");
 
-  return c10::make_intrusive<BERTEncoder>(std::move(strings), to_lower);
+  // throw error if version is not compatible
+  TORCH_CHECK(
+      version_str.compare("0.0.2") >= 0,
+      "Found unexpected version for serialized Vocab: " + version_str);
+
+  c10::optional<int64_t> default_index = {};
+  if (integers.size() > 0) {
+    default_index = integers[0];
+  }
+  return c10::make_intrusive<BERTEncoder>(
+      Vocab(std::move(strings), default_index));
 }
 
 } // namespace torchtext

torchtext/csrc/bert_tokenizer.h

@@ -6,15 +6,12 @@ namespace torchtext {
 
 typedef std::basic_string<uint32_t> UString;
 
-typedef std::tuple<bool, std::vector<std::string>> BERTEncoderStates;
-
 struct BERTEncoder : torch::CustomClassHolder {
-  BERTEncoder(const std::string& vocab_file, bool to_lower);
-  BERTEncoder(std::vector<std::string> tokens, bool to_lower);
+  BERTEncoder(const std::string& vocab_file);
+  BERTEncoder(Vocab vocab);
   std::vector<std::string> Tokenize(std::string text);
   std::vector<int64_t> Encode(std::string text);
   Vocab vocab_;
-  bool to_lower_;
 
  protected:
   UString _clean(UString text);
@@ -27,8 +24,7 @@ struct BERTEncoder : torch::CustomClassHolder {
   static std::string kUnkToken;
 };
 
-BERTEncoderStates _serialize_bert_encoder(
+VocabStates _serialize_bert_encoder(
     const c10::intrusive_ptr<BERTEncoder>& self);
-c10::intrusive_ptr<BERTEncoder> _deserialize_bert_encoder(
-    BERTEncoderStates states);
+c10::intrusive_ptr<BERTEncoder> _deserialize_bert_encoder(VocabStates states);
 } // namespace torchtext

torchtext/csrc/register_pybindings.cpp

@@ -217,16 +217,16 @@ PYBIND11_MODULE(_torchtext, m) {
           }));
 
   py::class_<BERTEncoder, c10::intrusive_ptr<BERTEncoder>>(m, "BERTEncoder")
-      .def(py::init<const std::string, bool>())
+      .def(py::init<const std::string>())
       .def("encode", &BERTEncoder::Encode)
       .def("tokenize", &BERTEncoder::Tokenize)
       .def(py::pickle(
           // __getstate__
-          [](const c10::intrusive_ptr<BERTEncoder>& self) -> BERTEncoderStates {
+          [](const c10::intrusive_ptr<BERTEncoder>& self) -> VocabStates {
             return _serialize_bert_encoder(self);
           },
           // __setstate__
-          [](BERTEncoderStates states) -> c10::intrusive_ptr<BERTEncoder> {
+          [](VocabStates states) -> c10::intrusive_ptr<BERTEncoder> {
             return _deserialize_bert_encoder(states);
           }));
 

torchtext/csrc/register_torchbindings.cpp

@@ -174,16 +174,16 @@ TORCH_LIBRARY_FRAGMENT(torchtext, m) {
           });
 
   m.class_<BERTEncoder>("BERTEncoder")
-      .def(torch::init<const std::string, bool>())
+      .def(torch::init<const std::string>())
       .def("encode", &BERTEncoder::Encode)
       .def("tokenize", &BERTEncoder::Tokenize)
       .def_pickle(
           // __getstate__
-          [](const c10::intrusive_ptr<BERTEncoder>& self) -> BERTEncoderStates {
+          [](const c10::intrusive_ptr<BERTEncoder>& self) -> VocabStates {
             return _serialize_bert_encoder(self);
           },
           // __setstate__
-          [](BERTEncoderStates states) -> c10::intrusive_ptr<BERTEncoder> {
+          [](VocabStates states) -> c10::intrusive_ptr<BERTEncoder> {
             return _deserialize_bert_encoder(states);
           });
   ;

torchtext/transforms.py

@@ -539,12 +539,11 @@ class BERTTokenizer(Module):
     Transform for BERT Tokenizer.
     """
 
-    def __init__(self, vocab_path: str, to_lower:bool, return_tokens=False) -> None:
+    def __init__(self, vocab_path: str, return_tokens=False) -> None:
         super().__init__()
-        self.bert_model = BERTEncoderPyBind(vocab_path, to_lower)
+        self.bert_model = BERTEncoderPyBind(vocab_path)
         self._return_tokens = return_tokens
         self._vocab_path = vocab_path
-        self._to_lower = to_lower
 
     @property
     def is_jitable(self):
@@ -609,7 +608,7 @@ def __prepare_scriptable__(self):
 
         if not self.is_jitable:
             tokenizer_copy = deepcopy(self)
-            tokenizer_copy.bert_model = torch.classes.torchtext.BERTEncoder(self._vocab_path, self._to_lower)
+            tokenizer_copy.bert_model = torch.classes.torchtext.BERTEncoder(self._vocab_path)
             return tokenizer_copy
 
         return self