[C++] Implement standalone Unicode encoding and decoding handling

Author: jcking

runtime/Cpp/runtime/src/ANTLRInputStream.cpp

@@ -9,7 +9,7 @@
 #include "misc/Interval.h"
 #include "IntStream.h"
 
-#include "support/StringUtils.h"
+#include "support/Utf8.h"
 #include "support/CPPUtils.h"
 
 #include "ANTLRInputStream.h"
@@ -35,28 +35,37 @@ ANTLRInputStream::ANTLRInputStream(std::istream &stream): ANTLRInputStream() {
   load(stream);
 }
 
-void ANTLRInputStream::load(const std::string &input) {
-  load(input.data(), input.size());
+void ANTLRInputStream::load(const std::string &input, bool lenient) {
+  load(input.data(), input.size(), lenient);
 }
 
-void ANTLRInputStream::load(const char *data, size_t length) {
+void ANTLRInputStream::load(const char *data, size_t length, bool lenient) {
   // Remove the UTF-8 BOM if present.
   const char *bom = "\xef\xbb\xbf";
-  if (length >= 3 && strncmp(data, bom, 3) == 0)
-    _data = antlrcpp::utf8_to_utf32(data + 3, data + length);
-  else
-    _data = antlrcpp::utf8_to_utf32(data, data + length);
+  if (length >= 3 && strncmp(data, bom, 3) == 0) {
+    data += 3;
+    length -= 3;
+  }
+  if (lenient) {
+    _data = Utf8::lenientDecode(std::string_view(data, length));
+  } else {
+    auto maybe_utf32 = Utf8::strictDecode(std::string_view(data, length));
+    if (!maybe_utf32.has_value()) {
+      throw IllegalArgumentException("UTF-8 string contains an illegal byte sequence");
+    }
+    _data = std::move(maybe_utf32).value();
+  }
   p = 0;
 }
 
-void ANTLRInputStream::load(std::istream &stream) {
+void ANTLRInputStream::load(std::istream &stream, bool lenient) {
   if (!stream.good() || stream.eof()) // No fail, bad or EOF.
     return;
 
   _data.clear();
 
   std::string s((std::istreambuf_iterator<char>(stream)), std::istreambuf_iterator<char>());
-  load(s.data(), s.length());
+  load(s.data(), s.length(), lenient);
 }
 
 void ANTLRInputStream::reset() {
@@ -144,7 +153,11 @@ std::string ANTLRInputStream::getText(const Interval &interval) {
     return "";
   }
 
-  return antlrcpp::utf32_to_utf8(_data.substr(start, count));
+  auto maybe_utf8 = Utf8::strictEncode(std::u32string_view(_data).substr(start, count));
+  if (!maybe_utf8.has_value()) {
+    throw IllegalArgumentException("Input stream contains invalid Unicode code points");
+  }
+  return std::move(maybe_utf8).value();
 }
 
 std::string ANTLRInputStream::getSourceName() const {
@@ -155,7 +168,11 @@ std::string ANTLRInputStream::getSourceName() const {
 }
 
 std::string ANTLRInputStream::toString() const {
-  return antlrcpp::utf32_to_utf8(_data);
+  auto maybe_utf8 = Utf8::strictEncode(_data);
+  if (!maybe_utf8.has_value()) {
+    throw IllegalArgumentException("Input stream contains invalid Unicode code points");
+  }
+  return std::move(maybe_utf8).value();
 }
 
 void ANTLRInputStream::InitializeInstanceFields() {

runtime/Cpp/runtime/src/ANTLRInputStream.h

@@ -18,7 +18,7 @@ namespace antlr4 {
   protected:
     /// The data being scanned.
     // UTF-32
-    UTF32String _data;
+    std::u32string _data;
 
     /// 0..n-1 index into string of next char </summary>
     size_t p;
@@ -34,9 +34,13 @@ namespace antlr4 {
     ANTLRInputStream(const char *data, size_t length);
     ANTLRInputStream(std::istream &stream);
 
-    virtual void load(const std::string &input);
-    virtual void load(const char *data, size_t length);
-    virtual void load(std::istream &stream);
+    virtual void load(const std::string &input, bool lenient);
+    virtual void load(const char *data, size_t length, bool lenient);
+    virtual void load(std::istream &stream, bool lenient);
+
+    virtual void load(const std::string &input) { load(input, false); }
+    virtual void load(const char *data, size_t length) { load(data, length, false); }
+    virtual void load(std::istream &stream) { load(stream, false); }
 
     /// Reset the stream so that it's in the same state it was
     /// when the object was created *except* the data array is not

runtime/Cpp/runtime/src/UnbufferedCharStream.cpp

@@ -5,7 +5,7 @@
 
 #include "misc/Interval.h"
 #include "Exceptions.h"
-#include "support/StringUtils.h"
+#include "support/Utf8.h"
 
 #include "UnbufferedCharStream.h"
 
@@ -195,7 +195,11 @@ std::string UnbufferedCharStream::getText(const misc::Interval &interval) {
   }
   // convert from absolute to local index
   size_t i = interval.a - bufferStartIndex;
-  return utf32_to_utf8(_data.substr(i, interval.length()));
+  auto maybe_utf8 = Utf8::strictEncode(std::u32string_view(_data).substr(i, interval.length()));
+  if (!maybe_utf8.has_value()) {
+    throw IllegalArgumentException("Unbuffered stream contains invalid Unicode code points");
+  }
+  return std::move(maybe_utf8).value();
 }
 
 size_t UnbufferedCharStream::getBufferStartIndex() const {

runtime/Cpp/runtime/src/antlr4-common.h

@@ -6,38 +6,33 @@
 #pragma once
 
 #include <algorithm>
-#include <assert.h>
 #include <atomic>
+#include <bitset>
+#include <cassert>
 #include <chrono>
+#include <climits>
+#include <cstdint>
+#include <cstdlib>
+#include <exception>
 #include <fstream>
 #include <iostream>
 #include <iterator>
 #include <limits>
-#include <limits.h>
 #include <list>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <set>
-#include <stdarg.h>
-#include <stdint.h>
-#include <stdlib.h>
 #include <sstream>
 #include <stack>
 #include <string>
-#include <typeinfo>
+#include <string_view>
 #include <type_traits>
+#include <typeinfo>
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
 #include <vector>
-#include <mutex>
-#include <exception>
-#include <bitset>
-#include <condition_variable>
-
-#ifndef USE_UTF8_INSTEAD_OF_CODECVT
-  #include <codecvt>
-#endif
 
 // Defines for the Guid class and other platform dependent stuff.
 #ifdef _WIN32
@@ -59,17 +54,6 @@
     typedef __int32 ssize_t;
   #endif
 
-  #if _MSC_VER >= 1900 && _MSC_VER < 2000
-    // VS 2015 has a known bug when using std::codecvt_utf8<char32_t>
-    // so we have to temporarily use __int32 instead.
-    // https://connect.microsoft.com/VisualStudio/feedback/details/1403302/unresolved-external-when-using-codecvt-utf8
-    typedef std::basic_string<__int32> i32string;
-
-    typedef i32string UTF32String;
-  #else
-    typedef std::u32string UTF32String;
-  #endif
-
   #ifdef ANTLR4CPP_EXPORTS
     #define ANTLR4CPP_PUBLIC __declspec(dllexport)
   #else
@@ -80,25 +64,14 @@
     #endif
   #endif
 
-  #if defined(_MSC_VER) && !defined(__clang__)
-    // clang-cl should escape this to prevent [ignored-attributes].
-    namespace std {
-      class ANTLR4CPP_PUBLIC exception; // Prevents warning C4275 from MSVC.
-    } // namespace std
-  #endif
-
 #elif defined(__APPLE__)
-  typedef std::u32string UTF32String;
-
   #define GUID_CFUUID
   #if __GNUC__ >= 4
     #define ANTLR4CPP_PUBLIC __attribute__ ((visibility ("default")))
   #else
     #define ANTLR4CPP_PUBLIC
   #endif
 #else
-  typedef std::u32string UTF32String;
-
   #define GUID_LIBUUID
   #if __GNUC__ >= 6
     #define ANTLR4CPP_PUBLIC __attribute__ ((visibility ("default")))

runtime/Cpp/runtime/src/support/StringUtils.cpp

@@ -7,40 +7,15 @@
 
 namespace antlrcpp {
 
-void replaceAll(std::string& str, std::string const& from, std::string const& to)
-{
-  if (from.empty())
-    return;
-
-  size_t start_pos = 0;
-  while ((start_pos = str.find(from, start_pos)) != std::string::npos) {
-    str.replace(start_pos, from.length(), to);
-    start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx'.
+  void replaceAll(std::string& str, std::string_view from, std::string_view to) {
+    if (from.empty())
+      return;
+
+    size_t start_pos = 0;
+    while ((start_pos = str.find(from, start_pos)) != std::string::npos) {
+      str.replace(start_pos, from.length(), to);
+      start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx'.
+    }
   }
-}
-
-std::string ws2s(std::wstring const& wstr) {
-#ifndef USE_UTF8_INSTEAD_OF_CODECVT
-  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
-  std::string narrow = converter.to_bytes(wstr);
-#else
-  std::string narrow;
-  utf8::utf32to8(wstr.begin(), wstr.end(), std::back_inserter(narrow));
-#endif
-
-  return narrow;
-}
-
-std::wstring s2ws(const std::string &str) {
-#ifndef USE_UTF8_INSTEAD_OF_CODECVT
-  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
-  std::wstring wide = converter.from_bytes(str);
-#else
-  std::wstring wide;
-  utf8::utf8to32(str.begin(), str.end(), std::back_inserter(wide));
-#endif
-
-  return wide;
-}
 
 } // namespace antrlcpp

runtime/Cpp/runtime/src/support/StringUtils.h

@@ -7,70 +7,8 @@
 
 #include "antlr4-common.h"
 
-#ifdef USE_UTF8_INSTEAD_OF_CODECVT
-#include "utf8.h"
-#endif
-
 namespace antlrcpp {
 
-  // For all conversions utf8 <-> utf32.
-  // I wouldn't prefer wstring_convert because: according to
-  // https://en.cppreference.com/w/cpp/locale/wstring_convert,
-  // wstring_convert is deprecated in C++17.
-  // utfcpp (https://github.com/nemtrif/utfcpp) is a substitution.
-#ifndef USE_UTF8_INSTEAD_OF_CODECVT
-  // VS 2015 and VS 2017 have different bugs in std::codecvt_utf8<char32_t> (VS 2013 works fine).
-  #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
-    typedef std::wstring_convert<std::codecvt_utf8<__int32>, __int32> UTF32Converter;
-  #else
-    typedef std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> UTF32Converter;
-  #endif
-#endif
-
-  // The conversion functions fails in VS2017, so we explicitly use a workaround.
-  template<typename T>
-  inline std::string utf32_to_utf8(T const& data)
-  {
-    #ifndef USE_UTF8_INSTEAD_OF_CODECVT
-      // Don't make the converter static or we have to serialize access to it.
-      thread_local UTF32Converter converter;
-
-      #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
-        const auto p = reinterpret_cast<const int32_t *>(data.data());
-        return converter.to_bytes(p, p + data.size());
-      #else
-        return converter.to_bytes(data);
-      #endif
-    #else
-      std::string narrow;
-      utf8::utf32to8(data.begin(), data.end(), std::back_inserter(narrow));
-      return narrow;
-    #endif
-  }
-
-  inline UTF32String utf8_to_utf32(const char* first, const char* last)
-  {
-    #ifndef USE_UTF8_INSTEAD_OF_CODECVT
-      thread_local UTF32Converter converter;
-
-      #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000
-        auto r = converter.from_bytes(first, last);
-        i32string s = reinterpret_cast<const int32_t *>(r.data());
-        return s;
-      #else
-        std::u32string s = converter.from_bytes(first, last);
-        return s;
-      #endif
-    #else
-      UTF32String wide;
-      utf8::utf8to32(first, last, std::back_inserter(wide));
-      return wide;
-    #endif
-  }
-
-  void replaceAll(std::string &str, std::string const& from, std::string const& to);
+  void replaceAll(std::string &str, std::string_view from, std::string_view to);
 
-  // string <-> wstring conversion (UTF-16), e.g. for use with Window's wide APIs.
-  ANTLR4CPP_PUBLIC std::string ws2s(std::wstring const& wstr);
-  ANTLR4CPP_PUBLIC std::wstring s2ws(std::string const& str);
 }

runtime/Cpp/runtime/src/support/Unicode.h

@@ -0,0 +1,26 @@
+/* Copyright (c) 2021 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+namespace antlrcpp {
+
+  class Unicode final {
+  public:
+    static constexpr char32_t REPLACEMENT_CHARACTER = 0xfffd;
+
+    static constexpr bool isValid(char32_t codePoint) {
+      return codePoint < 0xd800 || (codePoint > 0xdfff && codePoint <= 0x10ffff);
+    }
+
+  private:
+    Unicode() = delete;
+
+    Unicode(const Unicode&) = delete;
+
+    Unicode& operator=(const Unicode&) = delete;
+  };
+
+}