[C++] Implement standalone Unicode encoding and decoding handling

Author: jcking

runtime/Cpp/deploy-macos.sh

@@ -4,18 +4,6 @@
 rm -f -R antlr4-runtime build lib 2> /dev/null
 rm antlr4-cpp-runtime-macos.zip 2> /dev/null
 
-# Get utf8 dependency.
-mkdir -p runtime/thirdparty 2> /dev/null
-pushd runtime/thirdparty
-if [ ! -d utfcpp ]
-then
-    git clone https://github.com/nemtrif/utfcpp.git utfcpp
-    pushd utfcpp
-    git checkout tags/v3.1.1
-    popd
-fi
-popd
-
 # Binaries
 xcodebuild -project runtime/antlrcpp.xcodeproj \
            -target antlr4                      \
@@ -35,9 +23,6 @@ rm -f -R antlr4-runtime
 pushd runtime/src
 find . -name '*.h' | cpio -pdm ../../antlr4-runtime
 popd
-pushd runtime/thirdparty/utfcpp/source
-find . -name '*.h' | cpio -pdm ../../../../antlr4-runtime
-popd
 
 # Zip up and clean up
 zip -r antlr4-cpp-runtime-macos.zip antlr4-runtime lib

runtime/Cpp/runtime/CMakeLists.txt

@@ -35,46 +35,6 @@ add_custom_target(make_lib_output_dir ALL
 add_dependencies(antlr4_shared make_lib_output_dir)
 add_dependencies(antlr4_static make_lib_output_dir)
 
-find_package(utf8cpp QUIET)
-
-set(INSTALL_utf8cpp FALSE)
-
-if (utf8cpp_FOUND)
-  target_link_libraries(antlr4_shared utf8cpp)
-  target_link_libraries(antlr4_static utf8cpp)
-else()
-
-  # older utf8cpp doesn't define the package above
-  find_path(utf8cpp_HEADER utf8.h
-    PATH_SUFFIXES utf8cpp
-    )
-
-  if (utf8cpp_HEADER)
-    include_directories(${utf8cpp_HEADER})
-  else()
-    include(${CMAKE_ROOT}/Modules/ExternalProject.cmake)
-    set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
-    set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
-    ExternalProject_Add(
-      utf8cpp
-      GIT_REPOSITORY        "https://github.com/nemtrif/utfcpp"
-      GIT_TAG               "v3.1.1"
-      SOURCE_DIR            ${UTFCPP_DIR}
-      UPDATE_DISCONNECTED   1
-      CMAKE_ARGS            -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -DUTF8_TESTS=off -DUTF8_SAMPLES=off
-      STEP_TARGETS          build)
-
-    include_directories(
-      ${UTFCPP_DIR}/install/include/utf8cpp
-      ${UTFCPP_DIR}/install/include/utf8cpp/utf8
-      )
-
-    add_dependencies(antlr4_shared utf8cpp)
-    add_dependencies(antlr4_static utf8cpp)
-    set(INSTALL_utf8cpp TRUE)
-  endif()
-endif()
-
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")
   target_link_libraries(antlr4_shared ${UUID_LIBRARIES})
   target_link_libraries(antlr4_static ${UUID_LIBRARIES})
@@ -152,16 +112,3 @@ install(DIRECTORY "${PROJECT_SOURCE_DIR}/runtime/src/"
         COMPONENT dev
         FILES_MATCHING PATTERN "*.h"
         )
-
-if (INSTALL_utf8cpp)
-  install(FILES "${UTFCPP_DIR}/source/utf8.h"
-	  DESTINATION "include/antlr4-runtime")
-  install(DIRECTORY "${UTFCPP_DIR}/source/utf8"
-	  DESTINATION "include/antlr4-runtime"
-	  COMPONENT dev
-	  FILES_MATCHING PATTERN "*.h"
-	  )
-endif()
-
-
-

runtime/Cpp/runtime/src/ANTLRInputStream.cpp

@@ -9,7 +9,7 @@
 #include "misc/Interval.h"
 #include "IntStream.h"
 
-#include "support/StringUtils.h"
+#include "support/Utf8.h"
 #include "support/CPPUtils.h"
 
 #include "ANTLRInputStream.h"
@@ -35,28 +35,37 @@ ANTLRInputStream::ANTLRInputStream(std::istream &stream): ANTLRInputStream() {
   load(stream);
 }
 
-void ANTLRInputStream::load(const std::string &input) {
-  load(input.data(), input.size());
+void ANTLRInputStream::load(const std::string &input, bool lenient) {
+  load(input.data(), input.size(), lenient);
 }
 
-void ANTLRInputStream::load(const char *data, size_t length) {
+void ANTLRInputStream::load(const char *data, size_t length, bool lenient) {
   // Remove the UTF-8 BOM if present.
   const char *bom = "\xef\xbb\xbf";
-  if (length >= 3 && strncmp(data, bom, 3) == 0)
-    _data = antlrcpp::utf8_to_utf32(data + 3, data + length);
-  else
-    _data = antlrcpp::utf8_to_utf32(data, data + length);
+  if (length >= 3 && strncmp(data, bom, 3) == 0) {
+    data += 3;
+    length -= 3;
+  }
+  if (lenient) {
+    _data = Utf8::lenientDecode(std::string_view(data, length));
+  } else {
+    auto maybe_utf32 = Utf8::strictDecode(std::string_view(data, length));
+    if (!maybe_utf32.has_value()) {
+      throw IllegalArgumentException("UTF-8 string contains an illegal byte sequence");
+    }
+    _data = std::move(maybe_utf32).value();
+  }
   p = 0;
 }
 
-void ANTLRInputStream::load(std::istream &stream) {
+void ANTLRInputStream::load(std::istream &stream, bool lenient) {
   if (!stream.good() || stream.eof()) // No fail, bad or EOF.
     return;
 
   _data.clear();
 
   std::string s((std::istreambuf_iterator<char>(stream)), std::istreambuf_iterator<char>());
-  load(s.data(), s.length());
+  load(s.data(), s.length(), lenient);
 }
 
 void ANTLRInputStream::reset() {
@@ -144,7 +153,11 @@ std::string ANTLRInputStream::getText(const Interval &interval) {
     return "";
   }
 
-  return antlrcpp::utf32_to_utf8(_data.substr(start, count));
+  auto maybe_utf8 = Utf8::strictEncode(std::u32string_view(_data).substr(start, count));
+  if (!maybe_utf8.has_value()) {
+    throw IllegalArgumentException("Input stream contains invalid Unicode code points");
+  }
+  return std::move(maybe_utf8).value();
 }
 
 std::string ANTLRInputStream::getSourceName() const {
@@ -155,7 +168,11 @@ std::string ANTLRInputStream::getSourceName() const {
 }
 
 std::string ANTLRInputStream::toString() const {
-  return antlrcpp::utf32_to_utf8(_data);
+  auto maybe_utf8 = Utf8::strictEncode(_data);
+  if (!maybe_utf8.has_value()) {
+    throw IllegalArgumentException("Input stream contains invalid Unicode code points");
+  }
+  return std::move(maybe_utf8).value();
 }
 
 void ANTLRInputStream::InitializeInstanceFields() {

runtime/Cpp/runtime/src/ANTLRInputStream.h

@@ -18,7 +18,7 @@ namespace antlr4 {
   protected:
     /// The data being scanned.
     // UTF-32
-    UTF32String _data;
+    std::u32string _data;
 
     /// 0..n-1 index into string of next char </summary>
     size_t p;
@@ -34,9 +34,13 @@ namespace antlr4 {
     ANTLRInputStream(const char *data, size_t length);
     ANTLRInputStream(std::istream &stream);
 
-    virtual void load(const std::string &input);
-    virtual void load(const char *data, size_t length);
-    virtual void load(std::istream &stream);
+    virtual void load(const std::string &input, bool lenient);
+    virtual void load(const char *data, size_t length, bool lenient);
+    virtual void load(std::istream &stream, bool lenient);
+
+    virtual void load(const std::string &input) { load(input, false); }
+    virtual void load(const char *data, size_t length) { load(data, length, false); }
+    virtual void load(std::istream &stream) { load(stream, false); }
 
     /// Reset the stream so that it's in the same state it was
     /// when the object was created *except* the data array is not

runtime/Cpp/runtime/src/UnbufferedCharStream.cpp

@@ -5,7 +5,7 @@
 
 #include "misc/Interval.h"
 #include "Exceptions.h"
-#include "support/StringUtils.h"
+#include "support/Utf8.h"
 
 #include "UnbufferedCharStream.h"
 
@@ -195,7 +195,11 @@ std::string UnbufferedCharStream::getText(const misc::Interval &interval) {
   }
   // convert from absolute to local index
   size_t i = interval.a - bufferStartIndex;
-  return utf32_to_utf8(_data.substr(i, interval.length()));
+  auto maybe_utf8 = Utf8::strictEncode(std::u32string_view(_data).substr(i, interval.length()));
+  if (!maybe_utf8.has_value()) {
+    throw IllegalArgumentException("Unbuffered stream contains invalid Unicode code points");
+  }
+  return std::move(maybe_utf8).value();
 }
 
 size_t UnbufferedCharStream::getBufferStartIndex() const {

runtime/Cpp/runtime/src/antlr4-common.h

@@ -6,38 +6,33 @@
 #pragma once
 
 #include <algorithm>
-#include <assert.h>
 #include <atomic>
+#include <bitset>
+#include <cassert>
 #include <chrono>
+#include <climits>
+#include <cstdint>
+#include <cstdlib>
+#include <exception>
 #include <fstream>
 #include <iostream>
 #include <iterator>
 #include <limits>
-#include <limits.h>
 #include <list>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <set>
-#include <stdarg.h>
-#include <stdint.h>
-#include <stdlib.h>
 #include <sstream>
 #include <stack>
 #include <string>
-#include <typeinfo>
+#include <string_view>
 #include <type_traits>
+#include <typeinfo>
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
 #include <vector>
-#include <mutex>
-#include <exception>
-#include <bitset>
-#include <condition_variable>
-
-#ifndef USE_UTF8_INSTEAD_OF_CODECVT
-  #include <codecvt>
-#endif
 
 // Defines for the Guid class and other platform dependent stuff.
 #ifdef _WIN32
@@ -59,17 +54,6 @@
     typedef __int32 ssize_t;
   #endif
 
-  #if _MSC_VER >= 1900 && _MSC_VER < 2000
-    // VS 2015 has a known bug when using std::codecvt_utf8<char32_t>
-    // so we have to temporarily use __int32 instead.
-    // https://connect.microsoft.com/VisualStudio/feedback/details/1403302/unresolved-external-when-using-codecvt-utf8
-    typedef std::basic_string<__int32> i32string;
-
-    typedef i32string UTF32String;
-  #else
-    typedef std::u32string UTF32String;
-  #endif
-
   #ifdef ANTLR4CPP_EXPORTS
     #define ANTLR4CPP_PUBLIC __declspec(dllexport)
   #else
@@ -80,25 +64,14 @@
     #endif
   #endif
 
-  #if defined(_MSC_VER) && !defined(__clang__)
-    // clang-cl should escape this to prevent [ignored-attributes].
-    namespace std {
-      class ANTLR4CPP_PUBLIC exception; // Prevents warning C4275 from MSVC.
-    } // namespace std
-  #endif
-
 #elif defined(__APPLE__)
-  typedef std::u32string UTF32String;
-
   #define GUID_CFUUID
   #if __GNUC__ >= 4
     #define ANTLR4CPP_PUBLIC __attribute__ ((visibility ("default")))
   #else
     #define ANTLR4CPP_PUBLIC
   #endif
 #else
-  typedef std::u32string UTF32String;
-
   #define GUID_LIBUUID
   #if __GNUC__ >= 6
     #define ANTLR4CPP_PUBLIC __attribute__ ((visibility ("default")))

runtime/Cpp/runtime/src/support/StringUtils.cpp

@@ -7,40 +7,15 @@
 
 namespace antlrcpp {
 
-void replaceAll(std::string& str, std::string const& from, std::string const& to)
-{
-  if (from.empty())
-    return;
-
-  size_t start_pos = 0;
-  while ((start_pos = str.find(from, start_pos)) != std::string::npos) {
-    str.replace(start_pos, from.length(), to);
-    start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx'.
+  void replaceAll(std::string& str, std::string_view from, std::string_view to) {
+    if (from.empty())
+      return;
+
+    size_t start_pos = 0;
+    while ((start_pos = str.find(from, start_pos)) != std::string::npos) {
+      str.replace(start_pos, from.length(), to);
+      start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx'.
+    }
   }
-}
-
-std::string ws2s(std::wstring const& wstr) {
-#ifndef USE_UTF8_INSTEAD_OF_CODECVT
-  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
-  std::string narrow = converter.to_bytes(wstr);
-#else
-  std::string narrow;
-  utf8::utf32to8(wstr.begin(), wstr.end(), std::back_inserter(narrow));
-#endif
-
-  return narrow;
-}
-
-std::wstring s2ws(const std::string &str) {
-#ifndef USE_UTF8_INSTEAD_OF_CODECVT
-  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
-  std::wstring wide = converter.from_bytes(str);
-#else
-  std::wstring wide;
-  utf8::utf8to32(str.begin(), str.end(), std::back_inserter(wide));
-#endif
-
-  return wide;
-}
 
 } // namespace antrlcpp