Build the whole compiler with dune

Author: cknitt

.gitattributes

@@ -25,6 +25,7 @@ jobs:
 
     env:
       OCAMLRUNPARAM: b
+      DUNE_PROFILE: release
 
     steps:
       - name: "Windows: Set git to use LF"
@@ -45,6 +46,22 @@ jobs:
           opam-pin: false
           opam-depext: false
 
+      - name: "Install OPAM dependencies"
+        run: opam install . --deps-only
+
+      - name: "Build compiler"
+        run: |
+          opam exec -- dune build
+          ./scripts/copyExes.js
+
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Install npm packages
+        run: npm ci --ignore-scripts
+
       - name: "Check if syntax subfolder has changed"
         id: syntax-diff
         shell: bash
@@ -55,25 +72,13 @@ jobs:
             echo "syntax_status=changed" >> $GITHUB_ENV
           fi
 
-      - name: "Syntax: install OPAM dependencies"
-        if: env.syntax_status == 'changed'
-        run: opam install . --deps-only
-        working-directory: res_syntax
-
-      - name: "Syntax: build CLI"
-        if: env.syntax_status == 'changed'
-        run: opam exec -- dune build
-        working-directory: res_syntax
-
       - name: "Syntax: Run roundtrip tests"
         if: ${{ env.syntax_status == 'changed' && runner.os != 'Windows' }}
-        run: opam exec -- make roundtrip-test
-        working-directory: res_syntax
+        run: opam exec -- make test-syntax-roundtrip
 
       - name: "Syntax: Run tests (Windows)"
         if: ${{ env.syntax_status == 'changed' && runner.os == 'Windows' }}
-        run: opam exec -- make test
-        working-directory: res_syntax
+        run: opam exec -- make test-syntax
 
       # Required for ninja build
       - name: "Windows: Use MSVC"
@@ -85,23 +90,22 @@ jobs:
       - name: Build ninja
         run: node scripts/buildNinjaBinary.js
 
-      - name: Use Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16
+      - name: Build runtime/stdlib
+        if: runner.os != 'Windows'
+        run: |
+          opam exec -- dune exec -- node ./scripts/ninja.js config
+          opam exec -- dune exec -- node ./scripts/ninja.js build
 
-      - name: NPM install
-        run: opam exec -- npm ci
-        env:
-          RESCRIPT_FORCE_REBUILD: 1
+      - name: Check for changes in lib folder
+        run: git diff --exit-code lib/js lib/es6
 
       - name: Run tests
         if: runner.os != 'Windows'
-        run: opam exec -- node scripts/ciTest.js -all
+        run: opam exec -- dune exec -- node scripts/ciTest.js -all
 
       - name: Run tests (Windows)
         if: runner.os == 'Windows'
-        run: opam exec -- node scripts/ciTest.js -mocha -theme -format
+        run: opam exec -- dune exec -- node scripts/ciTest.js -mocha -theme -format
 
       - name: Get artifact info
         run: node .github/workflows/get_artifact_info.js
@@ -119,19 +123,14 @@ jobs:
           name: lib-ocaml
           path: lib/ocaml
 
-      # Important: the ninja build must run after the artifact upload because for some reason
-      # the bsc.exe created by the ninja build does not find the Pervasives module.
-      # So this is run here just to test the normal dev workflow, too.
-      - name: ninja config
-        if: runner.os != 'Windows'
-        run: opam exec -- node ./scripts/ninja.js config
-
-      - name: ninja build
-        if: runner.os != 'Windows'
-        run: opam exec -- node ./scripts/ninja.js build
-
-      - name: Check for changes in lib folder
-        run: git diff --exit-code lib/js lib/es6
+      - name: "Upload artifacts: cmi cache"
+        if: runner.os == 'Linux'
+        uses: actions/upload-artifact@v3
+        with:
+          name: cmij-cache
+          path: |
+            lib/cmi_cache.bin
+            lib/cmj_cache.bin
 
   package:
     needs: build
@@ -157,9 +156,6 @@ jobs:
       - name: Move artifacts
         run: ./scripts/moveArtifacts.sh
 
-      - name: Create ninja.tar.gz
-        run: git -C ninja archive --format=tar.gz HEAD -o ../vendor/ninja.tar.gz
-
       - name: Check artifact list
         run: node ./scripts/makeArtifactList.js -check
 

.github/workflows/ci.yml

@@ -1,11 +1,8 @@
 const fs = require("fs");
 const os = require("os");
+const { binDirName } = require("../../scripts/bin_path");
 
-const artifactPath =
-  process.platform === "darwin" && process.arch === "arm64"
-    ? process.platform + process.arch
-    : process.platform;
-
+const artifactPath = binDirName;
 const artifactName = "binaries-" + artifactPath;
 
 // Pass artifactPath and artifactName to subsequent GitHub actions

.github/workflows/get_artifact_info.js

@@ -45,39 +45,12 @@ _build
 .DS_Store
 
 node_modules
-oxml
-test.json
-jscomp/pre_load.js
-boot
 *.dump
 coverage
-*.g.js
-ppx_tools/
-vendor/ocaml/stdlib/target_camlheader
-vendor/ocaml/stdlib/target_camlheaderd
-man/
-lib/ocaml
-*.tgz
-bin/*
-jscomp/bin/*.exe
-
-# for npm publish
 *.exe
 
-!bin/bspack
-!/bin/bsb
-!/bin/bsc
-
-# This is intentional
-# so that `npm install && npm intall -g .`
-# the second install would use the cached ninja.exe (instead of being ignored)
-#
-#ninja
 .ninja_log
-.ninja_deps
 .bsdeps
-.bs_dir_cache
-
 .bsbuild
 lib/prebuilt.ninja
 lib/release.ninja
@@ -96,50 +69,20 @@ jscomp/cppoVendor.ninja
 jscomp/cppoEnv.ninja
 jscomp/bin/bsb
 jscomp/bin/bsc
+lib/ocaml
 lib/bs
 lib/js/jscomp/test
 jscomp/build_tests/*/lib/
 #ignore temporary directory
 *.goog.js
 *.jsoo.js
 *.so
-bin/ninja.exe
 tmp/
-jscomp/ext/hash_set.ml
-jscomp/ext/hash_set_poly.ml
-jscomp/ext/hash.ml
-jscomp/ext/hash_set_ident.ml
-jscomp/ext/hash_ident.ml
-jscomp/ext/map_ident.ml
-jscomp/ext/set_ident.ml
-jscomp/ext/hash_set_int.ml
-jscomp/ext/hash_int.ml
-jscomp/ext/map_int.ml
-jscomp/ext/vec_int.ml
-jscomp/ext/ordered_hash_map_local_ident.ml
-jscomp/ext/ordered_hash_set_string.ml
-jscomp/ext/vec.ml
-jscomp/ext/set_int.ml
-jscomp/ext/hash_set_string.ml
-jscomp/ext/hash_string.ml
-jscomp/ext/map_string.ml
-jscomp/ext/set_string.ml
-jscomp/ext/ext_json_parse.ml
-jscomp/syntax/reactjs_jsx_ppx_v2.ml
-jscomp/syntax/reactjs_jsx_ppx_v3.ml
-jscomp/bin/all_ounit_tests.ml
-native/
-vendor/ocaml
 jscomp/.lsp
-.vscode/launch.json
 tracing*.json
-darwin/bsc
 .nyc_output
-themes
+.vscode/launch.json
 .vscode/settings.json
-*.output # yacc output
-jscomp/ml/parser.output
-jscomp/ml/lexer.ml
 
 # opam local switch is stored in `_opam` folder
 _opam

.gitignore

@@ -3,6 +3,7 @@ lib/
 vendor/
 ninja/
 res_syntax/
+_build/
 _opam
 CHANGELOG.md
 README.md

.prettierignore

@@ -1,32 +1,58 @@
 SHELL = /bin/bash
 
-config:
-	./scripts/ninja.js config
-
-build: config
-	./scripts/ninja.js build
+build:
 	dune build
+	./scripts/copyExes.js
 
-watch: config
+watch:
 	dune build -w
 
-dce: build
+bench:
+	dune exec -- syntax_benchmarks
+
+dce:
 	opam exec reanalyze.exe -- -dce-cmt _build
 
+ninja/ninja:
+	./scripts/buildNinjaBinary.js
+
 test: build
-	npm test
+	dune exec -- node scripts/ciTest.js -all
+
+test-syntax: build
+	dune exec -- syntax_tests
+	dune exec -- bash ./scripts/test_syntax.sh
+#	make reanalyze
+	bash ./scripts/testok.sh
+
+test-syntax-roundtrip: build
+	dune exec -- syntax_tests
+	ROUNDTRIP_TEST=1 dune exec -- bash ./scripts/test.sh
+#	make reanalyze
+	bash ./scripts/testok.sh
 
 test-gentype: build
 	make -C jscomp/gentype_tests/typescript-react-example test
 
 test-all: test test-gentype
 
-lib: build
-	node scripts/install -force-lib-rebuild
+# reanalyze:
+# 	reanalyze.exe -set-exit-code -all-cmt _build/default -suppress syntax_tests,syntax_compiler_libs -exclude-paths syntax_compiler_libs 
+
+lib: build ninja/ninja
+	dune exec -- node scripts/ninja.js config
+	dune exec -- node scripts/ninja.js build
 
-artifacts: lib
+package: lib
+	./scripts/prebuilt.js
 	./scripts/makeArtifactList.js
 
+# format:
+# 	dune build @fmt --auto-promote
+
+# checkformat:
+# 	dune build @fmt
+
 clean-gentype:
 	make -C jscomp/gentype_tests/typescript-react-example clean
 
@@ -38,4 +64,4 @@ clean-all: clean clean-gentype
 
 .DEFAULT_GOAL := build
 
-.PHONY: build clean clean-gentype clean-all config lib test test-all test-gentype
+.PHONY: build watch bench dce test test-syntax test-syntax-roundtrip test-gentype test-all lib artifacts clean-gentype clean clean-all

Makefile

@@ -103,9 +103,6 @@ ReScript is licensed under LGPL version 3, with relaxed rules about creating and
 
 The ReScript parser (subdirectory `res_syntax`) is licensed under the [MIT License](res_syntax/LICENSE).
 
-`vendor/ocaml.tar.gz` contains the official [OCaml](https://ocaml.org) compiler (version 4.14.0).
-Refer to its copyright and license notices for information about its licensing.
-
 The `ninja` subdirectory contains the vendored [ninja](https://github.com/ninja-build/ninja) build system.
 Refer to its copyright and license notices for information about its licensing.
 

README.md

@@ -22,14 +22,19 @@ Required:
 - OCaml 4.10 or later
 - OS: macOS, Linux or Windows
 
-The syntax source code is located in the directory `res_syntax`. All following commands are meant to be executed in that directory.
+In the root of the rescript-compiler repo, run
 
 ```sh
 opam install . --deps-only --with-test
+```
+
+To build the syntax sources, run
+
+```sh
 make # or "dune build"
 ```
 
-This will produce the three binaries `rescript`, `tests` and `bench` (with `.exe` extension on Windows).
+This will produce the three binaries `res_parser`, `syntax_tests` and `syntax_benchmarks` (with `.exe` extension on Windows).
 
 We only build production binaries, even in dev mode. No need for a separate dev binary when the build is fast enough. Plus, this encourages proper benchmarking of the (production) binary each diff.
 

docs/Syntax.md

@@ -1,3 +1,31 @@
-(lang dune 2.0)
- (name rescript) 
- 
+(lang dune 2.3)
+
+(name rescript)
+
+(generate_opam_files true)
+
+(license LGPL-3.0-or-later)
+
+(authors "Hongbo Zhang <[email protected]>")
+
+(maintainers "Hongbo Zhang <[email protected]>" "Cristiano Calcagno")
+
+(homepage "https://github.com/rescript-lang/rescript-compiler")
+
+(bug_reports "https://github.com/rescript-lang/rescript-compiler/issues")
+
+(package
+ (name rescript)
+ (synopsis "ReScript compiler")
+ (depends
+  (ocaml
+   (>= 4.10))
+  (ocamlformat
+   (= 0.22.4))
+  (cppo
+   (= 1.6.9))
+  (js_of_ocaml-compiler
+   (= 4.0.0))
+  (reanalyze
+   (= 2.23.0))
+  dune))