cmj cache

Author: cknitt

.github/workflows/ci.yml

@@ -123,8 +123,10 @@ jobs:
         if: runner.os == 'Linux'
         uses: actions/upload-artifact@v3
         with:
-          name: lib-cmi-cache
-          path: lib/cmi_cache.bin
+          name: cmij-cache
+          path: |
+            lib/cmi_cache.bin
+            lib/cmj_cache.bin
 
       - name: Check for changes in lib folder
         run: git diff --exit-code lib/js lib/es6

jscomp/cmij/cmij.ml

@@ -0,0 +1,8 @@
+type t = { module_names : string array; module_data : bytes array }
+type cmi_data = Cmi_format.cmi_infos
+type cmj_data = { values : Js_cmj_format.keyed_cmj_value array; pure : bool }
+
+let marshal_cmi_data (cmi_data : cmi_data) = Marshal.to_bytes cmi_data []
+let marshal_cmj_data (cmj_data : cmj_data) = Marshal.to_bytes cmj_data []
+let unmarshal_cmi_data bytes : cmi_data = Marshal.from_bytes bytes 0
+let unmarshal_cmj_data bytes : cmj_data = Marshal.from_bytes bytes 0

jscomp/cmij/cmij.mli

@@ -0,0 +1,8 @@
+type t = { module_names : string array; module_data : bytes array }
+type cmi_data = Cmi_format.cmi_infos
+type cmj_data = { values : Js_cmj_format.keyed_cmj_value array; pure : bool }
+
+val marshal_cmi_data : cmi_data -> bytes
+val marshal_cmj_data : cmj_data -> bytes
+val unmarshal_cmi_data : bytes -> cmi_data
+val unmarshal_cmj_data : bytes -> cmj_data

jscomp/cmij/cmij_cache.ml

@@ -30,32 +30,17 @@ let get_files ext dir =
       if Ext_string.ends_with x ext then Some (Filename.concat dir x) else None)
   |> Array.to_list
 
-(** the cache should be readable and also update *)
-let check_digest output_file digest : bool =
-  if Sys.file_exists output_file then
-    match
-      Ext_list.filter
-        (String.split_on_char ' ' (Ext_io.load_file output_file))
-        (fun x -> x <> "")
-    with
-    | _head :: old_digest :: _tail -> Digest.equal digest old_digest
-    | _ -> false
-  else false
-
-let ( +> ) = Ext_buffer.add_string
-
 let from_cmj ~mode (files : string list) (output_file : string) : unit =
   let files =
     let cmp = Ext_filename.module_name in
-    List.sort
-      (fun filea fileb -> Ext_string_array.cmp (cmp filea) (cmp fileb))
-      files
+    files
+    |> List.sort (fun filea fileb ->
+           Ext_string_array.cmp (cmp filea) (cmp fileb))
+    |> Array.of_list
   in
-  let buf = Ext_buffer.create 10000 in
-
-  let abs =
-    Ext_list.map files (fun file ->
-        let module_name = Ext_filename.module_name file in
+  let module_names = Ext_array.map files Ext_filename.module_name in
+  let module_data =
+    Ext_array.map files (fun file ->
         let content : Js_cmj_format.t = Js_cmj_format.from_file file in
         let () =
           match mode with
@@ -72,29 +57,13 @@ let from_cmj ~mode (files : string list) (output_file : string) : unit =
           | Playground _ -> ()
         in
         (* prerr_endline (Ext_obj.dump content.package_spec); *)
-        let c = Marshal.to_string (content.values, content.pure) [] in
-        ( Printf.sprintf {|%S (* %d *)|} module_name (String.length c),
-          Printf.sprintf {|(* %s *)%S|} module_name c ))
+        let { Js_cmj_format.values; pure } = content in
+        Cmij_cache.marshal_cmj_data { values; pure })
   in
-  buf
-  +> Printf.sprintf
-       {|let module_names : string array = Obj.magic (
-%s
-)
-let module_data : string array = Obj.magic (
-%s
-)
-|}
-       (String.concat ",\n" (Ext_list.map abs fst))
-       (String.concat ",\n" (Ext_list.map abs snd));
-  buf +> "\n";
-  let digest = Digest.to_hex (Ext_buffer.digest buf) in
-  let same = check_digest output_file digest in
-  if not same then
-    let v = open_out_bin output_file in
-    Ext_pervasives.finally v ~clean:close_out (fun f ->
-        output_string f ("(* " ^ digest ^ " *) \n");
-        Ext_buffer.output_buffer f buf)
+  let cmj_cache = { Cmij_cache.module_names; module_data } in
+  let v = open_out_bin output_file in
+  Ext_pervasives.finally v ~clean:close_out (fun f ->
+      Marshal.to_channel f cmj_cache [])
 
 let from_cmi files output_file =
   let files =
@@ -110,21 +79,19 @@ let from_cmi files output_file =
         let module_name = Ext_filename.module_name file in
         let cmi = Cmi_format.read_cmi file in
         assert (cmi.cmi_name = module_name);
-        let content = Marshal.to_bytes cmi [] in
-        content)
+        Cmij_cache.marshal_cmi_data cmi)
   in
-  let cmi_cache = { Cmij.module_names; module_data } in
+  let cmi_cache = { Cmij_cache.module_names; module_data } in
   let v = open_out_bin output_file in
   Ext_pervasives.finally v ~clean:close_out (fun f ->
       Marshal.to_channel f cmi_cache [])
 
 let stdlib = "stdlib-406"
 let ( // ) = Filename.concat
 let ( |~ ) = Ext_string.contain_substring
-
-let cmi_target_file =
-  Filename.dirname Sys.argv.(0) // ".." // ".." // "lib" // "cmi_cache.bin"
-
+let lib_dir = Filename.dirname Sys.argv.(0) // ".." // ".." // "lib"
+let cmi_target_file = lib_dir // "cmi_cache.bin"
+let cmj_target_file = lib_dir // "cmj_cache.bin"
 let release_cmi = Array.exists (( = ) "-release") Sys.argv
 
 let () =
@@ -158,9 +125,7 @@ let () =
         (Filename.dirname Sys.argv.(0) // ".." // "others")
     @ third_party_cmj_files
   in
-  from_cmj ~mode cmj_files
-    (Filename.dirname Sys.argv.(0)
-    // ".." // "core" // "builtin_cmj_datasets.ml");
+  from_cmj ~mode cmj_files cmj_target_file;
   let third_party_cmi_files =
     match mode with
     | Native -> []

jscomp/cmij/cmij_cache.mli

@@ -25,15 +25,13 @@
 #ifdef RELEASE 
 (*true *)
 
-type cmi_cache = {module_names : string array; module_data : bytes array}
-
 let cmi_cache_path =
   let ( // ) = Filename.concat in
   Config.standard_library // Filename.parent_dir_name // "cmi_cache.bin"
 
 let load_cmi_cache () =
   let channel = open_in_bin cmi_cache_path in
-  let cache: cmi_cache = Marshal.from_channel channel in
+  let cache: Cmij_cache.t = Marshal.from_channel channel in
   close_in channel;
   cache
 
@@ -45,13 +43,12 @@ let load_cmi ~unit_name : Env.Persistent_signature.t option =
   | None ->
     if !Js_config.no_stdlib then None
     else
-      let {module_names; module_data} = Lazy.force cmi_cache in
+      let {Cmij_cache.module_names; module_data} = Lazy.force cmi_cache in
       match Ext_string_array.find_sorted module_names unit_name with
-      | Some cmi ->
+      | Some index ->
         if Js_config.get_diagnose () then
           Format.fprintf Format.err_formatter ">Cmi: %s@." unit_name;
-        let cmi : Cmi_format.cmi_infos = 
-          Marshal.from_bytes module_data.(cmi) 0 in   
+        let cmi = Cmij_cache.unmarshal_cmi_data module_data.(index) in
         if Js_config.get_diagnose () then
           Format.fprintf Format.err_formatter "<Cmi: %s@." unit_name;
         Some {filename = Sys.executable_name ; 

jscomp/cmij/cmij_main.ml

@@ -11,3 +11,5 @@
 ; There exist cyclic dependencies between core and frontend, therefore include frontend files here:
 
 (copy_files# ../frontend/*.{ml,mli})
+
+(copy_files# ../cmij/cmij_cache.{ml,mli})