pivot_root: Final adjustments and reconciliations

.github/workflows/build.yml

@@ -72,7 +72,7 @@ jobs:
             os: ubuntu-latest
             build_ocamlparam: ''
             ocamlparam: '_,Oclassic=1'
-            disable_testcases: 'ocaml/testsuite/tests/typing-local/regression_cmm_unboxing.ml ocaml/testsuite/tests/int64-unboxing/test.ml'
+            disable_testcases: 'testsuite/tests/typing-local/regression_cmm_unboxing.ml testsuite/tests/int64-unboxing/test.ml'
 
           # CR ccasinghino: We encountered build errors (missing autoconf, and
           # then other errors) when `macos-latest` started referring to version

.github/workflows/jane_ocaml5.yml

@@ -73,6 +73,7 @@ jobs:
       - name: Configure
         working-directory: ocaml-jst
         run: |
+          autoconf
           ./configure --prefix=$GITHUB_WORKSPACE/ocaml-5/_install --with-dune=$GITHUB_WORKSPACE/ocaml-414/_install/bin/dune
 
       - name: Check that resolved files build

Makefile

@@ -1,20 +1,19 @@
 SHELL = /usr/bin/env bash
 include Makefile.config
-include ocaml/Makefile.config
 export ARCH
 
-boot_ocamlc = ocaml/main_native.exe
+boot_ocamlc = main_native.exe
 boot_ocamlopt = boot_ocamlopt.exe
-boot_ocamlmklib = ocaml/tools/ocamlmklib.exe
-boot_ocamldep = ocaml/tools/ocamldep.exe
-boot_ocamlobjinfo = tools/flambda_backend_objinfo.exe
-ocamldir = ocaml
+boot_ocamlmklib = tools/ocamlmklib.exe
+boot_ocamldep = tools/ocamldep.exe
+boot_ocamlobjinfo = tools/objinfo.exe
+ocamldir = .
 toplevels_installed = top opttop
 
 $(ocamldir)/duneconf/jst-extra.inc:
 	echo > $@
 
-include ocaml/Makefile.common-jst
+include Makefile.common-jst
 
 .PHONY: ci
 ifeq ($(coverage),yes)

Makefile.common-jst

@@ -61,7 +61,7 @@ boot_targets = \
   $(boot_ocamlmklib) \
   $(boot_ocamldep) \
   $(boot_ocamlobjinfo) \
-  ocaml/ocamltest/ocamltest.native
+  ocamltest/ocamltest.native
 
 boot-compiler: _build/_bootinstall
 	RUNTIME_DIR=$(RUNTIME_DIR) $(dune) build $(ws_boot) $(coverage_dune_flags) $(boot_targets)
@@ -83,11 +83,11 @@ compiler: runtime-stdlib
           ASPP="$(ASPP)" ASPPFLAGS="$(ASPPFLAGS)" \
           $(dune) build $(ws_main) \
           --only-package=ocaml @install \
-          testsuite/tools/expect.exe \
-          testsuite/tools/codegen_main.exe \
-          testsuite/tools/asmgen_$(ARCH).o \
-          ocaml/testsuite/lib/lib.cm{,x}a \
-          ocaml/testsuite/lib/testing.cm{,x}a \
+          flambda-backend/testsuite/tools/expect.exe \
+          flambda-backend/testsuite/tools/codegen_main.exe \
+          flambda-backend/testsuite/tools/asmgen_$(ARCH).o \
+          testsuite/lib/lib.cm{,x}a \
+          testsuite/lib/testing.cm{,x}a \
           $(ocamldir)/tools/dumpobj.bc
 
 runtest: compiler
@@ -194,11 +194,8 @@ _install: compiler
 	for file in $(toplevels_installed); do \
 	  cp -f _install/lib/ocaml/compiler-libs/$${file}dirs.{cmi,mli,cmt,cmti} _install/lib/ocaml; \
 	done
-	shopt -s nullglob; for file in _build/main/$(ocamldir)/.ocaml{bytecomp,common,middleend,optcomp}.objs/{byte/*.{cmi,cma,cmt,cmti},native/*.{cmx,cmxa,cmxs}}; do \
-	  $(cpl) $$file _install/lib/ocaml/compiler-libs/ ; \
-	done
 	mkdir _install/lib/stublibs
-	cp _build/main/ocaml/parser.cmly _install/lib/ocaml/compiler-libs/
+	cp _build/main/parser.cmly _install/lib/ocaml/compiler-libs/
 	find _build/main/ \( -name "flambda2*.cmi" \
           -or -name "flambda2*.cmti" -or -name "flambda2*.cmt" \) \
           -exec cp -f {} _install/lib/ocaml/compiler-libs \;
@@ -227,38 +224,38 @@ install_for_test: _install
 	$(cpl) -a $(ocamldir)/testsuite _runtest/testsuite
 	 # replace backend-specific testsuite/tools with their new versions
 	rm -f _runtest/testsuite/tools/*
-	cp -a testsuite/tools/* _runtest/testsuite/tools/
-	cp $(main_build)/testsuite/tools/expect.exe _runtest/testsuite/tools/expect
+	cp -a flambda-backend/testsuite/tools/* _runtest/testsuite/tools/
+	cp $(main_build)/flambda-backend/testsuite/tools/expect.exe _runtest/testsuite/tools/expect
 	chmod 755 _runtest/testsuite/tools/expect
-	cp $(main_build)/testsuite/tools/codegen_main.exe \
+	cp $(main_build)/flambda-backend/testsuite/tools/codegen_main.exe \
           _runtest/testsuite/tools/codegen
 	chmod 755 _runtest/testsuite/tools/codegen
-	cp $(main_build)/testsuite/tools/asmgen_*.o \
+	cp $(main_build)/flambda-backend/testsuite/tools/asmgen_*.o \
           _runtest/testsuite/tools/
-	cp $(main_build)/ocaml/testsuite/lib/lib.{a,cmxa,cma} \
+	cp $(main_build)/testsuite/lib/lib.{a,cmxa,cma} \
           _runtest/testsuite/lib/
-	cp $(main_build)/ocaml/testsuite/lib/.lib.objs/byte/lib.cm* \
+	cp $(main_build)/testsuite/lib/.lib.objs/byte/lib.cm* \
           _runtest/testsuite/lib/
-	cp $(main_build)/ocaml/testsuite/lib/.lib.objs/native/lib.cm* \
+	cp $(main_build)/testsuite/lib/.lib.objs/native/lib.cm* \
           _runtest/testsuite/lib/
-	cp $(main_build)/ocaml/testsuite/lib/testing.{a,cmxa,cma} \
+	cp $(main_build)/testsuite/lib/testing.{a,cmxa,cma} \
           _runtest/testsuite/lib/
-	cp $(main_build)/ocaml/testsuite/lib/.testing.objs/byte/testing.cm* \
+	cp $(main_build)/testsuite/lib/.testing.objs/byte/testing.cm* \
           _runtest/testsuite/lib/
-	cp $(main_build)/ocaml/testsuite/lib/.testing.objs/native/testing.cm* \
+	cp $(main_build)/testsuite/lib/.testing.objs/native/testing.cm* \
           _runtest/testsuite/lib/
 	 # replace backend-specific testsuite/tests/asmcomp with their new versions
 	rm _runtest/testsuite/tests/asmcomp/*
-	cp -a testsuite/tests/asmcomp/* _runtest/testsuite/tests/asmcomp/
+	cp -a flambda-backend/testsuite/tests/asmcomp/* _runtest/testsuite/tests/asmcomp/
 	 # replace backend-specific testsuite/tests/asmgen with their new versions
 	rm _runtest/testsuite/tests/asmgen/*
-	cp -a testsuite/tests/asmgen/* _runtest/testsuite/tests/asmgen/
+	cp -a flambda-backend/testsuite/tests/asmgen/* _runtest/testsuite/tests/asmgen/
 	 # replace backend-specific testsuite/tests/unboxed-primitive-args with their new versions
 	rm _runtest/testsuite/tests/unboxed-primitive-args/*
-	cp -a testsuite/tests/unboxed-primitive-args/* _runtest/testsuite/tests/unboxed-primitive-args/
+	cp -a flambda-backend/testsuite/tests/unboxed-primitive-args/* _runtest/testsuite/tests/unboxed-primitive-args/
 	 # add extension library tests that are not supported by the upstream compiler
 	rm _runtest/testsuite/tests/lib-extensions/*
-	cp -a testsuite/tests/lib-extensions/* _runtest/testsuite/tests/lib-extensions
+	cp -a flambda-backend/testsuite/tests/lib-extensions/* _runtest/testsuite/tests/lib-extensions
 
 	cp $(ocamldir)/Makefile.* _runtest/
 
@@ -335,11 +332,11 @@ install_for_test: _install
 	# cp $(main_build)/$(ocamldir)/ocamldoc/.odoc_lib.objs/byte/*.cm* _runtest/ocamldoc
 	# ocamltest itself
 	mkdir _runtest/ocamltest
-	cp $(boot_build)/ocaml/ocamltest/ocamltest.native _runtest/ocamltest/ocamltest
+	cp $(boot_build)/ocamltest/ocamltest.native _runtest/ocamltest/ocamltest
 test: install_for_test
 	if [ "$(middle_end)" = "flambda2" ]; then \
-	  for dir in `cd ocaml/testsuite; ls -1 -d tests/*`; do \
-	    if ! grep -q "^  $$dir " testsuite/flambda2-test-list; then \
+	  for dir in `cd testsuite; ls -1 -d tests/*`; do \
+	    if ! grep -q "^  $$dir " flambda-backend/testsuite/flambda2-test-list; then \
 	      echo "  $$dir"; \
 	    fi; \
 	  done > _runtest/flambda2-test-list; \

backend/dune

@@ -35,4 +35,4 @@
      (with-stdout-to %{targets}
        (progn
          (bash "echo \\# 1 \\\"`cat contains-input-name`\\\"")
-         (bash "%{dep:../ocaml/tools/cvt_emit.exe} < `cat contains-input-name`"))))))
+         (bash "%{dep:../tools/cvt_emit.exe} < `cat contains-input-name`"))))))

flambda-backend/tests/backend/frame-too-long/t.expected

@@ -1 +1 @@
-i=10000 name=Raised at T.break in file "tests/backend/frame-too-long/t.ml", line 5, characters 2-15
+i=10000 name=Raised at T.break in file "flambda-backend/tests/backend/frame-too-long/t.ml", line 5, characters 2-15

flambda-backend/tests/simd/dune

@@ -12,7 +12,7 @@
  (language c)
  (names stubs)
  (flags -msse4.2)
- (include_dirs "../../ocaml/%{env:RUNTIME_DIR=runtime-dir-env-var-not-set}"))
+ (include_dirs "../../../%{env:RUNTIME_DIR=runtime-dir-env-var-not-set}"))
 
 ; Tests with external assembler
 

flambda-backend/tests/small_numbers/dune

@@ -5,7 +5,7 @@
  (language c)
  (names stubs)
  (flags -msse4.2)
- (include_dirs "../../ocaml/%{env:RUNTIME_DIR=runtime-dir-env-var-not-set}"))
+ (include_dirs "../../../%{env:RUNTIME_DIR=runtime-dir-env-var-not-set}"))
 
 ; Tests with external assembler
 

jane/build-resolved-files-for-ci

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/..
 
 set -euo pipefail
 
@@ -15,29 +15,29 @@ mlis=$(
   { # echo driver/{compenv,compmisc,main_args}.mli
     # echo file_formats/{cmi,cmo,cms,cmt}_format.mli
     # echo {utils,typing,lambda}/*.mli
-    echo ocaml/{parsing,typing,lambda}/*.mli
+    echo {parsing,typing,lambda}/*.mli
   } |
     tr ' ' '\n'
 )
 echo "$mlis"
 dune_targets=$(
   for mli in $mlis; do
     cmi=$(basename ${mli%.mli})
-    echo _build/default/ocaml/.ocamlcommon.objs/byte/$cmi.cmi
+    echo _build/default/.ocamlcommon.objs/byte/$cmi.cmi
   done
 )
 
 # ocamlcommon mls
 mls=$(
-  { echo ocaml/parsing/*.ml
-    echo ocaml/typing/*.ml
+  { echo parsing/*.ml
+    echo typing/*.ml
   } | tr ' ' '\n'
 )
 echo "$mls"
 dune_targets=$(
   for ml in $mls; do
     cmx=$(basename ${ml%.ml})
-    echo _build/default/ocaml/.ocamlcommon.objs/native/$cmx.cmx
+    echo _build/default/.ocamlcommon.objs/native/$cmx.cmx
   done
 )
 

runtime/caml/dune

@@ -28,14 +28,14 @@
  (deps
   instruct.h
   ../../stdlib/StdlibModules
-  ../../Makefile
+  ../../Makefile.upstream
   ../../Makefile.common
   ../../Makefile.config
   ../../Makefile.build_config
   ../../Makefile.config_if_required
   ../../Makefile.best_binaries)
  (action
-  (run make -s -C ../.. COMPUTE_DEPS=false runtime/caml/opnames.h)))
+  (run make -s -f Makefile.upstream -C ../.. COMPUTE_DEPS=false runtime/caml/opnames.h)))
 
 (rule
  (targets version.h)

runtime/dune

@@ -16,7 +16,7 @@
 
 (rule
   (targets sak)
-  (deps ../Makefile ../.depend ../Makefile.menhir ../Makefile.best_binaries
+  (deps ../Makefile.upstream ../.depend ../Makefile.menhir ../Makefile.best_binaries
     ../stdlib/StdlibModules ../.depend.menhir ../config.status
     ../Makefile.common ../Makefile.config_if_required
     ../Makefile.build_config ../Makefile.config sak.c
@@ -25,18 +25,18 @@
   (action
     (progn
      (bash "rm -f sak")
-     (bash "cd .. && make V=1 'SAK_LINK=cc -o $(1) $(2)' COMPUTE_DEPS=false runtime/sak")
+     (bash "cd .. && make -f Makefile.upstream V=1 'SAK_LINK=cc -o $(1) $(2)' COMPUTE_DEPS=false runtime/sak")
     )
   ))
 
 (rule
   (targets build_config.h)
   (mode fallback)
-  (deps sak ../Makefile)
+  (deps sak ../Makefile.upstream)
   (action
     (progn
      (bash "rm -f build_config.h")
-     (bash "cd .. && make V=1 runtime/build_config.h")
+     (bash "cd .. && make V=1 -f Makefile.upstream COMPUTE_DEPS=false runtime/build_config.h")
     )
   ))
 
@@ -69,7 +69,7 @@
    ../Makefile.config
    ../Makefile.build_config
    ../Makefile.config_if_required
-   ../Makefile.common ../Makefile
+   ../Makefile.common ../Makefile.upstream
    ../Makefile.menhir
    ../.depend
    ../.depend.menhir
@@ -89,7 +89,7 @@
    (progn
      (bash "touch .depend") ; hack.
      (bash "rm -f sak build_config.h") ; hack
-     (bash "cd .. && make V=1 -j8 runtime/libasmrun.a runtime/libasmrund.a runtime/libasmruni.a runtime/libasmrun_pic.a \
+     (bash "cd .. && make V=1 -j8 -f Makefile.upstream runtime/libasmrun.a runtime/libasmrund.a runtime/libasmruni.a runtime/libasmrun_pic.a \
     runtime/libasmrun_shared.so runtime/libcamlrun.a runtime/libcamlrund.a runtime/libcamlruni.a runtime/libcamlrun_pic.a \
     runtime/libcamlrun_shared.so runtime/ocamlrun runtime/ocamlrund runtime/ocamlruni runtime/ld.conf COMPUTE_DEPS=false")
      (bash "rm .depend")))))

testsuite/tests/lib-dynlink-initializers/test10_main.byte.reference

@@ -5,8 +5,8 @@ Called from Test10_plugin.f in file "test10_plugin.ml", line 6, characters 2-6
 Called from Test10_plugin in file "test10_plugin.ml", line 10, characters 2-6
 Called from Dynlink_internal_byte.Bytecode.run in file "otherlibs/dynlink/dynlink.ml", line 179, characters 16-25
 Re-raised at Dynlink_internal_byte.Bytecode.run in file "otherlibs/dynlink/dynlink.ml", lines 181-183, characters 6-137
-Called from Dynlink_common.Make.load.(fun) in file "ocaml/otherlibs/dynlink/dynlink_common.ml", line 382, characters 13-56
+Called from Dynlink_common.Make.load.(fun) in file "otherlibs/dynlink/dynlink_common.ml", line 382, characters 13-56
 Called from Stdlib__List.iter in file "list.ml", line 117, characters 12-15
-Called from Dynlink_common.Make.load in file "ocaml/otherlibs/dynlink/dynlink_common.ml", lines 378-387, characters 8-392
-Re-raised at Dynlink_common.Make.load in file "ocaml/otherlibs/dynlink/dynlink_common.ml", line 391, characters 8-17
+Called from Dynlink_common.Make.load in file "otherlibs/dynlink/dynlink_common.ml", lines 378-387, characters 8-392
+Re-raised at Dynlink_common.Make.load in file "otherlibs/dynlink/dynlink_common.ml", line 391, characters 8-17
 Called from Test10_main in file "test10_main.ml", lines 51-53, characters 13-69

testsuite/tests/tool-toplevel/pr6468.compilers.reference

@@ -8,21 +8,21 @@ Warning 21 [nonreturning-statement]: this statement never returns (or has an uns
 val g : unit -> int = <fun>
 Exception: Not_found.
 Raised at Stdlib__Map.Make.find in file "map.ml", line 146, characters 10-25
-Called from Env.find_type_data in file "ocaml/typing/env.ml", line 1287, characters 8-48
-Re-raised at Ident.find_same in file "ocaml/typing/ident.ml", line 307, characters 6-21
-Called from Env.IdTbl.find_same_without_locks in file "ocaml/typing/env.ml", line 432, characters 10-40
+Called from Env.find_type_data in file "typing/env.ml", line 1287, characters 8-48
+Re-raised at Ident.find_same in file "typing/ident.ml", line 307, characters 6-21
+Called from Env.IdTbl.find_same_without_locks in file "typing/env.ml", line 432, characters 10-40
 Re-raised at Stdlib__Map.Make.find in file "map.ml", line 146, characters 10-25
-Called from Env.find_type_data in file "ocaml/typing/env.ml", line 1287, characters 8-48
-Re-raised at Ident.find_same in file "ocaml/typing/ident.ml", line 307, characters 6-21
-Called from Env.IdTbl.find_same_without_locks in file "ocaml/typing/env.ml", line 432, characters 10-40
+Called from Env.find_type_data in file "typing/env.ml", line 1287, characters 8-48
+Re-raised at Ident.find_same in file "typing/ident.ml", line 307, characters 6-21
+Called from Env.IdTbl.find_same_without_locks in file "typing/env.ml", line 432, characters 10-40
 Re-raised at Stdlib__Map.Make.find in file "map.ml", line 146, characters 10-25
-Called from Env.find_type_data in file "ocaml/typing/env.ml", line 1287, characters 8-48
-Re-raised at Ident.find_same in file "ocaml/typing/ident.ml", line 307, characters 6-21
-Called from Env.IdTbl.find_same_without_locks in file "ocaml/typing/env.ml", line 432, characters 10-40
-Re-raised at Ident.find_same in file "ocaml/typing/ident.ml", line 307, characters 6-21
-Called from Translmod.toplevel_name in file "ocaml/lambda/translmod.ml", line 1599, characters 6-40
+Called from Env.find_type_data in file "typing/env.ml", line 1287, characters 8-48
+Re-raised at Ident.find_same in file "typing/ident.ml", line 307, characters 6-21
+Called from Env.IdTbl.find_same_without_locks in file "typing/env.ml", line 432, characters 10-40
+Re-raised at Ident.find_same in file "typing/ident.ml", line 307, characters 6-21
+Called from Translmod.toplevel_name in file "lambda/translmod.ml", line 1599, characters 6-40
 Re-raised at Stdlib__Hashtbl.find in file "hashtbl.ml", line 547, characters 13-28
-Called from Simplif.simplify_lets.simplif in file "ocaml/lambda/simplif.ml", line 571, characters 8-28
+Called from Simplif.simplify_lets.simplif in file "lambda/simplif.ml", line 571, characters 8-28
 Re-raised at f in file "//toplevel//", line 2, characters 11-26
 Called from g in file "//toplevel//", line 1, characters 11-15
 Called from <unknown> in file "//toplevel//", line 1, characters 0-4

tools/gen_compiler_libs_installation.ml

@@ -22,6 +22,17 @@ let () =
       | "zero_alloc_checker", ".mli" -> Some "mach_checks"
       | "cSE", (".cmi" | ".cmt" | ".cmx") -> Some "CSE"
       | "cSEgen", (".cmi" | ".cmt" | ".cmti" | ".cmx") -> Some "CSEgen"
+      | ( "allowance" | "cmt2annot" | "compression" | "debug" | "diffing"
+        | "diffing_with_keys" | "errortrace" | "file_sections" | "global_module"
+        | "import_info" | "includemod_errorprinter" | "jane_syntax_parsing"
+        | "language_extension_kernel" | "lazy_backtrack" | "lru" | "main"
+        | "main_native" | "maindriver" | "mode_intf" | "optmain" | "parser.pp"
+        | "parser_types" | "shape" | "shape_reduce" | "signature_group"
+        | "solver_intf" | "tmc" | "transl_array_comprehension"
+        | "transl_comprehension_utils" | "transl_list_comprehension"
+        | "unit_info" | "value_rec_types" ), ".mli" ->
+        (* Preserve pre-pivot_root behaviour by leaving these out *)
+        None
       | _, _ -> Some prefix
     in
     let tgt_dir_prefix =

toplevel/byte/dune

@@ -25,7 +25,7 @@
  (targets bytetop)
  ; This should be generated from the stdlib modules files
  (action (run %{ocaml_where}/../../bin/ocamlrun 
-              -I ../../../middle_end/flambda2/numbers/floats
+              -I ../../middle_end/flambda2/numbers/floats
               %{exe:../expunge.bc} %{dep:../topstart.bc} %{targets}
                     stdlib__Arg
                     stdlib__Array

toplevel/native/dune

@@ -24,22 +24,22 @@
    flambda_backend_common
    flambda2_ui
    flambda2 dynlink_internal unix)
- (modules genprintval_native opttoploop opttopdirs opttopmain))
+ (modules genprintval_native opttoploop opttopdirs topmain))
 
 (rule
   (targets genprintval_native.ml)
-  (deps %{project_root}/ocaml/toplevel/genprintval.ml)
+  (deps ../genprintval.ml)
   (action (copy %{deps} %{targets})))
 
 (executable
- (name opttopstart)
+ (name topstart)
  (modes native)
  (libraries ocamlopttoplevel)
- (modules opttopstart))
+ (modules topstart))
 
 (install
   (files
-    (opttopstart.exe as ocamlnat)
+    (topstart.exe as ocamlnat)
   )
   (section bin)
   (package ocaml))
@@ -56,7 +56,7 @@
     (.ocamlopttoplevel.objs/byte/opttoploop.cmi as compiler-libs/opttoploop.cmi)
     (.ocamlopttoplevel.objs/byte/opttoploop.cmt as compiler-libs/opttoploop.cmt)
     (.ocamlopttoplevel.objs/byte/opttoploop.cmti as compiler-libs/opttoploop.cmti)
-    (opttopmain.mli as compiler-libs/opttopmain.mli)
+    (topmain.mli as compiler-libs/opttopmain.mli)
   )
   (section lib)
   (package ocaml))