diff --git a/CHANGES.md b/CHANGES.md index 95391b7f1d..2d8533cddc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +### unreleased + +#### Internal + + + Use ppxlib instead of ocaml-migrate-parsetree 1.x. (#1482, @emillon) + ### 0.15.0 (2020-08-06) #### Changes diff --git a/lib/Cmts.ml b/lib/Cmts.ml index 7d7bbae8ea..7bb75bf26a 100644 --- a/lib/Cmts.ml +++ b/lib/Cmts.ml @@ -13,7 +13,6 @@ module Format = Format_ open Migrate_ast -open Asttypes open Parsetree type t = @@ -42,48 +41,6 @@ let find_at_position t loc pos = in Location.Multimap.find map loc -module Loc_tree : sig - include Non_overlapping_interval_tree.S with type itv = Location.t - - val of_ast : 'a Mapper.fragment -> 'a -> Source.t -> t * Location.t list -end = struct - include Non_overlapping_interval_tree.Make (Location) - - (* Use Ast_mapper to collect all locs in ast, and create tree of them. *) - - let of_ast fragment ast src = - let attribute (m : Ast_mapper.mapper) attr = - (* ignore location of docstrings *) - if Ast.Attr.is_doc attr then attr - else Ast_mapper.default_mapper.attribute m attr - in - let locs = ref [] in - let location _ loc = - locs := loc :: !locs ; - loc - in - let pat m p = - ( match p.ppat_desc with - | Ppat_record (flds, Open) -> - Option.iter (Source.loc_of_underscore src flds p.ppat_loc) - ~f:(fun loc -> locs := loc :: !locs) - | Ppat_constant _ -> locs := Source.loc_of_pat_constant src p :: !locs - | _ -> () ) ; - Ast_mapper.default_mapper.pat m p - in - let expr m e = - ( match e.pexp_desc with - | Pexp_constant _ -> locs := Source.loc_of_expr_constant src e :: !locs - | _ -> () ) ; - Ast_mapper.default_mapper.expr m e - in - let mapper = - Ast_mapper.{default_mapper with location; pat; attribute; expr} - in - Mapper.map_ast fragment mapper ast |> ignore ; - (of_list !locs, !locs) -end - (** Sets of comments supporting splitting by locations. *) module CmtSet : sig type t @@ -311,29 +268,27 @@ let init fragment ~debug source asts comments_n_docstrings = match locs with | [] -> add_cmts t `After ~prev:Location.none Location.none cmts | _ -> place t loc_tree locs cmts ) ; - let () = - let relocate_loc_stack loc stack = - List.iter stack ~f:(fun src -> relocate t ~src ~before:loc ~after:loc) - in - let expr (m : Ast_mapper.mapper) x = - relocate_loc_stack x.pexp_loc x.pexp_loc_stack ; - Ast_mapper.default_mapper.expr m x - in - let typ (m : Ast_mapper.mapper) x = - relocate_loc_stack x.ptyp_loc x.ptyp_loc_stack ; - Ast_mapper.default_mapper.typ m x - in - let pat (m : Ast_mapper.mapper) x = - relocate_loc_stack x.ppat_loc x.ppat_loc_stack ; - Ast_mapper.default_mapper.pat m x - in - let _ = - Mapper.map_ast fragment - Ast_mapper.{default_mapper with pat; typ; expr} - asts - in - () + let relocate_loc_stack loc stack = + List.iter stack ~f:(fun src -> relocate t ~src ~before:loc ~after:loc) + in + let mapper = + object + inherit Ppxlib.Ast_traverse.map as super + + method! pattern x = + relocate_loc_stack x.ppat_loc x.ppat_loc_stack ; + super#pattern x + + method! core_type x = + relocate_loc_stack x.ptyp_loc x.ptyp_loc_stack ; + super#core_type x + + method! expression x = + relocate_loc_stack x.pexp_loc x.pexp_loc_stack ; + super#expression x + end in + let _ = Mapper.map_ast fragment mapper asts in t let preserve fmt_x t = diff --git a/lib/Loc_tree.ml b/lib/Loc_tree.ml new file mode 100644 index 0000000000..55e6532dfe --- /dev/null +++ b/lib/Loc_tree.ml @@ -0,0 +1,51 @@ +(**************************************************************************) +(* *) +(* OCamlFormat *) +(* *) +(* Copyright (c) Facebook, Inc. and its affiliates. *) +(* *) +(* This source code is licensed under the MIT license found in *) +(* the LICENSE file in the root directory of this source tree. *) +(* *) +(**************************************************************************) + +open Migrate_ast +open Parsetree +include Non_overlapping_interval_tree.Make (Location) + +(** Use Ast_mapper to collect all locs in ast, and create tree of them. *) +let of_ast fragment ast src = + let locs = ref [] in + let add_loc loc = locs := loc :: !locs in + let mapper = + object + inherit Ppxlib.Ast_traverse.map as super + + method! location loc = add_loc loc ; loc + + method! pattern p = + ( match p.ppat_desc with + | Ppat_record (flds, Open) -> + Option.iter + (Source.loc_of_underscore src flds p.ppat_loc) + ~f:add_loc + | _ -> () ) ; + super#pattern p + + method! attribute attr = + (* ignore location of docstrings *) + if Ast.Attr.is_doc attr then attr else super#attribute attr + + (** Ast_traverse recurses down to locations in stacks *) + method! location_stack l = l + + method! expression e = + ( match e.pexp_desc with + | Pexp_constant _ -> + locs := Source.loc_of_expr_constant src e :: !locs + | _ -> () ) ; + super#expression e + end + in + Mapper.map_ast fragment mapper ast |> ignore ; + (of_list !locs, !locs) diff --git a/lib/Migrate_ast.ml b/lib/Migrate_ast.ml index fcee994771..85d698e0c3 100644 --- a/lib/Migrate_ast.ml +++ b/lib/Migrate_ast.ml @@ -9,14 +9,10 @@ (* *) (**************************************************************************) -let selected_version = Migrate_parsetree.Versions.ocaml_411 - -module Selected_version = Ast_411 -module Ast_mapper = Selected_version.Ast_mapper -module Ast_helper = Selected_version.Ast_helper +module Ast_helper = Ppxlib.Ast_helper module Parsetree = struct - include Selected_version.Parsetree + include Ppxlib.Parsetree let equal_core_type : core_type -> core_type -> bool = Poly.equal @@ -29,7 +25,7 @@ module Parsetree = struct end module Asttypes = struct - include Selected_version.Asttypes + include Ppxlib.Asttypes let is_private = function Private -> true | Public -> false @@ -41,30 +37,6 @@ module Asttypes = struct end module Mapper = struct - let structure = Selected_version.map_structure - - let signature = Selected_version.map_signature - - (* Missing from ocaml_migrate_parsetree *) - let use_file (mapper : Ast_mapper.mapper) use_file = - let open Parsetree in - List.map use_file ~f:(fun toplevel_phrase -> - match (toplevel_phrase : toplevel_phrase) with - | Ptop_def structure -> - Ptop_def (mapper.Ast_mapper.structure mapper structure) - | Ptop_dir {pdir_name; pdir_arg; pdir_loc} -> - let pdir_arg = - match pdir_arg with - | None -> None - | Some a -> - Some {a with pdira_loc= mapper.location mapper a.pdira_loc} - in - Ptop_dir - { pdir_name= - {pdir_name with loc= mapper.location mapper pdir_name.loc} - ; pdir_arg - ; pdir_loc= mapper.location mapper pdir_loc }) - type 'a fragment = | Structure : Parsetree.structure fragment | Signature : Parsetree.signature fragment @@ -76,22 +48,21 @@ module Mapper = struct | Signature -> Parsetree.equal_signature | Use_file -> List.equal Parsetree.equal_toplevel_phrase - let map_ast (type a) (x : a fragment) : Ast_mapper.mapper -> a -> a = + let map_ast (type a) (x : a fragment) (m : Ppxlib.Ast_traverse.map) : + a -> a = match x with - | Structure -> structure - | Signature -> signature - | Use_file -> use_file + | Structure -> m#structure + | Signature -> m#signature + | Use_file -> m#list m#toplevel_phrase end module Parse = struct - open Migrate_parsetree - - let implementation = Parse.implementation selected_version + let implementation = Ppxlib_ast.Parse.implementation - let interface = Parse.interface selected_version + let interface = Ppxlib_ast.Parse.interface let use_file lexbuf = - List.filter (Parse.use_file selected_version lexbuf) + List.filter (Ppxlib_ast.Parse.use_file lexbuf) ~f:(fun (p : Parsetree.toplevel_phrase) -> match p with | Ptop_def [] -> false @@ -104,32 +75,20 @@ module Parse = struct | Mapper.Use_file -> use_file lexbuf end -let to_current = - Migrate_parsetree.Versions.(migrate selected_version ocaml_current) - module Printast = struct - open Printast + let pp_sexp ppf sexp = Format.fprintf ppf "%a" (Sexp.pp_hum_indent 2) sexp + + let sexp_of = Ppxlib.Ast_traverse.sexp_of - let implementation f x = implementation f (to_current.copy_structure x) + let implementation ppf x = pp_sexp ppf (sexp_of#structure x) - let interface f x = interface f (to_current.copy_signature x) + let interface ppf x = pp_sexp ppf (sexp_of#signature x) - let expression f x = expression 0 f (to_current.copy_expression x) + let expression ppf x = pp_sexp ppf (sexp_of#expression x) - let payload f (x : Parsetree.payload) = - payload 0 f - ( match x with - | PStr x -> PStr (to_current.copy_structure x) - | PSig x -> PSig (to_current.copy_signature x) - | PTyp x -> PTyp (to_current.copy_core_type x) - | PPat (x, y) -> - PPat - ( to_current.copy_pattern x - , Option.map ~f:to_current.copy_expression y ) ) + let payload ppf x = pp_sexp ppf (sexp_of#payload x) - let use_file f (x : Parsetree.toplevel_phrase list) = - List.iter x ~f:(fun (p : Parsetree.toplevel_phrase) -> - top_phrase f (to_current.copy_toplevel_phrase p)) + let use_file ppf x = pp_sexp ppf (List.sexp_of_t sexp_of#toplevel_phrase x) let fragment (type a) : a Mapper.fragment -> _ -> a -> _ = function | Mapper.Structure -> implementation @@ -137,22 +96,7 @@ module Printast = struct | Mapper.Use_file -> use_file end -module Pprintast = struct - open Pprintast - - let structure f x = structure f (to_current.copy_structure x) - - let signature f x = signature f (to_current.copy_signature x) - - let core_type f x = core_type f (to_current.copy_core_type x) - - let expression f x = expression f (to_current.copy_expression x) - - let pattern f x = pattern f (to_current.copy_pattern x) - - let toplevel_phrase f x = - toplevel_phrase f (to_current.copy_toplevel_phrase x) -end +module Pprintast = Ppxlib.Pprintast module Position = struct open Lexing @@ -180,7 +124,7 @@ module Position = struct end module Location = struct - include Selected_version.Location + include Ppxlib.Location let fmt fs {loc_start; loc_end; loc_ghost} = Format.fprintf fs "(%a..%a)%s" Position.fmt loc_start Position.fmt diff --git a/lib/Migrate_ast.mli b/lib/Migrate_ast.mli index 7fcd309ef3..6f4b0e10f7 100644 --- a/lib/Migrate_ast.mli +++ b/lib/Migrate_ast.mli @@ -9,16 +9,10 @@ (* *) (**************************************************************************) -val selected_version : - Migrate_parsetree.Versions.OCaml_411.types - Migrate_parsetree.Versions.ocaml_version - -module Selected_version = Ast_411 -module Ast_mapper = Selected_version.Ast_mapper -module Ast_helper = Selected_version.Ast_helper +module Ast_helper = Ppxlib.Ast_helper module Parsetree : sig - include module type of Selected_version.Parsetree + include module type of Ppxlib.Parsetree val equal_core_type : core_type -> core_type -> bool @@ -30,7 +24,7 @@ module Parsetree : sig end module Asttypes : sig - include module type of Selected_version.Asttypes + include module type of Ppxlib.Asttypes val is_private : private_flag -> bool @@ -52,7 +46,7 @@ module Position : sig end module Location : sig - include module type of Selected_version.Location + include module type of Ppxlib.Location type comparator_witness @@ -139,7 +133,7 @@ module Mapper : sig val equal : 'a fragment -> 'a -> 'a -> bool - val map_ast : 'a fragment -> Ast_mapper.mapper -> 'a -> 'a + val map_ast : 'a fragment -> Ppxlib.Ast_traverse.map -> 'a -> 'a end module Parse : sig @@ -160,19 +154,7 @@ module Printast : sig val fragment : 'a Mapper.fragment -> Format.formatter -> 'a -> unit end -module Pprintast : sig - val core_type : Format.formatter -> Parsetree.core_type -> unit - - val pattern : Format.formatter -> Parsetree.pattern -> unit - - val toplevel_phrase : Format.formatter -> Parsetree.toplevel_phrase -> unit - - val expression : Format.formatter -> Parsetree.expression -> unit - - val structure : Format.formatter -> Parsetree.structure -> unit - - val signature : Format.formatter -> Parsetree.signature -> unit -end +module Pprintast = Ppxlib.Pprintast module Longident : sig type t = Longident.t = diff --git a/lib/Normalize.ml b/lib/Normalize.ml index 1029ab7a65..2083b764fe 100644 --- a/lib/Normalize.ml +++ b/lib/Normalize.ml @@ -25,26 +25,30 @@ type conf = let dedup_cmts fragment ast comments = let of_ast ast = let docs = ref (Set.empty (module Cmt)) in - let attribute m atr = - match atr with - | { attr_name= {txt= "ocaml.doc" | "ocaml.text"; _} - ; attr_payload= - PStr - [ { pstr_desc= - Pstr_eval - ( { pexp_desc= - Pexp_constant (Pconst_string (doc, _, None)) - ; pexp_loc - ; _ } - , [] ) - ; _ } ] - ; _ } -> - docs := Set.add !docs (Cmt.create ("*" ^ doc) pexp_loc) ; - atr - | _ -> Ast_mapper.default_mapper.attribute m atr + let mapper = + object + inherit Ppxlib.Ast_traverse.map as super + + method! attribute atr = + match atr with + | { attr_name= {txt= "ocaml.doc" | "ocaml.text"; _} + ; attr_payload= + PStr + [ { pstr_desc= + Pstr_eval + ( { pexp_desc= + Pexp_constant (Pconst_string (doc, _, None)) + ; pexp_loc + ; _ } + , [] ) + ; _ } ] + ; _ } -> + docs := Set.add !docs (Cmt.create ("*" ^ doc) pexp_loc) ; + atr + | _ -> super#attribute atr + end in - Mapper.map_ast fragment {Ast_mapper.default_mapper with attribute} ast - |> ignore ; + Mapper.map_ast fragment mapper ast |> ignore ; !docs in Set.(to_list (diff (of_list (module Cmt) comments) (of_ast ast))) @@ -179,189 +183,182 @@ let sort_attributes : attributes -> attributes = List.sort ~compare:Poly.compare let make_mapper conf ~ignore_doc_comments = - (* remove locations *) - let location _ _ = Location.none in let doc_attribute = function | {attr_name= {txt= "ocaml.doc" | "ocaml.text"; _}; _} -> true | _ -> false in - let attribute (m : Ast_mapper.mapper) (attr : attribute) = - match (attr.attr_name, attr.attr_payload) with - | ( {txt= ("ocaml.doc" | "ocaml.text") as txt; loc} - , PStr - [ { pstr_desc= - Pstr_eval - ( { pexp_desc= Pexp_constant (Pconst_string (doc, _, None)) - ; pexp_loc - ; pexp_attributes - ; _ } - , [] ) - ; pstr_loc } ] ) -> - let doc' = - if ignore_doc_comments then "IGNORED" - else - let c = {conf; normalize_code= m.structure m} in - docstring c doc - in - { attr_name= {txt; loc= m.location m loc} - ; attr_payload= - m.payload m - (PStr - [ { pstr_desc= - Pstr_eval - ( { pexp_desc= - Pexp_constant - (Pconst_string (doc', loc, None)) - ; pexp_loc= m.location m pexp_loc - ; pexp_attributes= m.attributes m pexp_attributes - ; pexp_loc_stack= [] } - , [] ) - ; pstr_loc= m.location m pstr_loc } ]) - ; attr_loc= m.location m attr.attr_loc } - | _ -> Ast_mapper.default_mapper.attribute m attr - in - (* sort attributes *) - let attributes (m : Ast_mapper.mapper) (atrs : attribute list) = - let atrs = - if ignore_doc_comments then - List.filter atrs ~f:(fun a -> not (doc_attribute a)) - else atrs - in - Ast_mapper.default_mapper.attributes m (sort_attributes atrs) - in - let expr (m : Ast_mapper.mapper) exp = - let exp = {exp with pexp_loc_stack= []} in - let {pexp_desc; pexp_attributes; _} = exp in - match pexp_desc with - (* convert [(c1; c2); c3] to [c1; (c2; c3)] *) - | Pexp_sequence - ({pexp_desc= Pexp_sequence (e1, e2); pexp_attributes= []; _}, e3) -> - m.expr m - (Exp.sequence e1 (Exp.sequence ~attrs:pexp_attributes e2 e3)) - | Pexp_poly ({pexp_desc= Pexp_constraint (e, t); _}, None) -> - m.expr m {exp with pexp_desc= Pexp_poly (e, Some t)} - | Pexp_constraint (e, {ptyp_desc= Ptyp_poly ([], _t); _}) -> m.expr m e - | _ -> Ast_mapper.default_mapper.expr m exp - in - let pat (m : Ast_mapper.mapper) pat = - let pat = {pat with ppat_loc_stack= []} in - let {ppat_desc; ppat_loc= loc1; ppat_attributes= attrs1; _} = pat in - (* normalize nested or patterns *) - match ppat_desc with - | Ppat_or - ( pat1 - , { ppat_desc= Ppat_or (pat2, pat3) - ; ppat_loc= loc2 - ; ppat_attributes= attrs2 - ; _ } ) -> - m.pat m - (Pat.or_ ~loc:loc1 ~attrs:attrs1 - (Pat.or_ ~loc:loc2 ~attrs:attrs2 pat1 pat2) - pat3) - | _ -> Ast_mapper.default_mapper.pat m pat - in - let typ (m : Ast_mapper.mapper) typ = - let typ = {typ with ptyp_loc_stack= []} in - Ast_mapper.default_mapper.typ m typ - in - let value_binding (m : Ast_mapper.mapper) vb = - let { pvb_pat= {ppat_desc; ppat_loc; ppat_attributes; _} - ; pvb_expr - ; pvb_loc - ; pvb_attributes } = - vb - in - match (ppat_desc, pvb_expr.pexp_desc) with - (* recognize and undo the pattern of code introduced by - ocaml/ocaml@fd0dc6a0fbf73323c37a73ea7e8ffc150059d6ff to fix - https://caml.inria.fr/mantis/view.php?id=7344 *) - | ( Ppat_constraint - ( ({ppat_desc= Ppat_var _; _} as p0) - , {ptyp_desc= Ptyp_poly ([], t0); _} ) - , Pexp_constraint (e0, t1) ) - when equal_core_type t0 t1 -> - m.value_binding m - (Vb.mk ~loc:pvb_loc ~attrs:pvb_attributes p0 - (Exp.constraint_ ~loc:ppat_loc ~attrs:ppat_attributes e0 t0)) - (* convert [let (x : t) = e] to [let x = (e : t)] *) - | Ppat_constraint (p0, t0), _ -> - m.value_binding m - (Vb.mk ~loc:pvb_loc ~attrs:pvb_attributes p0 - (Exp.constraint_ ~loc:ppat_loc ~attrs:ppat_attributes pvb_expr - t0)) - | _ -> Ast_mapper.default_mapper.value_binding m vb - in - let structure_item (m : Ast_mapper.mapper) (si : structure_item) = - match si.pstr_desc with - | Pstr_eval ({pexp_desc= Pexp_extension e; _}, []) -> - let e = m.extension m e in - let pstr_loc = m.location m si.pstr_loc in - {pstr_desc= Pstr_extension (e, []); pstr_loc} - | _ -> Ast_mapper.default_mapper.structure_item m si - in - let structure (m : Ast_mapper.mapper) (si : structure) = - let si = - if ignore_doc_comments then - List.filter si ~f:(fun si -> - match si.pstr_desc with - | Pstr_attribute a -> not (doc_attribute a) - | _ -> true) - else si - in - Ast_mapper.default_mapper.structure m si - in - let signature (m : Ast_mapper.mapper) (si : signature) = - let si = - if ignore_doc_comments then - List.filter si ~f:(fun si -> - match si.psig_desc with - | Psig_attribute a -> not (doc_attribute a) - | _ -> true) - else si - in - Ast_mapper.default_mapper.signature m si - in - let class_signature (m : Ast_mapper.mapper) (si : class_signature) = - let si = - if ignore_doc_comments then - let pcsig_fields = - List.filter si.pcsig_fields ~f:(fun si -> - match si.pctf_desc with - | Pctf_attribute a -> not (doc_attribute a) + object (self) + inherit Ppxlib.Ast_traverse.map as super + + (** Remove locations *) + method! location _ = Location.none + + method! attribute attr = + match (attr.attr_name, attr.attr_payload) with + | ( {txt= ("ocaml.doc" | "ocaml.text") as txt; loc} + , PStr + [ { pstr_desc= + Pstr_eval + ( { pexp_desc= + Pexp_constant (Pconst_string (doc, str_loc, None)) + ; pexp_loc + ; pexp_attributes + ; _ } + , [] ) + ; pstr_loc } ] ) -> + let doc' = + if ignore_doc_comments then "IGNORED" + else + let c = {conf; normalize_code= self#structure} in + docstring c doc + in + { attr_name= {txt; loc= self#location loc} + ; attr_payload= + self#payload + (PStr + [ { pstr_desc= + Pstr_eval + ( { pexp_desc= + Pexp_constant + (Pconst_string + (doc', self#location str_loc, None)) + ; pexp_loc= self#location pexp_loc + ; pexp_attributes= + self#attributes pexp_attributes + ; pexp_loc_stack= [] } + , [] ) + ; pstr_loc= self#location pstr_loc } ]) + ; attr_loc= self#location attr.attr_loc } + | _ -> super#attribute attr + + (** Sort attributes *) + method! attributes atrs = + let atrs = + if ignore_doc_comments then + List.filter atrs ~f:(fun a -> not (doc_attribute a)) + else atrs + in + super#attributes (sort_attributes atrs) + + method! expression exp = + let exp = {exp with pexp_loc_stack= []} in + let {pexp_desc; pexp_attributes; _} = exp in + match pexp_desc with + (* convert [(c1; c2); c3] to [c1; (c2; c3)] *) + | Pexp_sequence + ({pexp_desc= Pexp_sequence (e1, e2); pexp_attributes= []; _}, e3) + -> + self#expression + (Exp.sequence e1 (Exp.sequence ~attrs:pexp_attributes e2 e3)) + | Pexp_poly ({pexp_desc= Pexp_constraint (e, t); _}, None) -> + self#expression {exp with pexp_desc= Pexp_poly (e, Some t)} + | Pexp_constraint (e, {ptyp_desc= Ptyp_poly ([], _t); _}) -> + self#expression e + | _ -> super#expression exp + + method! location_stack _ = [] + + method! pattern pat = + let {ppat_desc; ppat_loc= loc1; ppat_attributes= attrs1; _} = pat in + (* normalize nested or patterns *) + match ppat_desc with + | Ppat_or + ( pat1 + , { ppat_desc= Ppat_or (pat2, pat3) + ; ppat_loc= loc2 + ; ppat_attributes= attrs2 + ; _ } ) -> + self#pattern + (Pat.or_ ~loc:loc1 ~attrs:attrs1 + (Pat.or_ ~loc:loc2 ~attrs:attrs2 pat1 pat2) + pat3) + | _ -> super#pattern pat + + method! value_binding vb = + let { pvb_pat= {ppat_desc; ppat_loc; ppat_attributes; _} + ; pvb_expr + ; pvb_loc + ; pvb_attributes } = + vb + in + match (ppat_desc, pvb_expr.pexp_desc) with + (* recognize and undo the pattern of code introduced by + ocaml/ocaml@fd0dc6a0fbf73323c37a73ea7e8ffc150059d6ff to fix + https://caml.inria.fr/mantis/view.php?id=7344 *) + | ( Ppat_constraint + ( ({ppat_desc= Ppat_var _; _} as p0) + , {ptyp_desc= Ptyp_poly ([], t0); _} ) + , Pexp_constraint (e0, t1) ) + when equal_core_type t0 t1 -> + self#value_binding + (Vb.mk ~loc:pvb_loc ~attrs:pvb_attributes p0 + (Exp.constraint_ ~loc:ppat_loc ~attrs:ppat_attributes e0 t0)) + (* convert [let (x : t) = e] to [let x = (e : t)] *) + | Ppat_constraint (p0, t0), _ -> + self#value_binding + (Vb.mk ~loc:pvb_loc ~attrs:pvb_attributes p0 + (Exp.constraint_ ~loc:ppat_loc ~attrs:ppat_attributes pvb_expr + t0)) + | _ -> super#value_binding vb + + method! structure_item si = + match si.pstr_desc with + | Pstr_eval ({pexp_desc= Pexp_extension e; _}, []) -> + let e = self#extension e in + let pstr_loc = self#location si.pstr_loc in + {pstr_desc= Pstr_extension (e, []); pstr_loc} + | _ -> super#structure_item si + + method! structure si = + let si = + if ignore_doc_comments then + List.filter si ~f:(fun si -> + match si.pstr_desc with + | Pstr_attribute a -> not (doc_attribute a) | _ -> true) - in - {si with pcsig_fields} - else si - in - Ast_mapper.default_mapper.class_signature m si - in - let class_structure (m : Ast_mapper.mapper) (si : class_structure) = - let si = - if ignore_doc_comments then - let pcstr_fields = - List.filter si.pcstr_fields ~f:(fun si -> - match si.pcf_desc with - | Pcf_attribute a -> not (doc_attribute a) + else si + in + super#structure si + + method! signature si = + let si = + if ignore_doc_comments then + List.filter si ~f:(fun si -> + match si.psig_desc with + | Psig_attribute a -> not (doc_attribute a) | _ -> true) - in - {si with pcstr_fields} - else si - in - Ast_mapper.default_mapper.class_structure m si - in - { Ast_mapper.default_mapper with - location - ; attribute - ; attributes - ; expr - ; pat - ; typ - ; value_binding - ; structure_item - ; signature - ; structure - ; class_signature - ; class_structure } + else si + in + super#signature si + + method! class_signature si = + let si = + if ignore_doc_comments then + let pcsig_fields = + List.filter si.pcsig_fields ~f:(fun si -> + match si.pctf_desc with + | Pctf_attribute a -> not (doc_attribute a) + | _ -> true) + in + {si with pcsig_fields} + else si + in + super#class_signature si + + method! class_structure si = + let si = + if ignore_doc_comments then + let pcstr_fields = + List.filter si.pcstr_fields ~f:(fun si -> + match si.pcf_desc with + | Pcf_attribute a -> not (doc_attribute a) + | _ -> true) + in + {si with pcstr_fields} + else si + in + super#class_structure si + end let normalize fragment c = Mapper.map_ast fragment (make_mapper c ~ignore_doc_comments:false) @@ -375,43 +372,45 @@ let make_docstring_mapper c docstrings = | {attr_name= {txt= "ocaml.doc" | "ocaml.text"; _}; _} -> true | _ -> false in - let attribute (m : Ast_mapper.mapper) attr = - match (attr.attr_name, attr.attr_payload) with - | ( {txt= ("ocaml.doc" | "ocaml.text") as txt; loc} - , PStr - [ { pstr_desc= - Pstr_eval - ( { pexp_desc= Pexp_constant (Pconst_string (doc, _, None)) - ; pexp_loc - ; pexp_attributes - ; _ } - , [] ) - ; pstr_loc } ] ) -> - let doc' = docstring c doc in - docstrings := (loc, doc) :: !docstrings ; - { attr_name= {txt; loc} - ; attr_payload= - m.payload m - (PStr - [ { pstr_desc= - Pstr_eval - ( { pexp_desc= - Pexp_constant - (Pconst_string (doc', loc, None)) - ; pexp_loc - ; pexp_attributes - ; pexp_loc_stack= [] } - , [] ) - ; pstr_loc } ]) - ; attr_loc= attr.attr_loc } - | _ -> Ast_mapper.default_mapper.attribute m attr - in - (* sort attributes *) - let attributes (m : Ast_mapper.mapper) atrs = - let atrs = List.filter atrs ~f:doc_attribute in - Ast_mapper.default_mapper.attributes m (sort_attributes atrs) - in - {Ast_mapper.default_mapper with attribute; attributes} + object + inherit Ppxlib.Ast_traverse.map as super + + method! attribute attr = + match (attr.attr_name, attr.attr_payload) with + | ( {txt= ("ocaml.doc" | "ocaml.text") as txt; loc} + , PStr + [ { pstr_desc= + Pstr_eval + ( { pexp_desc= + Pexp_constant (Pconst_string (doc, str_loc, None)) + ; pexp_loc + ; pexp_attributes + ; _ } + , [] ) + ; pstr_loc } ] ) -> + let doc' = docstring c doc in + docstrings := (loc, doc) :: !docstrings ; + { attr_name= {txt; loc} + ; attr_payload= + super#payload + (PStr + [ { pstr_desc= + Pstr_eval + ( { pexp_desc= + Pexp_constant + (Pconst_string (doc', str_loc, None)) + ; pexp_loc + ; pexp_attributes + ; pexp_loc_stack= [] } + , [] ) + ; pstr_loc } ]) + ; attr_loc= attr.attr_loc } + | _ -> super#attribute attr + + method! attributes atrs = + let atrs = List.filter atrs ~f:doc_attribute in + super#attributes (sort_attributes atrs) + end let docstrings (type a) (fragment : a Mapper.fragment) c s = let docstrings = ref [] in diff --git a/lib/Source.ml b/lib/Source.ml index ab0f1b1ee8..53b6df6ca2 100644 --- a/lib/Source.ml +++ b/lib/Source.ml @@ -106,7 +106,7 @@ let tokens_between t ~filter loc_start loc_end = match Lexer.token lexbuf with | Parser.EOF -> List.rev acc | tok -> - if filter tok then loop ((tok, Location.curr lexbuf) :: acc) + if filter tok then loop ((tok, Location.of_lexbuf lexbuf) :: acc) else loop acc in loop [] @@ -129,7 +129,7 @@ let find_after t f (loc : Location.t) = let rec loop () = match Lexer.token lexbuf with | Parser.EOF -> None - | tok -> if f tok then Some (Location.curr lexbuf) else loop () + | tok -> if f tok then Some (Location.of_lexbuf lexbuf) else loop () in loop () diff --git a/lib/Sugar.ml b/lib/Sugar.ml index 8d0d4e6cfd..00fb8cb278 100644 --- a/lib/Sugar.ml +++ b/lib/Sugar.ml @@ -310,7 +310,7 @@ let rec functor_type cmts ~for_functor_kw ~source_is_long | [] -> functor_type cmts ~for_functor_kw ~source_is_long body | _ -> ([], body) in - (Location.mkloc functor_arg pmty_loc :: xargs, xbody) + (Ppxlib.Loc.make ~loc:pmty_loc functor_arg :: xargs, xbody) | _ -> ([], xmty) (* The sugar is different when used with the [functor] keyword. The syntax @@ -333,7 +333,7 @@ let rec functor_ cmts ~for_functor_kw ~source_is_long ({ast= me; _} as xme) = | [] -> functor_ cmts ~for_functor_kw ~source_is_long body | _ -> ([], body) in - (Location.mkloc functor_arg pmod_loc :: xargs, xbody_me) + (Ppxlib.Loc.make ~loc:pmod_loc functor_arg :: xargs, xbody_me) | _ -> ([], xme) let mod_with pmty = diff --git a/lib/Translation_unit.ml b/lib/Translation_unit.ml index c2de4ef477..c6f368f3db 100644 --- a/lib/Translation_unit.ml +++ b/lib/Translation_unit.ml @@ -150,21 +150,21 @@ let print_error ?(fmt = Format.err_formatter) ~debug ~quiet ~input_name error "%!@{%a@}:@,\ @{Error@}: Docstring (** %s *) added.\n\ %!" - Location.print_loc loc_after (ellipsis_cmt msg) + Location.print loc_after (ellipsis_cmt msg) else if Location.compare loc_after Location.none = 0 then Format.fprintf fmt "%!@{%a@}:@,\ @{Error@}: Docstring (** %s *) dropped.\n\ %!" - Location.print_loc loc_before (ellipsis_cmt msg) + Location.print loc_before (ellipsis_cmt msg) else Format.fprintf fmt "%!@{%a@}:@,\ @{Error@}: Docstring (** %s *) moved to \ @{%a@}.\n\ %!" - Location.print_loc loc_before (ellipsis_cmt msg) - Location.print_loc loc_after + Location.print loc_before (ellipsis_cmt msg) + Location.print loc_after | Normalize.Unstable (loc, s) -> Format.fprintf fmt "%!@{%a@}:@,\ @@ -174,14 +174,14 @@ let print_error ?(fmt = Format.err_formatter) ~debug ~quiet ~input_name error source or disable the formatting using the option \ --no-parse-docstrings.\n\ %!" - Location.print_loc loc (ellipsis_cmt s)) + Location.print loc (ellipsis_cmt s)) | `Comment_dropped l when not quiet -> List.iter l ~f:(fun Cmt.{txt= msg; loc} -> Format.fprintf fmt "%!@{%a@}:@,\ @{Error@}: Comment (* %s *) dropped.\n\ %!" - Location.print_loc loc (ellipsis_cmt msg)) + Location.print loc (ellipsis_cmt msg)) | `Cannot_parse ((Syntaxerr.Error _ | Lexer.Error _) as exn) -> if debug then Location.report_exception fmt exn | `Warning50 l -> @@ -199,7 +199,7 @@ let check_all_locations fmt cmts_t = match Cmts.remaining_locs cmts_t with | [] -> () | l -> - let print l = Format.fprintf fmt "%a\n%!" Location.print_loc l in + let print l = Format.fprintf fmt "%a\n%!" Location.print l in Format.fprintf fmt "Warning: Some locations have not been considered\n%!" ; List.iter ~f:print (List.sort l ~compare:Location.compare) @@ -247,7 +247,7 @@ let format fragment ?output_file ~input_name ~source ~parsed conf opts = Some (dump_formatted ~input_name ?output_file ~suffix fmted) else None in - Location.input_name := input_name ; + Ocaml_common.Location.input_name := input_name ; (* iterate until formatting stabilizes *) let rec print_check ~i ~(conf : Conf.t) t ~source = let format ~box_debug = @@ -392,7 +392,7 @@ let parse_result fragment conf (opts : Conf.opts) ~source ~input_name = | parsed -> Ok parsed let parse_and_format fragment ?output_file ~input_name ~source conf opts = - Location.input_name := input_name ; + Ocaml_common.Location.input_name := input_name ; let open Result.Monad_infix in parse_result fragment conf opts ~source ~input_name >>= fun parsed -> diff --git a/lib/dune b/lib/dune index 26523eb0b5..7066c3913a 100644 --- a/lib/dune +++ b/lib/dune @@ -24,4 +24,4 @@ (:standard -open Base -open Import -open Compat)) ;;INSERT_BISECT_HERE;; (libraries format_ import ocaml-migrate-parsetree odoc.model odoc.parser - parse_wyc re uuseg uuseg.string token_latest compat)) + parse_wyc re uuseg uuseg.string token_latest compat ppxlib)) diff --git a/ocamlformat.opam b/ocamlformat.opam index 25ea7a7e38..55e19d1a66 100644 --- a/ocamlformat.opam +++ b/ocamlformat.opam @@ -31,9 +31,9 @@ depends: [ "fix" "fpath" "menhir" - "ocaml-migrate-parsetree" {>= "1.7.3"} "ocp-indent" {with-test} "odoc" {>= "1.4.2"} + "ppxlib" {>= "0.18.0"} "re" "stdio" {< "v0.15"} "uuseg" {>= "10.0.0"} diff --git a/vendor/parse-wyc/lib/annot.ml b/vendor/parse-wyc/lib/annot.ml index 21e712c438..e3f5560c26 100644 --- a/vendor/parse-wyc/lib/annot.ml +++ b/vendor/parse-wyc/lib/annot.ml @@ -1,50 +1,36 @@ open Ast_helper open Migrate_ast.Parsetree -module type Annotated = sig - type t - - val mk : unit -> t - - val is_generated : t -> bool -end - module Ext = struct let mk () = (Location.mkloc "merlin.hole" !default_loc, PStr []) let is_generated = function - | (({ txt = "merlin.hole"; _ }, PStr []) : extension) -> true + | (({ txt = "merlin.hole"; _ }, PStr []) : Ppxlib.extension) -> true | _ -> false end module Exp = struct - type t = expression - let mk () = Exp.extension (Ext.mk ()) - let is_generated e = + let is_generated (e : Ppxlib.expression) = match e.pexp_desc with | Pexp_extension ext when Ext.is_generated ext -> true | _ -> false end module Attr = struct - type t = attribute - let mk () = Attr.mk { txt = "merlin.hole.gen"; loc = Location.none } (PStr []) - let is_generated a = + let is_generated (a : Ppxlib.attribute) = match (a.attr_name.txt, a.attr_payload) with | "merlin.hole.gen", PStr [] -> true | _ -> false end module Class_exp = struct - type t = class_expr - let mk () = Cl.extension (Ext.mk ()) - let is_generated e = + let is_generated (e : Ppxlib.class_expr) = match e.pcl_desc with | Pcl_extension ext when Ext.is_generated ext -> true | _ -> false diff --git a/vendor/parse-wyc/lib/annot.mli b/vendor/parse-wyc/lib/annot.mli index eb400ba85f..4151782c48 100644 --- a/vendor/parse-wyc/lib/annot.mli +++ b/vendor/parse-wyc/lib/annot.mli @@ -1,15 +1,17 @@ -module type Annotated = sig - type t +module Exp : sig + val mk : unit -> Migrate_ast.Parsetree.expression - val mk : unit -> t - - val is_generated : t -> bool + val is_generated : Ppxlib.Parsetree.expression -> bool end -open Migrate_ast.Parsetree +module Attr : sig + val mk : unit -> Migrate_ast.Parsetree.attribute -module Exp : Annotated with type t = expression + val is_generated : Ppxlib.Parsetree.attribute -> bool +end -module Attr : Annotated with type t = attribute +module Class_exp : sig + val mk : unit -> Migrate_ast.Parsetree.class_expr -module Class_exp : Annotated with type t = class_expr + val is_generated : Ppxlib.Parsetree.class_expr -> bool +end diff --git a/vendor/parse-wyc/lib/dune b/vendor/parse-wyc/lib/dune index 4141129e80..555034fa56 100644 --- a/vendor/parse-wyc/lib/dune +++ b/vendor/parse-wyc/lib/dune @@ -1,6 +1,6 @@ (library (name parse_wyc) - (libraries menhirLib ocaml-migrate-parsetree) + (libraries menhirLib ocaml-migrate-parsetree ppxlib) (modules_without_implementation let_binding)) (ocamllex lexer) diff --git a/vendor/parse-wyc/lib/migrate_ast.ml b/vendor/parse-wyc/lib/migrate_ast.ml index 771a8e6487..884a4fba5f 100644 --- a/vendor/parse-wyc/lib/migrate_ast.ml +++ b/vendor/parse-wyc/lib/migrate_ast.ml @@ -1,47 +1,36 @@ module Selected_version = Migrate_parsetree.Ast_408 -module Ast_mapper = Selected_version.Ast_mapper module Parsetree = Selected_version.Parsetree module Asttypes = Selected_version.Asttypes module Mapper = struct - let structure = Selected_version.map_structure - - let signature = Selected_version.map_signature - - (* Missing from ocaml_migrate_parsetree *) - let use_file (mapper : Ast_mapper.mapper) use_file = - let open Parsetree in - List.map - (fun toplevel_phrase -> - match (toplevel_phrase : toplevel_phrase) with - | Ptop_def structure -> - Ptop_def (mapper.Ast_mapper.structure mapper structure) - | Ptop_dir { pdir_name; pdir_arg; pdir_loc } -> - let pdir_arg = - match pdir_arg with - | None -> None - | Some a -> - Some { a with pdira_loc = mapper.location mapper a.pdira_loc } - in - Ptop_dir - { - pdir_name = - { pdir_name with loc = mapper.location mapper pdir_name.loc }; - pdir_arg; - pdir_loc = mapper.location mapper pdir_loc; - }) - use_file - - type 'a fragment = - | Structure : Parsetree.structure fragment - | Signature : Parsetree.signature fragment - | Use_file : Parsetree.toplevel_phrase list fragment - - let map_ast (type a) (x : a fragment) : Ast_mapper.mapper -> a -> a = - match x with - | Structure -> structure - | Signature -> signature - | Use_file -> use_file + type ('omp, 'ppxlib) fragment = + | Structure + : ( Selected_version.Parsetree.structure, + Ppxlib.Parsetree.structure ) + fragment + | Signature + : ( Selected_version.Parsetree.signature, + Ppxlib.Parsetree.signature ) + fragment + | Use_file + : ( Selected_version.Parsetree.toplevel_phrase list, + Ppxlib.Parsetree.toplevel_phrase list ) + fragment + + let iter_ast (type o p) (fragment : (o, p) fragment) + (m : Ppxlib.Ast_traverse.iter) (x : p) = + match fragment with + | Structure -> m#structure x + | Signature -> m#signature x + | Use_file -> List.iter m#toplevel_phrase x + + let to_ppxlib (type o p) (f : (o, p) fragment) : o -> p = + let module Conv = Ppxlib_ast.Select_ast (Ppxlib_ast__.Versions.OCaml_408) in + let module To_ppxlib = Conv.Of_ocaml in + match f with + | Structure -> To_ppxlib.copy_structure + | Signature -> To_ppxlib.copy_signature + | Use_file -> List.map To_ppxlib.copy_toplevel_phrase end module Int = struct @@ -55,15 +44,9 @@ module Position = struct end module Location = struct - include Selected_version.Location - - let compare_start x y = Position.compare x.loc_start y.loc_start - - let compare_end x y = Position.compare x.loc_end y.loc_end + include Ppxlib.Location - let compare x y = - let st = compare_start x y in - if st = 0 then compare_end x y else st + let curr = of_lexbuf let merge x y = if Position.compare x.loc_end y.loc_start >= 0 then diff --git a/vendor/parse-wyc/lib/migrate_ast.mli b/vendor/parse-wyc/lib/migrate_ast.mli index eac325a4aa..4f60e22e2e 100644 --- a/vendor/parse-wyc/lib/migrate_ast.mli +++ b/vendor/parse-wyc/lib/migrate_ast.mli @@ -1,21 +1,32 @@ module Selected_version = Migrate_parsetree.Ast_408 -module Ast_mapper = Selected_version.Ast_mapper module Parsetree = Selected_version.Parsetree module Asttypes = Selected_version.Asttypes -module Location : sig - include module type of Selected_version.Location +module Mapper : sig + type ('omp, 'ppxlib) fragment = + | Structure + : ( Selected_version.Parsetree.structure, + Ppxlib.Parsetree.structure ) + fragment + | Signature + : ( Selected_version.Parsetree.signature, + Ppxlib.Parsetree.signature ) + fragment + | Use_file + : ( Selected_version.Parsetree.toplevel_phrase list, + Ppxlib.Parsetree.toplevel_phrase list ) + fragment - val compare : t -> t -> int + val iter_ast : + (_, 'ppxlib) fragment -> Ppxlib.Ast_traverse.iter -> 'ppxlib -> unit - val merge : t -> t -> t option + val to_ppxlib : ('omp, 'ppxlib) fragment -> 'omp -> 'ppxlib end -module Mapper : sig - type 'a fragment = - | Structure : Parsetree.structure fragment - | Signature : Parsetree.signature fragment - | Use_file : Parsetree.toplevel_phrase list fragment +module Location : sig + include module type of Ppxlib.Location + + val curr : Lexing.lexbuf -> t - val map_ast : 'a fragment -> Ast_mapper.mapper -> 'a -> 'a + val merge : t -> t -> t option end diff --git a/vendor/parse-wyc/lib/parse_wyc.ml b/vendor/parse-wyc/lib/parse_wyc.ml index edc166a305..ae30966b88 100644 --- a/vendor/parse-wyc/lib/parse_wyc.ml +++ b/vendor/parse-wyc/lib/parse_wyc.ml @@ -90,7 +90,8 @@ module With_recovery : PARSE_INTF = struct | _ -> Intermediate parser ) ) end -let entrypoint (type a) : a Mapper.fragment -> _ -> a I.checkpoint = function +let entrypoint (type o p) : (o, p) Mapper.fragment -> _ -> o I.checkpoint = + function | Mapper.Structure -> P.Incremental.implementation | Mapper.Signature -> P.Incremental.interface | Mapper.Use_file -> P.Incremental.use_file @@ -135,49 +136,46 @@ let merge_adj merge l = let normalize_locs locs = List.sort_uniq Location.compare locs |> merge_adj Location.merge -let invalid_locs fragment lexbuf = - let ast = parse_with_recovery fragment (lex_buf lexbuf) in +let invalid_locs : + type o p. (o, p) Mapper.fragment -> Lexing.lexbuf -> Warnings.loc list = + fun fragment lexbuf -> + let ast_omp = lex_buf lexbuf |> parse_with_recovery fragment in + let ast = Mapper.to_ppxlib fragment ast_omp in let loc_stack = Stack.create () in let loc_list = ref [] in - let make_mapper () = - let open Parsetree in - let default = Ast_mapper.default_mapper in - let expr m e = - if Annot.Exp.is_generated e then - loc_list := Stack.top loc_stack :: !loc_list; - default.expr m e - in - let class_expr m x = - if Annot.Class_exp.is_generated x then - loc_list := Stack.top loc_stack :: !loc_list; - default.class_expr m x - in - let wrap mapper loc f x = - if Stack.is_empty loc_stack then ( - Stack.push loc loc_stack; - let x = f mapper x in - ignore (Stack.pop loc_stack); - x ) - else f mapper x - in - let structure_item m x = wrap m x.pstr_loc default.structure_item x in - let signature_item m x = wrap m x.psig_loc default.signature_item x in - let attribute m x = - if Annot.Attr.is_generated x then - loc_list := Stack.top loc_stack :: !loc_list; - default.attribute m x - in - { - Ast_mapper.default_mapper with - attribute; - expr; - class_expr; - structure_item; - signature_item; - } + let wrap loc f x = + if Stack.is_empty loc_stack then ( + Stack.push loc loc_stack; + let x = f x in + ignore (Stack.pop loc_stack); + x ) + else f x + in + let iter = + object + inherit Ppxlib.Ast_traverse.iter as super + + method! attribute x = + if Annot.Attr.is_generated x then + loc_list := Stack.top loc_stack :: !loc_list; + super#attribute x + + method! expression e = + if Annot.Exp.is_generated e then + loc_list := Stack.top loc_stack :: !loc_list; + super#expression e + + method! class_expr x = + if Annot.Class_exp.is_generated x then + loc_list := Stack.top loc_stack :: !loc_list; + super#class_expr x + + method! structure_item x = wrap x.pstr_loc super#structure_item x + + method! signature_item x = wrap x.psig_loc super#signature_item x + end in - let mapper = make_mapper () in - ignore (Mapper.map_ast fragment mapper ast); + Mapper.iter_ast fragment iter ast; normalize_locs !loc_list let mk_parsable fragment source = diff --git a/vendor/parse-wyc/lib/parser.mly b/vendor/parse-wyc/lib/parser.mly index caa89cbf00..c8a4dd02b5 100644 --- a/vendor/parse-wyc/lib/parser.mly +++ b/vendor/parse-wyc/lib/parser.mly @@ -17,7 +17,7 @@ %{ -open Migrate_ast +open Migrate_parsetree.Ast_408 open Asttypes open Longident open Parsetree