From 22e42b3a6a1a01507973d0cedc3405d5fe8d223c Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Tue, 28 May 2024 00:26:34 -0700 Subject: [PATCH 01/10] middleware skeleton --- lib/inngest/middleware.ex | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 lib/inngest/middleware.ex diff --git a/lib/inngest/middleware.ex b/lib/inngest/middleware.ex new file mode 100644 index 0000000..f521e97 --- /dev/null +++ b/lib/inngest/middleware.ex @@ -0,0 +1,34 @@ +defmodule Inngest.Middleware do + @moduledoc """ + Inngest Middleware specification + """ + + @type opts :: + binary() + | tuple() + | atom() + | integer() + | float() + | [opts] + | map() + + @callback init(opts) :: opts + + @callback transform_input(map(), opts) :: map() + + @callback before_memoization(map(), opts) :: map() + + @callback after_memoization(map(), opts) :: map() + + @callback before_execution(map(), opts) :: map() + + @callback after_execution(map(), opts) :: map() + + @callback transform_output(map(), opts) :: map() + + @callback before_response(map(), opts) :: map() + + @callback before_send_events(map(), opts) :: map() + + @callback after_send_events(map(), opts) :: map() +end From 1fbf602ff7a4ca13da85a0b27ee3e53cf3f33137 Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Tue, 28 May 2024 22:36:44 -0700 Subject: [PATCH 02/10] update behavior interfaces --- lib/inngest/middleware.ex | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/lib/inngest/middleware.ex b/lib/inngest/middleware.ex index f521e97..7bae5f3 100644 --- a/lib/inngest/middleware.ex +++ b/lib/inngest/middleware.ex @@ -12,23 +12,34 @@ defmodule Inngest.Middleware do | [opts] | map() - @callback init(opts) :: opts + @type input_args :: %{ + ctx: %{event: map(), run_id: binary()}, + steps: map() + } - @callback transform_input(map(), opts) :: map() + @type input_ret :: %{ + ctx: any(), + steps: map() + } - @callback before_memoization(map(), opts) :: map() + @type output_args :: %{ + result: any(), + step: any() | nil + } - @callback after_memoization(map(), opts) :: map() + @type output_ret :: %{ + result: %{data: any()} + } - @callback before_execution(map(), opts) :: map() + @callback name() :: binary() - @callback after_execution(map(), opts) :: map() + @callback init() :: opts - @callback transform_output(map(), opts) :: map() + @callback transform_input(input_args, opts) :: map() - @callback before_response(map(), opts) :: map() + @callback before_execution(opts) :: :ok - @callback before_send_events(map(), opts) :: map() + @callback after_execution(opts) :: :ok - @callback after_send_events(map(), opts) :: map() + @callback transform_output(output_args, opts) :: output_ret end From 3c72a267728dbe0c2196aa9945b2f60aecae8fee Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Sun, 2 Jun 2024 10:58:25 -0700 Subject: [PATCH 03/10] rename directory to handler --- Makefile | 2 +- lib/inngest/{router => handler}/helper.ex | 0 lib/inngest/{router => handler}/invoke.ex | 0 lib/inngest/{router => handler}/phoenix.ex | 0 lib/inngest/{router => handler}/plug.ex | 0 lib/inngest/{router => handler}/register.ex | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename lib/inngest/{router => handler}/helper.ex (100%) rename lib/inngest/{router => handler}/invoke.ex (100%) rename lib/inngest/{router => handler}/phoenix.ex (100%) rename lib/inngest/{router => handler}/plug.ex (100%) rename lib/inngest/{router => handler}/register.ex (100%) diff --git a/Makefile b/Makefile index 55b5498..506a5c5 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ clean: .PHONY: docs docs: - mix docs --open + mix docs -f html --open .PHONY: changelog changelog: diff --git a/lib/inngest/router/helper.ex b/lib/inngest/handler/helper.ex similarity index 100% rename from lib/inngest/router/helper.ex rename to lib/inngest/handler/helper.ex diff --git a/lib/inngest/router/invoke.ex b/lib/inngest/handler/invoke.ex similarity index 100% rename from lib/inngest/router/invoke.ex rename to lib/inngest/handler/invoke.ex diff --git a/lib/inngest/router/phoenix.ex b/lib/inngest/handler/phoenix.ex similarity index 100% rename from lib/inngest/router/phoenix.ex rename to lib/inngest/handler/phoenix.ex diff --git a/lib/inngest/router/plug.ex b/lib/inngest/handler/plug.ex similarity index 100% rename from lib/inngest/router/plug.ex rename to lib/inngest/handler/plug.ex diff --git a/lib/inngest/router/register.ex b/lib/inngest/handler/register.ex similarity index 100% rename from lib/inngest/router/register.ex rename to lib/inngest/handler/register.ex From 91b6229314e5c27f5e7a2914469f2f9ff3f50b27 Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Sun, 2 Jun 2024 11:33:57 -0700 Subject: [PATCH 04/10] finalize middleware interface referencing specs and existing SDKs --- lib/inngest/handler/invoke.ex | 1 + lib/inngest/middleware.ex | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/inngest/handler/invoke.ex b/lib/inngest/handler/invoke.ex index f7b0e2d..d9c1c67 100644 --- a/lib/inngest/handler/invoke.ex +++ b/lib/inngest/handler/invoke.ex @@ -53,6 +53,7 @@ defmodule Inngest.Router.Invoke do } input = %Inngest.Function.Input{ + attempt: Map.get(ctx, "attempt", 0), event: Inngest.Event.from(event), events: Enum.map(events, &Inngest.Event.from/1), run_id: Map.get(ctx, "run_id"), diff --git a/lib/inngest/middleware.ex b/lib/inngest/middleware.ex index 7bae5f3..f48218b 100644 --- a/lib/inngest/middleware.ex +++ b/lib/inngest/middleware.ex @@ -13,18 +13,18 @@ defmodule Inngest.Middleware do | map() @type input_args :: %{ - ctx: %{event: map(), run_id: binary()}, - steps: map() + ctx: Inngest.Function.Input, + steps: list(map()) } @type input_ret :: %{ - ctx: any(), - steps: map() + ctx: Inngest.Function.Input, + steps: list(map()) } @type output_args :: %{ - result: any(), - step: any() | nil + result: %{data: any()}, + step: Inngest.GeneratorOpCode.t() | nil } @type output_ret :: %{ @@ -35,7 +35,11 @@ defmodule Inngest.Middleware do @callback init() :: opts - @callback transform_input(input_args, opts) :: map() + @callback transform_input(input_args, opts) :: input_ret + + @callback before_memoization(opts) :: :ok + + @callback after_memoization(opts) :: :ok @callback before_execution(opts) :: :ok From b3acf1a4205fc1bd35b0cec6aa2d196b296052c7 Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Sun, 2 Jun 2024 12:26:35 -0700 Subject: [PATCH 05/10] add additional key for loading functions --- lib/inngest/handler/helper.ex | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/inngest/handler/helper.ex b/lib/inngest/handler/helper.ex index 1d5587a..70dd158 100644 --- a/lib/inngest/handler/helper.ex +++ b/lib/inngest/handler/helper.ex @@ -25,6 +25,28 @@ defmodule Inngest.Router.Helper do Map.put(kv, :funcs, modules) end + def load_functions_from_path(%{functions: funcs} = kv) when is_list(funcs) do + modules = + funcs + |> Enum.map(&Path.wildcard/1) + |> List.flatten() + |> Stream.filter(&(!File.dir?(&1))) + |> Enum.uniq() + |> extract_modules() + + Map.put(kv, :funcs, modules) + end + + def load_functions_from_path(%{functions: funcs} = kv) when is_binary(funcs) do + modules = + funcs + |> Path.wildcard() + |> Enum.filter(&(!File.dir?(&1))) + |> extract_modules() + + Map.put(kv, :funcs, modules) + end + def load_functions_from_path(%{path: path} = kv) when is_binary(path) do modules = path From 15ef35e4ff0f593f3f000bcf31bd25c407a877b3 Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Sun, 2 Jun 2024 12:27:29 -0700 Subject: [PATCH 06/10] naive middleware loading --- lib/inngest/function/input.ex | 4 +++- lib/inngest/handler/helper.ex | 25 +++++++++++++++++++++++++ lib/inngest/handler/invoke.ex | 10 ++++++++++ lib/inngest/handler/plug.ex | 1 + 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/inngest/function/input.ex b/lib/inngest/function/input.ex index 3524eac..cec3723 100644 --- a/lib/inngest/function/input.ex +++ b/lib/inngest/function/input.ex @@ -29,6 +29,7 @@ defmodule Inngest.Function.Context do defstruct [ :attempt, :run_id, + :middleware, # ETS table :index, steps: %{} @@ -38,6 +39,7 @@ defmodule Inngest.Function.Context do attempt: number(), run_id: binary(), index: :ets.tid(), - steps: map() + steps: map(), + middleware: map() } end diff --git a/lib/inngest/handler/helper.ex b/lib/inngest/handler/helper.ex index 70dd158..5c5e4d0 100644 --- a/lib/inngest/handler/helper.ex +++ b/lib/inngest/handler/helper.ex @@ -12,6 +12,15 @@ defmodule Inngest.Router.Helper do end end + def load_middleware(params) do + if Config.path_runtime_eval() do + %{middleware: middleware} = load_middleware_from_path(params) + middleware + else + Map.get(params, :middleware, []) + end + end + @spec load_functions_from_path(map()) :: map() def load_functions_from_path(%{path: paths} = kv) when is_list(paths) do modules = @@ -59,6 +68,22 @@ defmodule Inngest.Router.Helper do def load_functions_from_path(kv), do: kv + @spec load_middleware_from_path(map()) :: map() + def load_middleware_from_path(%{middleware: middleware} = kv) when is_list(middleware) do + # modules = + # middleware + # |> IO.inspect() + # |> Enum.map(&Path.wildcard/1) + # |> List.flatten() + # |> Stream.filter(&(!File.dir?(&1))) + # |> Enum.uniq() + # |> extract_modules() + + Map.put(kv, :middleware, middleware) + end + + def load_middleware_from_path(kv), do: kv + defp extract_modules(files) do files |> Enum.flat_map(fn file -> diff --git a/lib/inngest/handler/invoke.ex b/lib/inngest/handler/invoke.ex index d9c1c67..a3596e7 100644 --- a/lib/inngest/handler/invoke.ex +++ b/lib/inngest/handler/invoke.ex @@ -45,10 +45,20 @@ defmodule Inngest.Router.Invoke do Enum.member?(func.slugs(), fn_slug) end) + # Initialize middlewares + middleware = + params + |> load_middleware() + |> Enum.into(%{}, fn mid -> + opts = mid.init() + {mid.name(), %{opts: opts, mid: mid}} + end) + ctx = %Inngest.Function.Context{ attempt: Map.get(ctx, "attempt", 0), run_id: Map.get(ctx, "run_id"), steps: Map.get(params, "steps"), + middleware: middleware, index: :ets.new(:index, [:set, :private]) } diff --git a/lib/inngest/handler/plug.ex b/lib/inngest/handler/plug.ex index ee6abd4..4019cb5 100644 --- a/lib/inngest/handler/plug.ex +++ b/lib/inngest/handler/plug.ex @@ -22,6 +22,7 @@ defmodule Inngest.Router.Plug do end |> Enum.into(%{}) |> Inngest.Router.Helper.load_functions_from_path() + |> Inngest.Router.Helper.load_middleware_from_path() |> Map.put(:framework, @framework) |> Macro.escape() From ee8d8efd71b05b8a6daf9dddaa26c665441e6c5f Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Sun, 2 Jun 2024 12:27:40 -0700 Subject: [PATCH 07/10] add test middleware --- test/support/cases/middleware/test.ex | 47 +++++++++++++++++++++++++++ test/support/router/plug.ex | 7 +++- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 test/support/cases/middleware/test.ex diff --git a/test/support/cases/middleware/test.ex b/test/support/cases/middleware/test.ex new file mode 100644 index 0000000..88b1df3 --- /dev/null +++ b/test/support/cases/middleware/test.ex @@ -0,0 +1,47 @@ +defmodule Inngest.Test.Case.Middleware.Test do + @moduledoc false + + @behaviour Inngest.Middleware + + @impl true + def init(), do: [] + + @impl true + def name(), do: "test" + + @impl true + def transform_input(input_args, _opts) do + IO.inspect("Transform input") + input_args + end + + @impl true + def transform_output(output_args, _opts) do + IO.inspect("Transform outputx") + output_args + end + + @impl true + def before_memoization(_opts) do + IO.inspect("Before memoization") + :ok + end + + @impl true + def after_memoization(_opts) do + IO.inspect("After memoization") + :ok + end + + @impl true + def before_execution(_opts) do + IO.inspect("Before execution") + :ok + end + + @impl true + def after_execution(_opts) do + IO.inspect("After execution") + :ok + end +end diff --git a/test/support/router/plug.ex b/test/support/router/plug.ex index 8ee4543..af43424 100644 --- a/test/support/router/plug.ex +++ b/test/support/router/plug.ex @@ -25,7 +25,12 @@ defmodule Inngest.Test.PlugRouter do |> send_resp(200, data) end - inngest("/api/inngest", path: "test/support/cases/*") + inngest("/api/inngest", + functions: "test/support/cases/*", + middleware: [ + Inngest.Test.Case.Middleware.Test + ] + ) match _ do send_resp(conn, 404, "oops\n") From e70a12a703542ef05d0fca3e30588ba2f1124346 Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Sun, 2 Jun 2024 12:34:25 -0700 Subject: [PATCH 08/10] change variable name --- lib/inngest/function/input.ex | 4 ---- lib/inngest/handler/invoke.ex | 8 +++----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/inngest/function/input.ex b/lib/inngest/function/input.ex index cec3723..828652e 100644 --- a/lib/inngest/function/input.ex +++ b/lib/inngest/function/input.ex @@ -27,8 +27,6 @@ defmodule Inngest.Function.Context do """ defstruct [ - :attempt, - :run_id, :middleware, # ETS table :index, @@ -36,8 +34,6 @@ defmodule Inngest.Function.Context do ] @type t() :: %__MODULE__{ - attempt: number(), - run_id: binary(), index: :ets.tid(), steps: map(), middleware: map() diff --git a/lib/inngest/handler/invoke.ex b/lib/inngest/handler/invoke.ex index a3596e7..776f2ea 100644 --- a/lib/inngest/handler/invoke.ex +++ b/lib/inngest/handler/invoke.ex @@ -54,9 +54,7 @@ defmodule Inngest.Router.Invoke do {mid.name(), %{opts: opts, mid: mid}} end) - ctx = %Inngest.Function.Context{ - attempt: Map.get(ctx, "attempt", 0), - run_id: Map.get(ctx, "run_id"), + fnctx = %Inngest.Function.Context{ steps: Map.get(params, "steps"), middleware: middleware, index: :ets.new(:index, [:set, :private]) @@ -73,12 +71,12 @@ defmodule Inngest.Router.Invoke do resp = case Config.is_dev() do true -> - invoke(func, ctx, input) + invoke(func, fnctx, input) false -> with sig <- conn |> Plug.Conn.get_req_header(Headers.signature()) |> List.first(), true <- Signature.signing_key_valid?(sig, Config.signing_key(), body) do - invoke(func, ctx, input) + invoke(func, fnctx, input) else _ -> SdkResponse.from_result({:error, "unable to verify signature"}, retry: false) From fc3e18e8fa09c065acce3fabea51811d7ee65273 Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Sun, 2 Jun 2024 12:56:29 -0700 Subject: [PATCH 09/10] mark callbacks as optional for middleware --- lib/inngest/middleware.ex | 12 ++++++++++-- test/support/cases/middleware/test.ex | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/inngest/middleware.ex b/lib/inngest/middleware.ex index f48218b..f86c416 100644 --- a/lib/inngest/middleware.ex +++ b/lib/inngest/middleware.ex @@ -33,10 +33,13 @@ defmodule Inngest.Middleware do @callback name() :: binary() - @callback init() :: opts + # NOTE: what to make available on init? + @callback init(opts) :: opts @callback transform_input(input_args, opts) :: input_ret + @callback transform_output(output_args, opts) :: output_ret + @callback before_memoization(opts) :: :ok @callback after_memoization(opts) :: :ok @@ -45,5 +48,10 @@ defmodule Inngest.Middleware do @callback after_execution(opts) :: :ok - @callback transform_output(output_args, opts) :: output_ret + @optional_callbacks transform_input: 2, + transform_output: 2, + before_memoization: 1, + after_memoization: 1, + before_execution: 1, + after_execution: 1 end diff --git a/test/support/cases/middleware/test.ex b/test/support/cases/middleware/test.ex index 88b1df3..66014a5 100644 --- a/test/support/cases/middleware/test.ex +++ b/test/support/cases/middleware/test.ex @@ -4,7 +4,7 @@ defmodule Inngest.Test.Case.Middleware.Test do @behaviour Inngest.Middleware @impl true - def init(), do: [] + def init(_), do: [] @impl true def name(), do: "test" From b38dff132289513201acabc6d834acf63690891d Mon Sep 17 00:00:00 2001 From: Darwin D Wu Date: Sun, 2 Jun 2024 13:16:01 -0700 Subject: [PATCH 10/10] update init args --- lib/inngest/handler/invoke.ex | 44 +++++++++++++++++++++++++++-------- lib/inngest/middleware.ex | 8 ++++++- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/lib/inngest/handler/invoke.ex b/lib/inngest/handler/invoke.ex index 776f2ea..a4362b0 100644 --- a/lib/inngest/handler/invoke.ex +++ b/lib/inngest/handler/invoke.ex @@ -38,6 +38,27 @@ defmodule Inngest.Router.Invoke do %{private: %{raw_body: [body]}} = conn, %{"event" => event, "events" => events, "ctx" => ctx, "fnId" => fn_slug} = params ) do + input = %Inngest.Function.Input{ + attempt: Map.get(ctx, "attempt", 0), + event: Inngest.Event.from(event), + events: Enum.map(events, &Inngest.Event.from/1), + run_id: Map.get(ctx, "run_id"), + step: Inngest.StepTool + } + + # prepare steps to be passed into middlewares + steps = + case get_in(params, ["ctx", "stack", "stack"]) do + nil -> + [] + + stack -> + Enum.into(stack, [], fn hash -> + data = get_in(params, ["steps", hash]) + %{id: hash, data: data} + end) + end + func = params |> load_functions() @@ -50,24 +71,27 @@ defmodule Inngest.Router.Invoke do params |> load_middleware() |> Enum.into(%{}, fn mid -> - opts = mid.init() + arg = %{input: input, func: func, steps: steps} + opts = mid.init(arg) {mid.name(), %{opts: opts, mid: mid}} end) + # Transform inputs + steps = + steps + # TODO: Apply each middleware to the step data + # |> Stream.map(fn step -> + # end) + |> Enum.into(%{}, fn %{id: id, data: data} -> + {id, data} + end) + fnctx = %Inngest.Function.Context{ - steps: Map.get(params, "steps"), + steps: steps, middleware: middleware, index: :ets.new(:index, [:set, :private]) } - input = %Inngest.Function.Input{ - attempt: Map.get(ctx, "attempt", 0), - event: Inngest.Event.from(event), - events: Enum.map(events, &Inngest.Event.from/1), - run_id: Map.get(ctx, "run_id"), - step: Inngest.StepTool - } - resp = case Config.is_dev() do true -> diff --git a/lib/inngest/middleware.ex b/lib/inngest/middleware.ex index f86c416..b09d5fc 100644 --- a/lib/inngest/middleware.ex +++ b/lib/inngest/middleware.ex @@ -12,6 +12,12 @@ defmodule Inngest.Middleware do | [opts] | map() + @type init_args :: %{ + input: Inngest.Function.Input, + func: Inngest.Function, + steps: list(map()) + } + @type input_args :: %{ ctx: Inngest.Function.Input, steps: list(map()) @@ -34,7 +40,7 @@ defmodule Inngest.Middleware do @callback name() :: binary() # NOTE: what to make available on init? - @callback init(opts) :: opts + @callback init(init_args) :: opts @callback transform_input(input_args, opts) :: input_ret