Skip to content

Commit

Permalink
FB Messenger Webhooks (#3) (#8)
Browse files Browse the repository at this point in the history
* fix(load-api): define base uri

* fix(load-api): define dev environment

* fix(load-api): tbd

* fix(load-api): tbd

* fix(load-api): tbd

* fix(load-api): incorrect token format passed

* fix(schema): discrete schema, jvm mode in dev

* fix(cloud-run): increase ram in dev

* fix(cloud-run): migrate some secrets to gha from gcp

* fix(cloud-run): adjust parameters for jvm startup

* fix(load-api-integration): debug lines

* fix(load-api-integration): debug lines

* fix(load-api-integration): debug lines

* fix(load-api-integration): fix incorrect env var set

* fix(exception-handling): add class converter for FPISvcEx

* fix(exception-handling): add class converter for FPISvcEx

* fix(exception-handling): add class converter for FPISvcEx

* fix(exception-handling): add class converter for FPISvcEx

* fix(fpi-api): bump latest load and login api to latest snapshots

* fix(TrustedLoginApi): add HandleRestEx annotation

* fix(IApi): revert HandleRestEx

* fix(IApi): quarkus.rest-client.providers

* fix(IApi): ApiRepoExHandler

* fix(trusted-login): Message update

* fix(fpi): bump to 1.1.27

* fix(fpi): bump to 1.1.28

* fix(dto): enhance polymorphic jackson serdes

* fix(fpi-deps): sync load and meta to latest snapshots

* fix(fpi): bump to 1.1.31

* fix(fpi): bump to 1.1.31

* fix(meta): handle exceptions

* fix(fpi): bump to 1.1.32, revert JsonTypeInfo

* fix(error-handling): add retries and error messages

* fix(error-handling): add retries and error messages

* fix(bot-api): experience enhancements

* fix(bot-api): experience enhancements

* fix(bot-api): experience enhancements

* fix(bot-api): experience enhancements

* fix(docker): switch to smaller runtime only image

* fix(docker): rollback to ubi8 image

* fix(load-api): handle typing indicator ex

* fix(load-api): handle typing indicator ex

* fix(load-api): add jsoncreator mode in ApiStatus

* fix(webhooks): better error response

* fix(webhooks): move webhook logging logic to processor

* fix(webhooks): skip duplicate mids

* fix(config): dev environment tweaks, prefer cpu throttling

* fix(config): disable cpu throttling

* fix(config): errors

* fix(gcp): cpu throttling in prod env

* prepare for release
  • Loading branch information
vincejv authored Nov 1, 2022
1 parent 5f5a18a commit 2b52749
Show file tree
Hide file tree
Showing 11 changed files with 343 additions and 73 deletions.
20 changes: 11 additions & 9 deletions .github/workflows/release-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ env:
SERVICE: fpi-bot-api-dev
REGION: asia-east1
SONAR_PROJECT_KEY: vincejv_fpi-bot-api
NATIVE_IMAGE_BUILDER: quay.io/quarkus/ubi-quarkus-mandrel:22.2-java17
SERVICE_CPU: 1000m
SERVICE_MEMORY: 384Mi
SERVICE_MEMORY: 768Mi
SERVICE_ENV: dev

jobs:
Expand Down Expand Up @@ -62,7 +61,7 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=${{ env.SONAR_PROJECT_KEY }} -Dsonar.qualitygate.wait=true
run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=${{ env.SONAR_PROJECT_KEY }} -Dsonar.qualitygate.wait=true -P allow-snapshots

deploy_to_cloud:
name: Deploy to Cloud Run
Expand Down Expand Up @@ -102,8 +101,8 @@ jobs:
run: |
echo "service_image_path=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GHUB_REPO_NAME }}/${{ env.SERVICE }}:${{ steps.gen_ver.outputs.artifact_version }}" >> $GITHUB_OUTPUT
- name: Generate native image
run: mvn -B package -Pnative -Dquarkus.native.container-build=true -Dquarkus.native.builder-image=${{ env.NATIVE_IMAGE_BUILDER }}
- name: Create JVM package
run: mvn -B package -P allow-snapshots

- name: Google Auth
id: gcp-auth
Expand Down Expand Up @@ -141,6 +140,7 @@ jobs:
uses: docker/build-push-action@v3
with:
context: .
file: Dockerfile.dev
push: true
tags: | # ${{ steps.meta.outputs.tags }} - (For public repositories like docker hub)
${{ steps.image_version.outputs.service_image_path }}
Expand All @@ -154,8 +154,9 @@ jobs:
region: ${{ env.REGION }}
image: ${{ steps.image_version.outputs.service_image_path }}
project_id: ${{ env.PROJECT_ID }}
flags: --cpu ${{ env.SERVICE_CPU }} --memory ${{ env.SERVICE_MEMORY }}
flags: --cpu ${{ env.SERVICE_CPU }} --memory ${{ env.SERVICE_MEMORY }} --timeout 900 --no-cpu-throttling
env_vars: |
DB_NAME=bot-api-dev
OIDC_CLIENT_ID=${{ secrets.OIDC_CLIENT_ID }}
OIDC_AUTH_URL=${{ secrets.OIDC_AUTH_URL }}
LOGIN_BASE_URI=${{ secrets.LOGIN_BASE_URI }}
Expand All @@ -164,11 +165,12 @@ jobs:
META_FB_VRFY_TOKEN=${{ secrets.META_FB_VRFY_TOKEN }}
FPI_APP_TO_APP_USERN=${{ secrets.FPI_APP_TO_APP_USERN }}
USER_BASE_URI=${{ secrets.USER_BASE_URI }}
LOAD_API_BASE_URI=${{ secrets.LOAD_API_BASE_URI }}
META_FB_APP_SECRET=${{ secrets.META_FB_APP_SECRET }}
META_FB_PAGE_ACCESS_TOKEN=${{ secrets.META_FB_PAGE_ACCESS_TOKEN }}
secrets: |
MONGO_CONN_STRING=vbl_mongo_connection_string:latest
OIDC_SECRET=oidc_secret_keycloak:latest
META_FB_APP_SECRET=meta_fb_app_secret:latest
META_FB_PAGE_ACCESS_TOKEN=meta_fb_page_access_token:latest
FPI_APP_TO_APP_PASSW=fpi_bot_api_creds_secret_key:latest
labels: |
env=${{ env.SERVICE_ENV }}
Expand Down Expand Up @@ -210,7 +212,7 @@ jobs:
MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
run: mvn -B deploy -Prelease-for-oss
run: mvn -B deploy -P release-for-oss,allow-snapshots

pr_update:
name: Pull request update
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/release-main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,9 @@ jobs:
region: ${{ env.REGION }}
image: ${{ needs.versioning.outputs.service_image_path }}
project_id: ${{ env.PROJECT_ID }}
flags: --cpu ${{ env.SERVICE_CPU }} --memory ${{ env.SERVICE_MEMORY }}
flags: --cpu ${{ env.SERVICE_CPU }} --memory ${{ env.SERVICE_MEMORY }} --cpu-throttling
env_vars: |
DB_NAME=bot-api-prod
OIDC_CLIENT_ID=${{ secrets.OIDC_CLIENT_ID }}
OIDC_AUTH_URL=${{ secrets.OIDC_AUTH_URL }}
LOGIN_BASE_URI=${{ secrets.LOGIN_BASE_URI }}
Expand Down
91 changes: 91 additions & 0 deletions Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
####
# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode
#
# Before building the container image run:
#
# ./mvnw package
#
# Then, build the image with:
#
# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/code-with-quarkus-jvm .
#
# Then run the container using:
#
# docker run -i --rm -p 8080:8080 quarkus/code-with-quarkus-jvm
#
# If you want to include the debug port into your docker image
# you will have to expose the debug port (default 5005) like this : EXPOSE 8080 5005
#
# Then run the container using :
#
# docker run -i --rm -p 8080:8080 quarkus/code-with-quarkus-jvm
#
# This image uses the `run-java.sh` script to run the application.
# This scripts computes the command line to execute your Java application, and
# includes memory/GC tuning.
# You can configure the behavior using the following environment properties:
# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class")
# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options
# in JAVA_OPTS (example: "-Dsome.property=foo")
# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is
# used to calculate a default maximal heap memory based on a containers restriction.
# If used in a container without any memory constraints for the container then this
# option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio
# of the container available memory as set here. The default is `50` which means 50%
# of the available memory is used as an upper boundary. You can skip this mechanism by
# setting this value to `0` in which case no `-Xmx` option is added.
# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This
# is used to calculate a default initial heap memory based on the maximum heap memory.
# If used in a container without any memory constraints for the container then this
# option has no effect. If there is a memory constraint then `-Xms` is set to a ratio
# of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx`
# is used as the initial heap size. You can skip this mechanism by setting this value
# to `0` in which case no `-Xms` option is added (example: "25")
# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS.
# This is used to calculate the maximum value of the initial heap memory. If used in
# a container without any memory constraints for the container then this option has
# no effect. If there is a memory constraint then `-Xms` is limited to the value set
# here. The default is 4096MB which means the calculated value of `-Xms` never will
# be greater than 4096MB. The value of this variable is expressed in MB (example: "4096")
# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output
# when things are happening. This option, if set to true, will set
# `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true").
# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example:
# true").
# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787").
# - CONTAINER_CORE_LIMIT: A calculated core limit as described in
# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2")
# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024").
# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion.
# (example: "20")
# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking.
# (example: "40")
# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection.
# (example: "4")
# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus
# previous GC times. (example: "90")
# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20")
# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100")
# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should
# contain the necessary JRE command-line options to specify the required GC, which
# will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC).
# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080")
# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080")
# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be
# accessed directly. (example: "foo.example.com,bar.example.com")
#
###
FROM registry.access.redhat.com/ubi8/openjdk-17-runtime:1.14

ENV LANGUAGE='en_US:en'

# We make four distinct layers so if there are application changes the library layers can be re-used
COPY --chown=185 *core/target/quarkus-app/lib/ /deployments/lib/
COPY --chown=185 *core/target/quarkus-app/*.jar /deployments/
COPY --chown=185 *core/target/quarkus-app/app/ /deployments/app/
COPY --chown=185 *core/target/quarkus-app/quarkus/ /deployments/quarkus/

EXPOSE 8080
USER 185
ENV JAVA_OPTS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Dvertx.options.maxEventLoopExecuteTime=3600000000000"
ENV JAVA_APP_JAR="/deployments/quarkus-run.jar"
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package com.abavilla.fpi.bot.entity.enums;

import static com.abavilla.fpi.fw.util.FWConst.UNKNOWN_PREFIX;

import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;

import com.abavilla.fpi.fw.entity.enums.IBaseEnum;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonValue;
import io.quarkus.runtime.annotations.RegisterForReflection;
import lombok.AllArgsConstructor;
import lombok.Getter;

@Getter
@AllArgsConstructor
@RegisterForReflection
public enum SenderAction implements IBaseEnum {
MARK_SEEN(1, "MARK_SEEN"),
TYPING_ON(2, "TYPING_ON"),
TYPING_OFF(3, "TYPING_OFF"),
REACT(4, "REACT"),
UNREACT(5, "UNREACT"),
UNKNOWN(-1, UNKNOWN_PREFIX);

/**
* Ordinal id to enum mapping
*/
private static final Map<Integer, IBaseEnum> ENUM_MAP = new HashMap<>();

static {
for(IBaseEnum w : EnumSet.allOf(SenderAction.class))
ENUM_MAP.put(w.getId(), w);
}

/**
* The enum ordinal id
*/
private final int id;

/**
* The enum value
*/
private final String value;

/**
* Creates an enum based from given string value
*
* @param value the string value
* @return the created enum
*/
@JsonCreator(mode = JsonCreator.Mode.DELEGATING)
public static SenderAction fromValue(String value) {
return (SenderAction) IBaseEnum.fromValue(value, ENUM_MAP, UNKNOWN);
}

/**
* Creates an enum based from given an ordinal id
*
* @param id the ordinal id
* @return the created enum
*/
public static SenderAction fromId(int id) {
return (SenderAction) IBaseEnum.fromId(id, ENUM_MAP, UNKNOWN);
}

/**
* {@inheritDoc}
*/
@Override
@JsonValue
public String toString() {
return value;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,25 @@
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;

import com.abavilla.fpi.bot.service.MetaMsgrSvc;
import com.abavilla.fpi.bot.entity.meta.MetaMsgEvt;
import com.abavilla.fpi.bot.mapper.meta.MetaMsgEvtMapper;
import com.abavilla.fpi.bot.repo.MetaMsgEvtRepo;
import com.abavilla.fpi.bot.service.MetaMsgrApiSvc;
import com.abavilla.fpi.fw.dto.impl.RespDto;
import com.abavilla.fpi.fw.exceptions.ApiSvcEx;
import com.abavilla.fpi.fw.util.DateUtil;
import com.abavilla.fpi.load.ext.dto.QueryDto;
import com.abavilla.fpi.load.ext.rest.LoadQueryApi;
import com.abavilla.fpi.login.ext.dto.LoginDto;
import com.abavilla.fpi.login.ext.dto.SessionDto;
import com.abavilla.fpi.login.ext.dto.WebhookLoginDto;
import com.abavilla.fpi.login.ext.rest.TrustedLoginApi;
import com.abavilla.fpi.meta.ext.codec.MetaMsgEvtCodec;
import com.abavilla.fpi.meta.ext.dto.msgr.ext.MetaMsgEvtDto;
import com.mongodb.ErrorCategory;
import com.mongodb.MongoWriteException;
import io.netty.handler.codec.http.HttpResponseStatus;
import io.quarkus.logging.Log;
import io.quarkus.vertx.ConsumeEvent;
import io.smallrye.mutiny.Uni;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -44,39 +55,82 @@ public class MetaMsgEvtPcsr {
LoadQueryApi loadApi;

@Inject
MetaMsgrSvc metaMsgrSvc;
MetaMsgrApiSvc metaMsgrSvc;

@Inject
MetaMsgEvtMapper metaMsgEvtMapper;

@Inject
MetaMsgEvtRepo metaMsgEvtRepo;

@ConsumeEvent(value = "meta-msg-evt", codec = MetaMsgEvtCodec.class)
public Uni<Void> process(MetaMsgEvtDto evt) {
Log.info("Received event: " + evt);
if (StringUtils.isNotBlank(evt.getContent())) {
// verify if person is registered
var metaId = evt.getSender();
WebhookLoginDto login = new WebhookLoginDto();
login.setUsername(metaId);
return loginApi.webhookAuthenticate(login).chain(session -> {
if (StringUtils.equals(session.getStatus(),
SessionDto.SessionStatus.ESTABLISHED.toString())) {
var query = new QueryDto();
query.setQuery(evt.getContent());
return loadApi.query(query, session.getResp().getAccessToken()).chain(resp ->
sendMsgrMsg(evt, "Received your query, current status is " + resp.getStatus())
);
}
return sendUnauthorizedMsg(evt, session.getStatus());
return metaMsgrSvc.sendTypingIndicator(evt.getSender()).chain(() -> {
Log.info("Processing event: " + evt.getMetaMsgId());
MetaMsgEvt metaMsgEvt = metaMsgEvtMapper.mapToEntity(evt);
metaMsgEvt.setDateCreated(DateUtil.now());
metaMsgEvt.setDateUpdated(DateUtil.now());
return metaMsgEvtRepo.persist(metaMsgEvt).chain(() -> {
Log.info("Logged to db: " + metaMsgEvt.getMetaMsgId());
// verify if person is registered
var metaId = evt.getSender();
WebhookLoginDto login = new WebhookLoginDto();
login.setUsername(metaId);
Log.info("Authenticating user: " + login.getUsername());
return loginApi.webhookAuthenticate(login)
// process load
.chain(session -> processLoadQuery(login, session, evt))
// login failures
.onFailure(ApiSvcEx.class).recoverWithUni(ex -> handleApiEx(evt, ex))
// failures to send messenger
.onFailure().recoverWithItem(this::handleMsgEx)
.replaceWithVoid();
})
// login failures
.onFailure().recoverWithUni(throwable -> sendUnauthorizedMsg(evt, throwable.getMessage()))
.replaceWithVoid();
.onFailure(ex -> ex instanceof MongoWriteException wEx &&
wEx.getError().getCategory().equals(ErrorCategory.DUPLICATE_KEY))
.recoverWithUni(throwable -> {
Log.warn("Received duplicate mid: " + evt.getMetaMsgId());
return Uni.createFrom().voidItem();
});
}).onFailure().recoverWithItem(this::handleMsgEx);
}

return Uni.createFrom().voidItem();
}

private Uni<Void> sendUnauthorizedMsg(MetaMsgEvtDto evt, String msg) {
return sendMsgrMsg(evt, "Unauthorized user" + msg);
private Uni<Void> processLoadQuery(LoginDto login, RespDto<SessionDto> session, MetaMsgEvtDto evt) {
Log.info("Authenticated: " + login.getUsername());
if (StringUtils.equals(session.getStatus(),
SessionDto.SessionStatus.ESTABLISHED.toString())) {
var query = new QueryDto();
query.setQuery(evt.getContent());
return loadApi.query(query, "Bearer " + session.getResp().getAccessToken()).chain(resp -> {
Log.info("Query received, response is " + resp);
return sendMsgrMsg(evt, "Received your query, current status is " + resp.getStatus());
});
}
return sendMsgrMsg(evt, session.getStatus());
}

private Void handleMsgEx(Throwable sendMsgEx) {
Log.error("Message sending failed: " + sendMsgEx.getMessage(), sendMsgEx);
return null;
}

private Uni<Void> handleApiEx(MetaMsgEvtDto evt, Throwable ex) {
Log.error("Error while processing evt: " + evt.getMetaMsgId(), ex);
var apiSvcEx = (ApiSvcEx) ex;
if (!HttpResponseStatus.INTERNAL_SERVER_ERROR.equals(apiSvcEx.getHttpResponseStatus())) {
return sendMsgrMsg(evt,
apiSvcEx.getJsonResponse(RespDto.class).getError());
} else {
return sendMsgrMsg(evt, "Error occurred, please try again");
}
}

private Uni<Void> sendMsgrMsg(MetaMsgEvtDto evt, String msg) {
Log.info("Sending msgr msg: " + msg + " event: " + evt.getMetaMsgId());
return metaMsgrSvc.sendMsg(msg, evt.getSender()).replaceWithVoid();
}

Expand Down
Loading

0 comments on commit 2b52749

Please sign in to comment.