Skip to content

Commit

Permalink
Merge pull request #16 from zimmerman-team/feat/new-iati-cloud-setup
Browse files Browse the repository at this point in the history 
Direct indexing conversion
  • Loading branch information
@stephanoshadjipetrou
stephanoshadjipetrou authored Aug 4, 2022
2 parents 8077ce1 + fc0e8f3 commit 35105f4
Show file tree
Hide file tree
Showing 64 changed files with 6,736 additions and 1,546 deletions.
2 changes: 0 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ PROJECT_URL=http://localhost:3000
# data
DS_SOLR_API=https://iati.cloud/search
DS_REST_API=https://iati.cloud/api
MFA_PUBLISHER_REF=FI-3
HDRO_API=http://ec2-54-174-131-205.compute-1.amazonaws.com/API/HDRO_API.php
UM_FI_API=https://um.fi/o/public-api/v1/content
Expand All @@ -59,7 +58,6 @@ Project will be running on http://localhost:4200/api
BACKEND_PORT=4200
MFA_PUBLISHER_REF=FI-3
PROJECT_URL=http://localhost:4200
DS_REST_API=https://iati.cloud/api
DS_SOLR_API=https://iati.cloud/search
HDRO_API=http://ec2-54-174-131-205.compute-1.amazonaws.com/API/HDRO_API.php
UM_FI_API=https://um.fi/o/public-api/v1/content
Expand Down
94 changes: 60 additions & 34 deletions server/controllers/detail-api/activity.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import axios from "axios";
import get from "lodash/get";
import find from "lodash/find";
import sumBy from "lodash/sumBy";
import querystring from "querystring";
import { genericError } from "../../utils/general";
import {
activityMetadataFl,
activityTransactionsFl
} from "../../static/activityDetailConsts";
import { parseJsonSolrField } from "../../utils/parseJsonSolrField";
import {
getCountries,
getDates,
Expand All @@ -20,6 +20,23 @@ import {
getTransactions
} from "./utils/activity";
import { getFieldValueLang } from "../../utils/getFieldValueLang";
import {
AF_DESCRIPTION_NARRATIVE,
AF_DESCRIPTION_NARRATIVE_LANG,
AF_IATI_IDENTIFIER,
AF_REPORTING_ORG_NARRATIVE,
AF_REPORTING_ORG_REF,
AF_REPORTING_ORG_TYPE_CODE,
AF_REPORTING_ORG_TYPE_NAME,
AF_TITLE_NARRATIVE,
AF_TITLE_NARRATIVE_LANG,
AF_TRANSACTION_COUNTRY,
AF_TRANSACTION_REGION,
AF_TRANSACTION_SECTOR_CODE,
AF_TRANSACTION_TYPE_CODE,
AF_TRANSACTION_VALUE
} from "../../static/apiFilterFields";
import { organisationTypeCodelist } from "../filter-api/utils/codelists";

export function activityDetail(req: any, res: any) {
const lang = req.body.lang || "en";
Expand All @@ -31,16 +48,16 @@ export function activityDetail(req: any, res: any) {
}
const decodedId = decodeURIComponent(req.body.activityId);
const metadata = {
q: `iati_identifier:"${decodedId}"`,
q: `${AF_IATI_IDENTIFIER}:"${decodedId}"`,
fl: activityMetadataFl
};
const disbursements = {
q: `iati_identifier:"${decodedId}" AND (transaction_type:3)`,
q: `${AF_IATI_IDENTIFIER}:"${decodedId}" AND (${AF_TRANSACTION_TYPE_CODE}:3)`,
fl: activityTransactionsFl,
rows: 1000
};
const commitments = {
q: `iati_identifier:"${decodedId}" AND (transaction_type:2)`,
q: `${AF_IATI_IDENTIFIER}:"${decodedId}" AND (${AF_TRANSACTION_TYPE_CODE}:2)`,
fl: activityTransactionsFl,
rows: 1000
};
Expand Down Expand Up @@ -87,62 +104,71 @@ export function activityDetail(req: any, res: any) {
res.json({
data: {
metadata: {
iati_identifier: get(activityMetaData, "iati_identifier", ""),
reporting_org_ref: get(activityMetaData, "reporting_org_ref", ""),
iati_identifier: get(
activityMetaData,
`["${AF_IATI_IDENTIFIER}"]`,
""
),
reporting_org_ref: get(
activityMetaData,
AF_REPORTING_ORG_REF,
""
),
reporting_org_narrative: get(
activityMetaData,
"reporting_org_narrative",
`["${AF_REPORTING_ORG_NARRATIVE}"]`,
[""]
)[0],
reporting_org_type: parseJsonSolrField(
activityMetaData.reporting_org,
"type.name"
reporting_org_type: get(
activityMetaData,
AF_REPORTING_ORG_TYPE_NAME,
find(organisationTypeCodelist, {
code: get(
activityMetaData,
`["${AF_REPORTING_ORG_TYPE_CODE}"]`,
""
)
})?.name || ""
),
title: getFieldValueLang(
lang,
get(activityMetaData, "title_narrative_text", [""]),
get(activityMetaData, "title_narrative_lang", [""])
get(activityMetaData, `["${AF_TITLE_NARRATIVE}"]`, [""]),
get(activityMetaData, `["${AF_TITLE_NARRATIVE_LANG}"]`, [""])
),
dates: getDates(get(activityMetaData, "activity_date", [])),
dates: getDates(activityMetaData),
description: getFieldValueLang(
lang,
get(activityMetaData, "description_narrative_text", [""]),
get(activityMetaData, "description_lang", [""])
),
participating_orgs: getParticipatingOrgs(
get(activityMetaData, "participating_org", []),
lang
get(activityMetaData, `["${AF_DESCRIPTION_NARRATIVE}"]`, [""]),
get(activityMetaData, `["${AF_DESCRIPTION_NARRATIVE_LANG}"]`, [
""
])
),
participating_orgs: getParticipatingOrgs(activityMetaData, lang),
summary: getSummary(activityMetaData),
countries: getCountries(
get(activityMetaData, "recipient_country", []),
get(activityMetaData, "transaction_recipient_country_code", []),
activityMetaData,
get(activityMetaData, AF_TRANSACTION_COUNTRY, []),
lang
),
regions: getRegions(
get(activityMetaData, "recipient_region", []),
get(activityMetaData, "transaction_recipient_region_code", []),
activityMetaData,
get(activityMetaData, AF_TRANSACTION_REGION, []),
lang
),
sectors: getSectors(
get(activityMetaData, "sector", []),
get(activityMetaData, "transaction_sector_code", []),
lang
),
default_aid_types: getDefaultAidTypes(
get(activityMetaData, "default_aid_type", []),
activityMetaData,
get(activityMetaData, AF_TRANSACTION_SECTOR_CODE, null),
lang
),
policy_markers: getPolicyMarkers(
get(activityMetaData, "policy_marker", [])
)
default_aid_types: getDefaultAidTypes(activityMetaData, lang),
policy_markers: getPolicyMarkers(activityMetaData)
},
transactions: getTransactions([
...disbursementsData,
...commitmentsData
]),
disbursementsTotal: sumBy(disbursementsData, "transaction_value"),
commitmentsTotal: sumBy(commitmentsData, "transaction_value")
disbursementsTotal: sumBy(disbursementsData, AF_TRANSACTION_VALUE),
commitmentsTotal: sumBy(commitmentsData, AF_TRANSACTION_VALUE)
}
});
})
Expand Down
95 changes: 54 additions & 41 deletions server/controllers/detail-api/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,27 @@ import { orgTypesCodelist } from "../../static/orgTypesCodelist";
import { locationsMapping } from "../../static/locationsMapping";
import { partnerCountries } from "../../static/partnerCountries";
import { sectorTranslations } from "../../static/sectorTranslations";
import {
AF_COUNTRY,
AF_REPORTING_ORG_REF,
AF_PARTICIPATING_ORG_REF,
AF_PARTICIPATING_ORG_NARRATIVE,
AF_REGION,
AF_REGION_NAME,
AF_SECTOR,
AF_TAG_NARRATIVE,
AF_PARTICIPATING_ORG_TYPE
} from "../../static/apiFilterFields";

export function detailPageName(req: any, res: any) {
const values = {
q: `reporting_org_ref:${process.env.MFA_PUBLISHER_REF} AND ${[
q: `${AF_REPORTING_ORG_REF}:${process.env.MFA_PUBLISHER_REF} AND ${[
req.body.detail_type
]}:(${get(req.body.filters, `${req.body.detail_type}[0]`, "")})`,
fl:
"participating_org_ref,participating_org_narrative,recipient_region_code,recipient_region_name,sector_code",
]}:(${get(req.body.filters, `["${req.body.detail_type}"][0]`, "")})`,
fl: `${AF_PARTICIPATING_ORG_REF},${AF_PARTICIPATING_ORG_NARRATIVE},${AF_REGION},${AF_REGION_NAME},${AF_SECTOR}`,
rows: 1
};

axios
.get(
`${process.env.DS_SOLR_API}/activity/?${querystring.stringify(
Expand All @@ -37,9 +48,9 @@ export function detailPageName(req: any, res: any) {
.then(response => {
const data = get(response.data, "response.docs[0]", null);
let result = "";
if (req.body.detail_type === "sector_code") {
if (req.body.detail_type === AF_SECTOR) {
const fsector = find(sectorTranslations, {
code: parseInt(req.body.filters.sector_code[0], 10)
code: parseInt(req.body.filters[AF_SECTOR][0], 10)
});
if (fsector) {
res.json({
Expand All @@ -56,9 +67,9 @@ export function detailPageName(req: any, res: any) {
res.json({
data: {
names: {
name: req.body.filters.sector_code[0],
name_fi: req.body.filters.sector_code[0],
name_se: req.body.filters.sector_code[0]
name: req.body.filters[AF_SECTOR][0],
name_fi: req.body.filters[AF_SECTOR][0],
name_se: req.body.filters[AF_SECTOR][0]
},
description: ""
}
Expand All @@ -67,14 +78,14 @@ export function detailPageName(req: any, res: any) {
}
if (
(!data ||
(req.body.filters.participating_org_ref &&
req.body.filters.participating_org_ref.length > 1)) &&
req.body.detail_type === "participating_org_ref"
(req.body.filters[AF_PARTICIPATING_ORG_REF] &&
req.body.filters[AF_PARTICIPATING_ORG_REF].length > 1)) &&
req.body.detail_type === AF_PARTICIPATING_ORG_REF
) {
const fOrgMapping = find(orgMapping, {
code: parseInt(
req.body.filters.participating_org_ref[
req.body.filters.participating_org_ref.length - 1
req.body.filters[AF_PARTICIPATING_ORG_REF][
req.body.filters[AF_PARTICIPATING_ORG_REF].length - 1
],
10
)
Expand All @@ -92,25 +103,23 @@ export function detailPageName(req: any, res: any) {
}
}
if (req.body.detail_type && data) {
if (req.body.detail_type === "recipient_country_code") {
const iso3 = getCountryISO3(
req.body.filters.recipient_country_code[0]
);
if (req.body.detail_type === AF_COUNTRY) {
const iso3 = getCountryISO3(req.body.filters[AF_COUNTRY][0]);
const fCountry = find(translatedCountries, {
code: req.body.filters.recipient_country_code[0]
code: req.body.filters[AF_COUNTRY][0]
});
let isPartner = false;
let region = "";
isPartner =
find(
partnerCountries,
(p: string) => p === req.body.filters.recipient_country_code[0]
(p: string) => p === req.body.filters[AF_COUNTRY][0]
) !== undefined;
Object.keys(locationsMapping).forEach((key: string) => {
const fRegion = find(
locationsMapping[key],
(c: string) =>
c === get(req.body, "filters.recipient_country_code[0]", "")
c === get(req.body, `filters[${AF_COUNTRY}][0]`, "")
);
if (fRegion) {
region = key;
Expand Down Expand Up @@ -197,17 +206,17 @@ export function detailPageName(req: any, res: any) {
name: get(
fCountry,
"info.name",
req.body.filters.recipient_country_code[0]
req.body.filters[AF_COUNTRY][0]
),
name_fi: get(
fCountry,
"info.name_fi",
req.body.filters.recipient_country_code[0]
req.body.filters[AF_COUNTRY][0]
),
name_se: get(
fCountry,
"info.name_se",
req.body.filters.recipient_country_code[0]
req.body.filters[AF_COUNTRY][0]
),
news: newsData.map((n: any) => ({
title: n.title,
Expand Down Expand Up @@ -243,18 +252,23 @@ export function detailPageName(req: any, res: any) {
}
}
if (
req.body.detail_type === "participating_org_ref" &&
req.body.detail_type === AF_PARTICIPATING_ORG_REF &&
result.length === 0
) {
const refIndex = findIndex(
data.participating_org_ref,
(ref: string) => ref === req.body.filters.participating_org_ref[0]
data[AF_PARTICIPATING_ORG_REF],
(ref: string) =>
ref === req.body.filters[AF_PARTICIPATING_ORG_REF][0]
);
result = get(
data,
`${AF_PARTICIPATING_ORG_NARRATIVE}[${refIndex}]`,
""
);
result = get(data, `participating_org_narrative[${refIndex}]`, "");
const fOrgMapping = find(orgMapping, {
code: parseInt(
req.body.filters.participating_org_ref[
req.body.filters.participating_org_ref.length - 1
req.body.filters[AF_PARTICIPATING_ORG_REF][
req.body.filters[AF_PARTICIPATING_ORG_REF].length - 1
],
10
)
Expand All @@ -263,42 +277,41 @@ export function detailPageName(req: any, res: any) {
result = fOrgMapping.info.name;
}
}
if (req.body.detail_type === "recipient_region_code") {
if (req.body.detail_type === AF_REGION) {
const refIndex = findIndex(
data.recipient_region_code,
(ref: string) => ref === req.body.filters.recipient_region_code[0]
data[AF_REGION],
(ref: string) => ref === req.body.filters[AF_REGION][0]
);
result = get(data, `recipient_region_name[${refIndex}]`, "");
result = get(data, `${AF_REGION}[${refIndex}]`, "");
}
if (req.body.detail_type === "tag_narrative") {
result = req.body.filters.tag_narrative[0];
if (req.body.detail_type === AF_TAG_NARRATIVE) {
result = req.body.filters[AF_TAG_NARRATIVE][0];
// get(
// thematicAreaNames,
// req.body.filters.tag_narrative[0].replace("|", ","),
// ""
// ).replace(" is the main priority area in this activity", "");
}
if (req.body.detail_type === "participating_org_type") {
if (req.body.detail_type === AF_PARTICIPATING_ORG_TYPE) {
result = get(
find(orgTypesCodelist, {
code: req.body.filters.participating_org_type[0]
code: req.body.filters[AF_PARTICIPATING_ORG_TYPE][0]
}),
"name",
""
);
}
}
if (
req.body.detail_type !== "recipient_country_code" &&
req.body.detail_type !== "sector_code"
req.body.detail_type !== AF_COUNTRY &&
req.body.detail_type !== AF_SECTOR
) {
res.json({
data: [result]
});
}
})
.catch(errors => {
console.log(errors);
genericError(errors, res);
});
}
Loading

0 comments on commit 35105f4

Please sign in to comment.