From 5e05ef6239a62da9baab72a7eecff9a75cd267ca Mon Sep 17 00:00:00 2001 From: Galia Ruppin Date: Tue, 10 Dec 2024 11:51:52 +0200 Subject: [PATCH] fix iplan url query --- server/api/controller/cron.js | 38 +++++++------- server/api/lib/iplanApi.js | 86 +++++++++++++++---------------- server/api/model/plan.js | 2 +- server/bin/iplan | 20 ++++---- server/metrics.js | 95 ++++++++++++++++++----------------- 5 files changed, 121 insertions(+), 120 deletions(-) diff --git a/server/api/controller/cron.js b/server/api/controller/cron.js index 1a6d93c3..c7e67ee1 100644 --- a/server/api/controller/cron.js +++ b/server/api/controller/cron.js @@ -20,13 +20,13 @@ const PlanAreaChangesController = require('../controller/plan_area_changes'); const getPlanTagger = require('../lib/tags'); const PlanStatusChange = require('../model/plan_status_change'); const { meirimStatuses } = require('../constants'); -const { report } = require('../../metrics') +const { report } = require('../../metrics'); const iplan = (limit = -1) => iplanApi .getBlueLines() .then(iPlans => { - report({metricName: "iplane.bluelines.count", value: iPlans.length}) + report({ metricName: 'iplane.bluelines.count', value: iPlans.length }); // limit blue lines found so we output only *limit* plans if (limit > -1) { @@ -35,7 +35,7 @@ const iplan = (limit = -1) => return Bluebird.mapSeries(iPlans, iPlan => fetchIplan(iPlan)); }).catch(e => { - Log.error(`Error fetching new plans`, e); + Log.error('Error fetching new plans', e); }); const fix_geodata = () => { @@ -69,27 +69,27 @@ const complete_mavat_data = () => }) .fetchAll() .then(planCollection => { - Log.info({ message: "found incomplete mavat data", length: planCollection.length }) + Log.info({ message: 'found incomplete mavat data', length: planCollection.length }); return planCollection; }) .then(planCollection => - Bluebird.mapSeries(planCollection.models, plan => { - Log.info({ message: "about to work on plan", id: plan.get("id"), url: plan.get('plan_url') }) + Bluebird.mapSeries(planCollection.models, plan => { + Log.info({ message: 'about to work on plan', id: plan.get('id'), url: plan.get('plan_url') }); return MavatAPI.getByPlan(plan) .then(mavatData => { - Log.info({message: 'Saving with mavat',data: JSON.stringify(mavatData)}); + Log.info({ message: 'Saving with mavat',data: JSON.stringify(mavatData) }); return Plan.setMavatData(plan, mavatData).then(Promise.all([plan.save(), PlanAreaChangesController.refreshPlanAreaChanges(plan.id, plan.attributes.areaChanges) - ])) + ])); }) .catch((e) => { - Log.error({message: "failure in complete_mavat_data", e, id: plan.get("id")}) + Log.error({ message: 'failure in complete_mavat_data', e, id: plan.get('id') }); // do nothing on error }).finally(() => { - Log.info({ message: "finish working on plan", id: plan.get("id") }) - }) + Log.info({ message: 'finish working on plan', id: plan.get('id') }); + }); }) ); @@ -121,7 +121,7 @@ const sendPlanningAlerts = () => { limit: 1 }) .then(unsentPlans => { - report({ metricName: "planning_alerts.unsent", value: unsentPlans.models.length }) + report({ metricName: 'planning_alerts.unsent', value: unsentPlans.models.length }); Log.debug('Got', unsentPlans.models.length, 'Plans'); return unsentPlans.models; }) @@ -418,8 +418,8 @@ const fetchPlanStatus = () => { const dateString = moment(date).format('YYYY-MM-DD h:mm'); return Plan.query(qb => { qb.leftJoin('status_mapping', 'plan.status', '=' ,'status_mapping.mavat_status') - .whereRaw(`(status_mapping.meirim_status is null or status_mapping.meirim_status != '${meirimStatuses.APPROVED}') and plan.MP_ID NOT LIKE \'NOT_FOUND\' and (plan.last_visited_status < '${dateString}' OR plan.last_visited_status IS NULL)`) - .orderBy('plan.last_visited_status','asc'); + .whereRaw(`(status_mapping.meirim_status is null or status_mapping.meirim_status != '${meirimStatuses.APPROVED}') and plan.MP_ID NOT LIKE \'NOT_FOUND\' and (plan.last_visited_status < '${dateString}' OR plan.last_visited_status IS NULL)`) + .orderBy('plan.last_visited_status','asc'); qb.limit(planStatusLimit); }) .fetchAll() @@ -427,8 +427,8 @@ const fetchPlanStatus = () => { Bluebird.mapSeries(planCollection.models, plan => { Log.info({ - planId: plan.get("id") , - message: "working on plan", + planId: plan.get('id') , + message: 'working on plan', last_visited_status: plan.get('last_visited_status'), status:plan.get('status'), }); @@ -461,7 +461,7 @@ const fetchPlanStatus = () => { return; } const mostRecentStatus = mostRecent[0].attributes.status; - Log.info({message: 'updating plan status', status: mostRecentStatus, planId: plan.get('id'), oldStatus: plan.get('status')}); + Log.info({ message: 'updating plan status', status: mostRecentStatus, planId: plan.get('id'), oldStatus: plan.get('status') }); await plan.save({ 'last_visited_status': now , 'status': mostRecentStatus }); // save all plan statuses into plan_status_change table @@ -475,13 +475,13 @@ const fetchPlanStatus = () => { await sendEmailIfNeeded(plan, planStatuses, mavatStatus); } catch (err) { - Log.error({err}); + Log.error({ err }); const now = moment().format('YYYY-MM-DD HH:mm:ss'); await plan.save({ 'last_visited_status': now }); } }) .catch(async (err)=> { - Log.error({err}); + Log.error({ err }); const now = moment().format('YYYY-MM-DD HH:mm:ss'); await plan.save({ 'last_visited_status': now }); }); diff --git a/server/api/lib/iplanApi.js b/server/api/lib/iplanApi.js index 72550689..3c49673d 100644 --- a/server/api/lib/iplanApi.js +++ b/server/api/lib/iplanApi.js @@ -2,7 +2,6 @@ const axios = require('axios'); const GeoJSON = require('esri-to-geojson'); const Bluebird = require('bluebird'); const _ = require('lodash'); -const { map, chunk, reduce, extend, get } = require('lodash'); // const proj4 = require('proj4'); const reproject = require('reproject'); const Config = require('../lib/config'); @@ -13,7 +12,7 @@ const BASE_AGS_URL = // "https://ags.iplan.gov.il/arcgis/rest/services/" + // "PlanningPublic/Xplan_2039/MapServer"; -const MAVAT_SERVICE_ID = 1; +//const MAVAT_SERVICE_ID = 1; const options = { rejectUnauthorized: false, @@ -23,39 +22,39 @@ const options = { json: true }; -const fields = [ - 'objectid', - 'shape', - // 'plan_area_code', - // 'jurstiction_code', - 'plan_county_name', - // 'plan_county_code', - 'entity_subtype_desc', - 'pl_number', - 'pl_name', - 'pl_area_dunam', - 'depositing_date', - 'mp_id', - // 'DATE_SAF', - // 'PL_LAST_DEPOSIT_DATE', - // 'PL_REJECTION_DATE', - // 'PLAN_CHARACTOR_NAME', - // 'מטרות', - // 'PQ_AUTHORISED_QUANTITY_110', - // 'PQ_AUTHORISED_QUANTITY_120', - 'pl_date_8', - 'pl_landuse_string', - // 'station', - 'station_desc', - 'pl_by_auth_of', - 'pl_url', - 'shape_area', - 'quantity_delta_120', - 'quantity_delta_125', - 'last_update', - 'pl_order_print_version', - 'pl_tasrit_prn_version' -]; +// const fields = [ +// 'objectid', +// 'shape', +// // 'plan_area_code', +// // 'jurstiction_code', +// 'plan_county_name', +// // 'plan_county_code', +// 'entity_subtype_desc', +// 'pl_number', +// 'pl_name', +// 'pl_area_dunam', +// 'depositing_date', +// 'mp_id', +// // 'DATE_SAF', +// // 'PL_LAST_DEPOSIT_DATE', +// // 'PL_REJECTION_DATE', +// // 'PLAN_CHARACTOR_NAME', +// // 'מטרות', +// // 'PQ_AUTHORISED_QUANTITY_110', +// // 'PQ_AUTHORISED_QUANTITY_120', +// 'pl_date_8', +// 'pl_landuse_string', +// // 'station', +// 'station_desc', +// 'pl_by_auth_of', +// 'pl_url', +// 'shape_area', +// 'quantity_delta_120', +// 'quantity_delta_125', +// 'last_update', +// 'pl_order_print_version', +// 'pl_tasrit_prn_version' +// ]; // const EPSG2039 = proj4.Proj( // '+proj=tmerc +lat_0=31.73439361111111 +lon_0=35.20451694444445 +k=1.0000067 +x_0=219529.584 +y_0=626907.39 +ellps=GRS80 +towgs84=-48,55,52,0,0,0,0 +units=m +no_defs' @@ -65,10 +64,13 @@ const EPSG3857 = '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs'; // TODO: save the links of the new website and the old website -const buildMavatURL = (serviceId, fieldsToFill, whereClause, return_geom) => { - return `${BASE_AGS_URL}/${serviceId}/query?f=json&outFields=${fieldsToFill.join( - ',' - )}&returnGeometry=${return_geom}&where=${whereClause}&orderByFields=last_update DESC&outSR=3857`; +//const buildMavatURL = (serviceId, fieldsToFill, whereClause, return_geom) +const buildMavatURL = () => { + // return `${BASE_AGS_URL}/${serviceId}/query?f=json&outFields=${fieldsToFill.join( + // ',' + // )}&returnGeometry=${return_geom}&where=${whereClause}&outSR=3857`; + + return 'https://ags.iplan.gov.il/arcgisiplan/rest/services/PlanningPublic/Xplan/MapServer/1/query?f=json&outFields=objectid,shape,plan_county_name,entity_subtype_desc,pl_number&returnGeometry=true&where=objectid%3E0&outSR=3857'; }; @@ -82,12 +84,12 @@ const getBlueLines = async () => { // we need MP_ID field to know the id in the new mavat website. // xplan doesn't have MP_ID in the polygons API (service id of 0) // so we query the centroid API (service id of 1) as well in order to get MP_ID. - const urlWithPolygons = buildMavatURL(MAVAT_SERVICE_ID, fields, 'objectid > 0', 'true'); - + const urlWithPolygons = buildMavatURL(); + Log.info(`urlWithPolygons: ${urlWithPolygons}`); try { const responseWithPolygons = await axios.get(urlWithPolygons, options); const geojson = GeoJSON.fromEsri(responseWithPolygons.data, {}); - Log.info('mavat plans', {plans: geojson.features.length}); + Log.info('mavat plans', { plans: geojson.features.length }); // Need to populate all plans with their MP_ID // TODO- export to a mapping function diff --git a/server/api/model/plan.js b/server/api/model/plan.js index 052a93e2..f1cc0dad 100644 --- a/server/api/model/plan.js +++ b/server/api/model/plan.js @@ -367,7 +367,7 @@ class Plan extends Model { PLAN_COUNTY_NAME: iPlan.properties.PLAN_COUNTY_NAME || '', PL_NUMBER: iPlan.properties.PL_NUMBER || '', PL_NAME: iPlan.properties.PL_NAME || '', - plan_display_name: Plan.cleanPlanName(iPlan.properties.PL_NAME), + plan_display_name: iPlan.properties.PL_NAME? Plan.cleanPlanName(iPlan.properties.PL_NAME): '', // 'PLAN_CHARACTOR_NAME': iPlan.properties.PLAN_CHARACTOR_NAME || '', data: iPlan.properties, geom: iPlan.geometry, diff --git a/server/bin/iplan b/server/bin/iplan index cc1a0948..dedcd89e 100644 --- a/server/bin/iplan +++ b/server/bin/iplan @@ -2,17 +2,15 @@ const controller = require('../api/controller/cron'); const Log = require('../api/lib/log'); -const axios = require('axios'); -const Config = require('../api/lib/config'); -const { runAndReport } = require('../metrics') +const { runAndReport } = require('../metrics'); -runAndReport({ func: () => { -return controller - .iplan() - .then(async () => { - Log.info('iplan completed'); - }) - .finally(() => process.exit()); -}, name: "iplan"}) +runAndReport({ + func: () => { + return controller.iplan().then(async () => { + Log.info('iplan completed'); + }) + .finally(() => process.exit()); + }, name: 'iplan' +}); diff --git a/server/metrics.js b/server/metrics.js index 58ec3983..6311a924 100644 --- a/server/metrics.js +++ b/server/metrics.js @@ -4,9 +4,9 @@ const { Resource } = require('@opentelemetry/resources'); const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions'); const { Metadata } = require('@grpc/grpc-js'); const Log = require('./api/lib/log'); -const Config = require("config") +const Config = require('config'); -const env = process.env.NODE_ENV +const env = process.env.NODE_ENV; const apikey = Config.get('coralogix.apikey'); const url = Config.get('coralogix.metricsEndpoint'); const serviceName = Config.get('coralogix.serviceName'); @@ -16,69 +16,70 @@ const metadata = new Metadata(); metadata.add('Authorization', 'Bearer ' + apikey); const collectorOptions = { - url, - metadata: metadata + url, + metadata: metadata }; const metricExporter = new OTLPMetricExporter(collectorOptions); const meterProvider = new MeterProvider({ - resource: new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: serviceName, - }), + resource: new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: serviceName, + }), }); meterProvider.addMetricReader(new PeriodicExportingMetricReader({ - exporter: metricExporter, - exportIntervalMillis: 100, + exporter: metricExporter, + exportIntervalMillis: 100, })); ['SIGINT', 'SIGTERM'].forEach(signal => { - process.on(signal, () => meterProvider.shutdown().catch(console.error)); + process.on(signal, () => meterProvider.shutdown().catch(console.error)); }); const meter = meterProvider.getMeter('meirim'); const report = async ({ metricName, value = 1, attributes = {} }) => { - try { - const h = meter.createHistogram(metricName) - h.record(value, { - ...attributes, - env, - }) - } catch (err) { - console.error("failed to report metric", err); - } + try { + const h = meter.createHistogram(metricName); + h.record(value, { + ...attributes, + env, + }); + } catch (err) { + console.error('failed to report metric', err); + } }; -async function runAndReport({name, func, timeout = 60*60*1000 }) { - try { - await Promise.race( - [func(), - new Promise((resolve, reject) => { - setTimeout(() => { - reject(new Error("timeout from runAndReport")) - },timeout) - })]) - report({ - metricName: "job", - attributes: {result: "success", "job-id": name} - }) - await new Promise((r) => setTimeout(r, 3000)) - process.exit(); - } catch (e) { - report({ - metricName: "job", - attributes: {result: "failed", "job-id": name} - }) - await new Promise((r) => setTimeout(r, 3000)) - Log.error("failed to run - ", name, e) - process.exit(1); - } -} +async function runAndReport({ name, func, timeout = 60 * 60 * 1000 }) { + try { + await Promise.race( + [func(), + new Promise((resolve, reject) => { + setTimeout(() => { + reject(new Error('timeout from runAndReport')); + }, timeout); + })] + ); + report({ + metricName: 'job', + attributes: { result: 'success', 'job-id': name } + }); + await new Promise((r) => setTimeout(r, 3000)); + process.exit(); + } catch (e) { + report({ + metricName: 'job', + attributes: { result: 'failed', 'job-id': name } + }); + await new Promise((r) => setTimeout(r, 3000)); + Log.error('failed to run - ', name, e); + process.exit(1); + } +} module.exports = { - report, - runAndReport -} + report, + runAndReport +};