Skip to content

Commit

Permalink
fix iplan url query
Browse files Browse the repository at this point in the history
  • Loading branch information
@gruppin
gruppin committed Dec 10, 2024
1 parent 6daec0a commit 5e05ef6
Show file tree
Hide file tree
Showing 5 changed files with 121 additions and 120 deletions.
38 changes: 19 additions & 19 deletions server/api/controller/cron.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,13 @@ const PlanAreaChangesController = require('../controller/plan_area_changes');
const getPlanTagger = require('../lib/tags');
const PlanStatusChange = require('../model/plan_status_change');
const { meirimStatuses } = require('../constants');
const { report } = require('../../metrics')
const { report } = require('../../metrics');

const iplan = (limit = -1) =>
iplanApi
.getBlueLines()
.then(iPlans => {
report({metricName: "iplane.bluelines.count", value: iPlans.length})
report({ metricName: 'iplane.bluelines.count', value: iPlans.length });

// limit blue lines found so we output only *limit* plans
if (limit > -1) {
Expand All @@ -35,7 +35,7 @@ const iplan = (limit = -1) =>

return Bluebird.mapSeries(iPlans, iPlan => fetchIplan(iPlan));
}).catch(e => {
Log.error(`Error fetching new plans`, e);
Log.error('Error fetching new plans', e);
});

const fix_geodata = () => {
Expand Down Expand Up @@ -69,27 +69,27 @@ const complete_mavat_data = () =>
})
.fetchAll()
.then(planCollection => {
Log.info({ message: "found incomplete mavat data", length: planCollection.length })
Log.info({ message: 'found incomplete mavat data', length: planCollection.length });
return planCollection;
})
.then(planCollection =>

Bluebird.mapSeries(planCollection.models, plan => {
Log.info({ message: "about to work on plan", id: plan.get("id"), url: plan.get('plan_url') })
Bluebird.mapSeries(planCollection.models, plan => {
Log.info({ message: 'about to work on plan', id: plan.get('id'), url: plan.get('plan_url') });

return MavatAPI.getByPlan(plan)
.then(mavatData => {
Log.info({message: 'Saving with mavat',data: JSON.stringify(mavatData)});
Log.info({ message: 'Saving with mavat',data: JSON.stringify(mavatData) });
return Plan.setMavatData(plan, mavatData).then(Promise.all([plan.save(),
PlanAreaChangesController.refreshPlanAreaChanges(plan.id, plan.attributes.areaChanges)
]))
]));
})
.catch((e) => {
Log.error({message: "failure in complete_mavat_data", e, id: plan.get("id")})
Log.error({ message: 'failure in complete_mavat_data', e, id: plan.get('id') });
// do nothing on error
}).finally(() => {
Log.info({ message: "finish working on plan", id: plan.get("id") })
})
Log.info({ message: 'finish working on plan', id: plan.get('id') });
});
})
);

Expand Down Expand Up @@ -121,7 +121,7 @@ const sendPlanningAlerts = () => {
limit: 1
})
.then(unsentPlans => {
report({ metricName: "planning_alerts.unsent", value: unsentPlans.models.length })
report({ metricName: 'planning_alerts.unsent', value: unsentPlans.models.length });
Log.debug('Got', unsentPlans.models.length, 'Plans');
return unsentPlans.models;
})
Expand Down Expand Up @@ -418,17 +418,17 @@ const fetchPlanStatus = () => {
const dateString = moment(date).format('YYYY-MM-DD h:mm');
return Plan.query(qb => {
qb.leftJoin('status_mapping', 'plan.status', '=' ,'status_mapping.mavat_status')
.whereRaw(`(status_mapping.meirim_status is null or status_mapping.meirim_status != '${meirimStatuses.APPROVED}') and plan.MP_ID NOT LIKE \'NOT_FOUND\' and (plan.last_visited_status < '${dateString}' OR plan.last_visited_status IS NULL)`)
.orderBy('plan.last_visited_status','asc');
.whereRaw(`(status_mapping.meirim_status is null or status_mapping.meirim_status != '${meirimStatuses.APPROVED}') and plan.MP_ID NOT LIKE \'NOT_FOUND\' and (plan.last_visited_status < '${dateString}' OR plan.last_visited_status IS NULL)`)
.orderBy('plan.last_visited_status','asc');
qb.limit(planStatusLimit);
})
.fetchAll()
.then(planCollection =>
Bluebird.mapSeries(planCollection.models, plan => {

Log.info({
planId: plan.get("id") ,
message: "working on plan",
planId: plan.get('id') ,
message: 'working on plan',
last_visited_status: plan.get('last_visited_status'),
status:plan.get('status'),
});
Expand Down Expand Up @@ -461,7 +461,7 @@ const fetchPlanStatus = () => {
return;
}
const mostRecentStatus = mostRecent[0].attributes.status;
Log.info({message: 'updating plan status', status: mostRecentStatus, planId: plan.get('id'), oldStatus: plan.get('status')});
Log.info({ message: 'updating plan status', status: mostRecentStatus, planId: plan.get('id'), oldStatus: plan.get('status') });
await plan.save({ 'last_visited_status': now , 'status': mostRecentStatus });

// save all plan statuses into plan_status_change table
Expand All @@ -475,13 +475,13 @@ const fetchPlanStatus = () => {
await sendEmailIfNeeded(plan, planStatuses, mavatStatus);
}
catch (err) {
Log.error({err});
Log.error({ err });
const now = moment().format('YYYY-MM-DD HH:mm:ss');
await plan.save({ 'last_visited_status': now });
}
})
.catch(async (err)=> {
Log.error({err});
Log.error({ err });
const now = moment().format('YYYY-MM-DD HH:mm:ss');
await plan.save({ 'last_visited_status': now });
});
Expand Down
86 changes: 44 additions & 42 deletions server/api/lib/iplanApi.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ const axios = require('axios');
const GeoJSON = require('esri-to-geojson');
const Bluebird = require('bluebird');
const _ = require('lodash');
const { map, chunk, reduce, extend, get } = require('lodash');
// const proj4 = require('proj4');
const reproject = require('reproject');
const Config = require('../lib/config');
Expand All @@ -13,7 +12,7 @@ const BASE_AGS_URL =
// "https://ags.iplan.gov.il/arcgis/rest/services/" +
// "PlanningPublic/Xplan_2039/MapServer";

const MAVAT_SERVICE_ID = 1;
//const MAVAT_SERVICE_ID = 1;

const options = {
rejectUnauthorized: false,
Expand All @@ -23,39 +22,39 @@ const options = {
json: true
};

const fields = [
'objectid',
'shape',
// 'plan_area_code',
// 'jurstiction_code',
'plan_county_name',
// 'plan_county_code',
'entity_subtype_desc',
'pl_number',
'pl_name',
'pl_area_dunam',
'depositing_date',
'mp_id',
// 'DATE_SAF',
// 'PL_LAST_DEPOSIT_DATE',
// 'PL_REJECTION_DATE',
// 'PLAN_CHARACTOR_NAME',
// 'מטרות',
// 'PQ_AUTHORISED_QUANTITY_110',
// 'PQ_AUTHORISED_QUANTITY_120',
'pl_date_8',
'pl_landuse_string',
// 'station',
'station_desc',
'pl_by_auth_of',
'pl_url',
'shape_area',
'quantity_delta_120',
'quantity_delta_125',
'last_update',
'pl_order_print_version',
'pl_tasrit_prn_version'
];
// const fields = [
// 'objectid',
// 'shape',
// // 'plan_area_code',
// // 'jurstiction_code',
// 'plan_county_name',
// // 'plan_county_code',
// 'entity_subtype_desc',
// 'pl_number',
// 'pl_name',
// 'pl_area_dunam',
// 'depositing_date',
// 'mp_id',
// // 'DATE_SAF',
// // 'PL_LAST_DEPOSIT_DATE',
// // 'PL_REJECTION_DATE',
// // 'PLAN_CHARACTOR_NAME',
// // 'מטרות',
// // 'PQ_AUTHORISED_QUANTITY_110',
// // 'PQ_AUTHORISED_QUANTITY_120',
// 'pl_date_8',
// 'pl_landuse_string',
// // 'station',
// 'station_desc',
// 'pl_by_auth_of',
// 'pl_url',
// 'shape_area',
// 'quantity_delta_120',
// 'quantity_delta_125',
// 'last_update',
// 'pl_order_print_version',
// 'pl_tasrit_prn_version'
// ];

// const EPSG2039 = proj4.Proj(
// '+proj=tmerc +lat_0=31.73439361111111 +lon_0=35.20451694444445 +k=1.0000067 +x_0=219529.584 +y_0=626907.39 +ellps=GRS80 +towgs84=-48,55,52,0,0,0,0 +units=m +no_defs'
Expand All @@ -65,10 +64,13 @@ const EPSG3857 =
'+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs';

// TODO: save the links of the new website and the old website
const buildMavatURL = (serviceId, fieldsToFill, whereClause, return_geom) => {
return `${BASE_AGS_URL}/${serviceId}/query?f=json&outFields=${fieldsToFill.join(
','
)}&returnGeometry=${return_geom}&where=${whereClause}&orderByFields=last_update DESC&outSR=3857`;
//const buildMavatURL = (serviceId, fieldsToFill, whereClause, return_geom)
const buildMavatURL = () => {
// return `${BASE_AGS_URL}/${serviceId}/query?f=json&outFields=${fieldsToFill.join(
// ','
// )}&returnGeometry=${return_geom}&where=${whereClause}&outSR=3857`;

return 'https://ags.iplan.gov.il/arcgisiplan/rest/services/PlanningPublic/Xplan/MapServer/1/query?f=json&outFields=objectid,shape,plan_county_name,entity_subtype_desc,pl_number&returnGeometry=true&where=objectid%3E0&outSR=3857';
};


Expand All @@ -82,12 +84,12 @@ const getBlueLines = async () => {
// we need MP_ID field to know the id in the new mavat website.
// xplan doesn't have MP_ID in the polygons API (service id of 0)
// so we query the centroid API (service id of 1) as well in order to get MP_ID.
const urlWithPolygons = buildMavatURL(MAVAT_SERVICE_ID, fields, 'objectid > 0', 'true');

const urlWithPolygons = buildMavatURL();
Log.info(`urlWithPolygons: ${urlWithPolygons}`);
try {
const responseWithPolygons = await axios.get(urlWithPolygons, options);
const geojson = GeoJSON.fromEsri(responseWithPolygons.data, {});
Log.info('mavat plans', {plans: geojson.features.length});
Log.info('mavat plans', { plans: geojson.features.length });

// Need to populate all plans with their MP_ID
// TODO- export to a mapping function
Expand Down
2 changes: 1 addition & 1 deletion server/api/model/plan.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,7 +367,7 @@ class Plan extends Model {
PLAN_COUNTY_NAME: iPlan.properties.PLAN_COUNTY_NAME || '',
PL_NUMBER: iPlan.properties.PL_NUMBER || '',
PL_NAME: iPlan.properties.PL_NAME || '',
plan_display_name: Plan.cleanPlanName(iPlan.properties.PL_NAME),
plan_display_name: iPlan.properties.PL_NAME? Plan.cleanPlanName(iPlan.properties.PL_NAME): '',
// 'PLAN_CHARACTOR_NAME': iPlan.properties.PLAN_CHARACTOR_NAME || '',
data: iPlan.properties,
geom: iPlan.geometry,
Expand Down
20 changes: 9 additions & 11 deletions server/bin/iplan
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,15 @@

const controller = require('../api/controller/cron');
const Log = require('../api/lib/log');
const axios = require('axios');
const Config = require('../api/lib/config');
const { runAndReport } = require('../metrics')
const { runAndReport } = require('../metrics');


runAndReport({ func: () => {
return controller
.iplan()
.then(async () => {
Log.info('iplan completed');
})
.finally(() => process.exit());
}, name: "iplan"})
runAndReport({
func: () => {
return controller.iplan().then(async () => {
Log.info('iplan completed');
})
.finally(() => process.exit());
}, name: 'iplan'
});

Loading

0 comments on commit 5e05ef6

Please sign in to comment.