diff --git a/.gitignore b/.gitignore index fda2ff5..80ed9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,4 @@ node_modules csv writes test-writes -db/ETL -ETL mongodata diff --git a/db/ETL/pipelines/characteristicReviewsPipeline.js b/db/ETL/extract-pipelines/characteristicReviewsPipeline.js similarity index 100% rename from db/ETL/pipelines/characteristicReviewsPipeline.js rename to db/ETL/extract-pipelines/characteristicReviewsPipeline.js diff --git a/db/ETL/pipelines/characteristicsPipeline.js b/db/ETL/extract-pipelines/characteristicsPipeline.js similarity index 100% rename from db/ETL/pipelines/characteristicsPipeline.js rename to db/ETL/extract-pipelines/characteristicsPipeline.js diff --git a/db/ETL/pipelines/reviewPhotosPipeline.js b/db/ETL/extract-pipelines/reviewPhotosPipeline.js similarity index 100% rename from db/ETL/pipelines/reviewPhotosPipeline.js rename to db/ETL/extract-pipelines/reviewPhotosPipeline.js diff --git a/db/ETL/pipelines/reviewsPipeline.js b/db/ETL/extract-pipelines/reviewsPipeline.js similarity index 90% rename from db/ETL/pipelines/reviewsPipeline.js rename to db/ETL/extract-pipelines/reviewsPipeline.js index 97e574d..631feaf 100644 --- a/db/ETL/pipelines/reviewsPipeline.js +++ b/db/ETL/extract-pipelines/reviewsPipeline.js @@ -32,5 +32,3 @@ pipeline(inputStream, csvParser, transformStream, outputStream, (err) => { } }); -// console.log('!!!', Number("Fri Nov 13 2020 13:34:42 GMT-0500 (Eastern Standard Time)")); - diff --git a/db/ETL/transforms/dateTransform.js b/db/ETL/transforms/dateTransform.js index 3b78067..1ff174e 100644 --- a/db/ETL/transforms/dateTransform.js +++ b/db/ETL/transforms/dateTransform.js @@ -11,10 +11,4 @@ module.exports.dateTransform = (reviewObject) => { return reviewObject; -} - -// const obj = { -// date: '1606045468366' -// } - -// dateTransform(obj); +} \ No newline at end of file diff --git a/db/index.js b/db/index.js index 5803263..7f920db 100644 --- a/db/index.js +++ b/db/index.js @@ -2,7 +2,7 @@ const mongoose = require('mongoose'); const PORT = 27017; -const db = mongoose.connect(`mongodb://db:${PORT}/SDC`, { useNewUrlParser: true }) +const db = mongoose.connect(`mongodb://localhost:${PORT}/SDC`, { useNewUrlParser: true }) .then((connection) => { console.log('------ DB connected ------') console.log('Collections: ', Object.keys(mongoose.connection.collections)); diff --git a/db/loaders/db-load-characteristicReviews.js b/db/load-pipelines/db-load-characteristicReviews.js similarity index 100% rename from db/loaders/db-load-characteristicReviews.js rename to db/load-pipelines/db-load-characteristicReviews.js diff --git a/db/loaders/db-load-characteristics.js b/db/load-pipelines/db-load-characteristics.js similarity index 100% rename from db/loaders/db-load-characteristics.js rename to db/load-pipelines/db-load-characteristics.js diff --git a/db/loaders/db-load-reviewPhotos.js b/db/load-pipelines/db-load-reviewPhotos.js similarity index 100% rename from db/loaders/db-load-reviewPhotos.js rename to db/load-pipelines/db-load-reviewPhotos.js diff --git a/db/load-pipelines/db-load-reviews.js b/db/load-pipelines/db-load-reviews.js new file mode 100644 index 0000000..fa1e850 --- /dev/null +++ b/db/load-pipelines/db-load-reviews.js @@ -0,0 +1,34 @@ +const ProductReview = require('../models/Reviews.js'); +const fs = require('fs'); +const { pipeline } = require('stream'); +const split = require('split'); + + +const reviewsReadStream = fs.createReadStream('../etl/writes/realReviews.json'); + +reviewsReadStream.pipe(split(JSON.parse)) + .on('data', (reviewObj) => { + const newReview = new ProductReview({ + product_id: reviewObj.product_id, + review_id: reviewObj.id, + rating: reviewObj.rating, + date: reviewObj.date, + summary: reviewObj.summary, + body: reviewObj.body, + recommend: reviewObj.recommend, + reported: reviewObj.reported, + reviewer_name: reviewObj.reviewer_name, + reviewer_email: reviewObj.reviewer_email, + response: reviewObj.response, + helpfulness: reviewObj.helpfulness, + photos: [] + }).save().catch((err) => {console.log(err.message)}); + + }) + .on('error', (err) => { + console.log(err); + }) + .on('end', () => { + console.log('///Reviews loading completed///'); + }) + diff --git a/db/loaders/db-load-reviews.js b/db/loaders/db-load-reviews.js deleted file mode 100644 index 0176c99..0000000 --- a/db/loaders/db-load-reviews.js +++ /dev/null @@ -1,116 +0,0 @@ -const ProductReview = require('../models/Reviews.js'); -const fs = require('fs'); -const { pipeline } = require('stream'); -const split = require('split'); - - -const reviewsReadStream = fs.createReadStream('../etl/writes/realReviews.json'); -// const reviewPhotosStream = fs.createReadStream('../etl/writes/test-writes/dummyReviewPhotos.ndjson'); - -reviewsReadStream.pipe(split(JSON.parse)) - .on('data', (reviewObj) => { - // console.log(reviewObj); - const newReview = new ProductReview({ - product_id: reviewObj.product_id, - review_id: reviewObj.id, - rating: reviewObj.rating, - date: reviewObj.date, - summary: reviewObj.summary, - body: reviewObj.body, - recommend: reviewObj.recommend, - reported: reviewObj.reported, - reviewer_name: reviewObj.reviewer_name, - reviewer_email: reviewObj.reviewer_email, - response: reviewObj.response, - helpfulness: reviewObj.helpfulness, - photos: [] - }).save().catch((err) => {console.log(err.message)}); - - }) - .on('error', (err) => { - console.log(err); - }) - .on('end', () => { - console.log('///Reviews loading completed///'); - }) - - - - - - - - - -/* - -const ProductReview = require('./Reviews.js'); -const fs = require('fs'); -const { pipeline } = require('stream'); -const split = require('split'); - - -const reviewsReadStream = fs.createReadStream('../etl/writes/test-writes/dummyReviews.ndjson'); -const reviewPhotosStream = fs.createReadStream('../etl/writes/test-writes/dummyReviewPhotos.ndjson'); - -reviewsReadStream.pipe(split(JSON.parse)) - .on('data', (reviewObj) => { - const product_id = reviewObj.product_id; - const newReview = { - product_id: product_id, - review_id: reviewObj.id, - rating: reviewObj.rating, - date: reviewObj.date, - summary: reviewObj.summary, - body: reviewObj.body, - recommend: reviewObj.recommend, - reported: reviewObj.reported, - reviewer_name: reviewObj.reviewer_name, - reviewer_email: reviewObj.reviewer_email, - response: reviewObj.response, - helpfulness: reviewObj.helpfulness, - photos: [] - }; - - ProductReview.findOneAndUpdate( - {product_id}, - { - product_id, - $push: { - results: newReview - } - }, - {upsert: true} - - ) - .catch((err) => { - console.log(err); - }); - - }) - .on('error', (err) => { - console.log(err); - }) - .on('end', () => { - console.log('Process completed'); - }) - - -// reviewPhotosStream.pipe(split(JSON.parse)) -// .on('data', (obj) => { -// const targetReviewId = obj.review_id; -// const photoURL = obj.url; - -// ProductReview.findOneAndUpdate( -// { -// results.review_id: targetReviewId -// }, -// { -// $push: { -// results.photos: photoURL; -// } -// } -// ) -// }) - -*/ \ No newline at end of file diff --git a/db/models/CharacteristicReviews.js b/db/models/CharacteristicReviews.js index 8a0bdb1..071877d 100644 --- a/db/models/CharacteristicReviews.js +++ b/db/models/CharacteristicReviews.js @@ -1,6 +1,5 @@ const mongoose = require('mongoose'); const db = require('../index.js'); -const fs = require('fs'); const characteristicReviewSchema = new mongoose.Schema({ characteristic_id: {type: Number, required: true}, @@ -11,22 +10,4 @@ const characteristicReviewSchema = new mongoose.Schema({ const CharacteristicReviews = mongoose.model('CharacteristicReviews', characteristicReviewSchema); - -module.exports = CharacteristicReviews; - -/* -CSV: - -id,characteristic_id,review_id,value -1,1,1,4 -2,2,1,3 -3,3,1,5 -4,4,1,4 -5,1,2,4 -6,2,2,4 -7,3,2,5 -8,4,2,4 -9,5,3,4 -10,5,4,5 - -*/ \ No newline at end of file +module.exports = CharacteristicReviews; \ No newline at end of file diff --git a/db/models/Characteristics.js b/db/models/Characteristics.js index 4d4d12c..6904335 100644 --- a/db/models/Characteristics.js +++ b/db/models/Characteristics.js @@ -1,6 +1,5 @@ const mongoose = require('mongoose'); const db = require('../index.js'); -const fs = require('fs'); const characteristicSchema = new mongoose.Schema({ characteristic_id: {type: Number, required: true, unique: true}, @@ -11,18 +10,4 @@ const characteristicSchema = new mongoose.Schema({ const Characteristics = mongoose.model('Characteristics', characteristicSchema); - module.exports = Characteristics; - -/* -CSV: - -id,product_id,name -1,1,"Fit" -2,1,"Length" -3,1,"Comfort" -4,1,"Quality" -5,2,"Quality" -6,3,"Fit" - -*/ \ No newline at end of file diff --git a/db/models/GroupedReviews.js b/db/models/GroupedReviews.js index ed06caa..f288250 100644 --- a/db/models/GroupedReviews.js +++ b/db/models/GroupedReviews.js @@ -10,5 +10,4 @@ const GroupedReviews = mongoose.model('reviewsGroupedByProductId', groupedReview - module.exports = GroupedReviews; \ No newline at end of file diff --git a/db/models/ProductMeta.js b/db/models/ProductMeta.js index 6730835..6e15ce6 100644 --- a/db/models/ProductMeta.js +++ b/db/models/ProductMeta.js @@ -10,4 +10,5 @@ const productMetaSchema = new mongoose.Schema({ const ProductMeta = mongoose.model('metaData-merged', productMetaSchema); + module.exports = ProductMeta; \ No newline at end of file diff --git a/db/models/ReviewPhotos.js b/db/models/ReviewPhotos.js index 42578b5..d85cfaa 100644 --- a/db/models/ReviewPhotos.js +++ b/db/models/ReviewPhotos.js @@ -1,6 +1,5 @@ const mongoose = require('mongoose'); const db = require('../index.js'); -const fs = require('fs'); const reviewPhotosSchema = new mongoose.Schema({ review_id: {type: Number, unique: true}, @@ -10,5 +9,4 @@ const reviewPhotosSchema = new mongoose.Schema({ const ReviewPhotos = mongoose.model('ReviewPhotos', reviewPhotosSchema); - module.exports = ReviewPhotos; \ No newline at end of file diff --git a/db/models/Reviews.js b/db/models/Reviews.js index 923f5a3..4790c21 100644 --- a/db/models/Reviews.js +++ b/db/models/Reviews.js @@ -1,7 +1,5 @@ const mongoose = require('mongoose'); const db = require('../index.js'); -const fs = require('fs'); -const { pipeline } = require('stream'); const productReviewSchema = new mongoose.Schema({ product_id: {type: Number, required: true}, @@ -23,5 +21,4 @@ const ProductReview = mongoose.models.ProductReview || mongoose.model('ProductRe - module.exports = ProductReview; \ No newline at end of file diff --git a/package.json b/package.json index afaaefe..ddde49c 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,6 @@ "description": "", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", "start": "nodemon ./server/index.js" }, "repository": { @@ -31,7 +30,6 @@ "uuid": "^8.3.2" }, "devDependencies": { - "eslint": "^7.25.0", "nodemon": "^2.0.7" } } diff --git a/server/index.js b/server/index.js index 1a92754..e30ab40 100644 --- a/server/index.js +++ b/server/index.js @@ -9,8 +9,6 @@ const PORT = 8000; app.use(express.json()); - - app.use('/reviews/meta', meta); app.use('/reviews', reviews); app.get('/loaderio-2ae007f3d5a6844c03745ca34aa724a2/', (req, res) => { diff --git a/server/routes/metaRoutes.js b/server/routes/metaRoutes.js index 18534c0..94dbb0e 100644 --- a/server/routes/metaRoutes.js +++ b/server/routes/metaRoutes.js @@ -7,8 +7,6 @@ router.get('/', (req, res) => { const { product_id } = req.query; - //TODO: explore saving changes on first fetch to save time on subsequent fetches. - ProductMeta.findOne({product_id}) .then((metaData) => { const inputCharac = metaData.characteristics; @@ -30,56 +28,5 @@ router.get('/', (req, res) => { }); -/* - -"characteristics": [ - { - "name": "Length", - "avgValue": 3.5, - "count": 2 - }, -] - -{ - ..., -"characteristics": { - "Fit": { - "id": 36825, - "value": "3.3571428571428571" - }, - "Length": { - "id": 36826, - "value": "3.1428571428571429" - }, - "Comfort": { - "id": 36827, - "value": "3.6428571428571429" - }, - "Quality": { - "id": 36828, - "value": "3.7647058823529412" - } - } -} -*/ - -/* When fetching data from /reviews/meta, the product_id should be included in the request as params - -Example request: - -axios.get('/reviews/meta', { - params: { - product_id: 11002 - } -}) - .then(({data}) => { - console.log(data) - }) - .catch((err) => { - console.log(err); - }) -} -*/ - module.exports = router; \ No newline at end of file diff --git a/server/routes/reviewsRoutes.js b/server/routes/reviewsRoutes.js index 5821cf5..7549e47 100644 --- a/server/routes/reviewsRoutes.js +++ b/server/routes/reviewsRoutes.js @@ -81,6 +81,7 @@ router.put('/:review_id/report', (req, res) => { }) }); + //TODO: refactor - DRY router.put('/:review_id/helpful', (req, res) => { const review_id = Number(req.params.review_id); @@ -120,7 +121,6 @@ router.put('/:review_id/helpful', (req, res) => { }); - router.post('/', (req, res) => { // EXTRACT REVIEW DETAILS FROM REQ BODY const { @@ -228,40 +228,9 @@ module.exports = router; /* - { - product_id: integer, - rating: integer, - summary: string, - body: string, - recommend: bool, - name: string, - email: string, - photos: [url, url, url], - characteristics: { - characteristic_id: int, - characteristic_id: int - characteristic_id: int - } - } - - { - product_id: 1, - rating: 5, - summary: 'a new review by Pep', - body: 'This is a test review, newly created by Pep.', - recommend: true, - name: PepPep, - email: Pep@pepmail.com, - photos: ['https://cdn.buttercms.com/ZF8K2t8hT8OoNR3W42bX', 'https://cdn.buttercms.com/ZF8K2t8hT8OoNR3W42bX'], - characteristics: { - 1: 5, - 2: 5, - 3: 5, - 4: 5 - } - } - - {"product_id":1,"rating":5,"summary":"a new review by Pep","body":"This is a test review, newly created by Pep.","recommend":true,"name":"PepPep","email":"Pep@pepmail.com","photos":["https://cdn.buttercms.com/ZF8K2t8hT8OoNR3W42bX","https://cdn.buttercms.com/ZF8K2t8hT8OoNR3W42bX"],"characteristics":{"1":5,"2":5,"3":5,"4":5}} + JSON for a dummy post: + + {"product_id":1,"rating":5,"summary":"A new review by some user","body":"This is a test review, newly created by some user.","recommend":true,"name":"SomeUser","email":"user@gmail.com","photos":["https://cdn.buttercms.com/ZF8K2t8hT8OoNR3W42bX","https://cdn.buttercms.com/ZF8K2t8hT8OoNR3W42bX"],"characteristics":{"1":5,"2":5,"3":5,"4":5}} */