Code cleanup.

.gitignore

@@ -2,6 +2,4 @@ node_modules
 csv
 writes
 test-writes
-db/ETL
-ETL
 mongodata

db/ETL/pipelines/reviewsPipeline.js → db/ETL/extract-pipelines/reviewsPipeline.js

@@ -32,5 +32,3 @@ pipeline(inputStream, csvParser, transformStream, outputStream, (err) => {
   }
 });
 
-// console.log('!!!', Number("Fri Nov 13 2020 13:34:42 GMT-0500 (Eastern Standard Time)"));
-

db/ETL/transforms/dateTransform.js

@@ -11,10 +11,4 @@ module.exports.dateTransform = (reviewObject) => {
 
   return reviewObject;
 
-}
-
-// const obj = {
-//   date: '1606045468366'
-// }
-
-// dateTransform(obj);
+}

db/index.js

@@ -2,7 +2,7 @@ const mongoose = require('mongoose');
 
 const PORT = 27017;
 
-const db = mongoose.connect(`mongodb://db:${PORT}/SDC`, { useNewUrlParser: true })
+const db = mongoose.connect(`mongodb://localhost:${PORT}/SDC`, { useNewUrlParser: true })
   .then((connection) => {
     console.log('------ DB connected ------')
     console.log('Collections: ', Object.keys(mongoose.connection.collections));

db/load-pipelines/db-load-reviews.js

@@ -0,0 +1,34 @@
+const ProductReview = require('../models/Reviews.js');
+const fs = require('fs');
+const { pipeline } = require('stream');
+const split = require('split');
+
+
+const reviewsReadStream = fs.createReadStream('../etl/writes/realReviews.json');
+
+reviewsReadStream.pipe(split(JSON.parse))
+  .on('data', (reviewObj) => {
+    const newReview = new ProductReview({
+      product_id: reviewObj.product_id,
+      review_id: reviewObj.id,
+      rating: reviewObj.rating,
+      date: reviewObj.date,
+      summary: reviewObj.summary,
+      body: reviewObj.body,
+      recommend: reviewObj.recommend,
+      reported: reviewObj.reported,
+      reviewer_name: reviewObj.reviewer_name,
+      reviewer_email: reviewObj.reviewer_email,
+      response: reviewObj.response,
+      helpfulness: reviewObj.helpfulness,
+      photos: []
+    }).save().catch((err) => {console.log(err.message)});
+
+  })
+  .on('error', (err) => {
+    console.log(err);
+  })
+  .on('end', () => {
+    console.log('///Reviews loading completed///');
+  })
+

db/models/CharacteristicReviews.js

@@ -1,6 +1,5 @@
 const mongoose = require('mongoose');
 const db = require('../index.js');
-const fs = require('fs');
 
 const characteristicReviewSchema = new mongoose.Schema({
   characteristic_id: {type: Number, required: true},
@@ -11,22 +10,4 @@ const characteristicReviewSchema = new mongoose.Schema({
 const CharacteristicReviews = mongoose.model('CharacteristicReviews', characteristicReviewSchema);
 
 
-
-module.exports = CharacteristicReviews;
-
-/*
-CSV:
-
-id,characteristic_id,review_id,value
-1,1,1,4
-2,2,1,3
-3,3,1,5
-4,4,1,4
-5,1,2,4
-6,2,2,4
-7,3,2,5
-8,4,2,4
-9,5,3,4
-10,5,4,5
-
-*/
+module.exports = CharacteristicReviews;

db/models/Characteristics.js

@@ -1,6 +1,5 @@
 const mongoose = require('mongoose');
 const db = require('../index.js');
-const fs = require('fs');
 
 const characteristicSchema = new mongoose.Schema({
   characteristic_id: {type: Number, required: true, unique: true},
@@ -11,18 +10,4 @@ const characteristicSchema = new mongoose.Schema({
 const Characteristics = mongoose.model('Characteristics', characteristicSchema);
 
 
-
 module.exports = Characteristics;
-
-/*
-CSV:
-
-id,product_id,name
-1,1,"Fit"
-2,1,"Length"
-3,1,"Comfort"
-4,1,"Quality"
-5,2,"Quality"
-6,3,"Fit"
-
-*/

db/models/GroupedReviews.js

@@ -10,5 +10,4 @@ const GroupedReviews = mongoose.model('reviewsGroupedByProductId', groupedReview
 
 
 
-
 module.exports = GroupedReviews;

db/models/ProductMeta.js

@@ -10,4 +10,5 @@ const productMetaSchema = new mongoose.Schema({
 
 const ProductMeta = mongoose.model('metaData-merged', productMetaSchema);
 
+
 module.exports = ProductMeta;

db/models/ReviewPhotos.js

@@ -1,6 +1,5 @@
 const mongoose = require('mongoose');
 const db = require('../index.js');
-const fs = require('fs');
 
 const reviewPhotosSchema = new mongoose.Schema({
   review_id: {type: Number, unique: true},
@@ -10,5 +9,4 @@ const reviewPhotosSchema = new mongoose.Schema({
 const ReviewPhotos = mongoose.model('ReviewPhotos', reviewPhotosSchema);
 
 
-
 module.exports = ReviewPhotos;

db/models/Reviews.js

@@ -1,7 +1,5 @@
 const mongoose = require('mongoose');
 const db = require('../index.js');
-const fs = require('fs');
-const { pipeline } = require('stream');
 
 const productReviewSchema = new mongoose.Schema({
   product_id: {type: Number, required: true},
@@ -23,5 +21,4 @@ const ProductReview = mongoose.models.ProductReview || mongoose.model('ProductRe
 
 
 
-
 module.exports = ProductReview;

package.json

@@ -4,7 +4,6 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
     "start": "nodemon ./server/index.js"
   },
   "repository": {
@@ -31,7 +30,6 @@
     "uuid": "^8.3.2"
   },
   "devDependencies": {
-    "eslint": "^7.25.0",
     "nodemon": "^2.0.7"
   }
 }

server/index.js

@@ -9,8 +9,6 @@ const PORT = 8000;
 
 app.use(express.json());
 
-
-
 app.use('/reviews/meta', meta);
 app.use('/reviews', reviews);
 app.get('/loaderio-2ae007f3d5a6844c03745ca34aa724a2/', (req, res) => {

server/routes/metaRoutes.js

@@ -7,8 +7,6 @@ router.get('/', (req, res) => {
 
   const { product_id } = req.query;
 
-  //TODO: explore saving changes on first fetch to save time on subsequent fetches.
-
   ProductMeta.findOne({product_id})
     .then((metaData) => {
       const inputCharac = metaData.characteristics;
@@ -30,56 +28,5 @@ router.get('/', (req, res) => {
 
 });
 
-/*
-
-"characteristics": [
-  {
-      "name": "Length",
-      "avgValue": 3.5,
-      "count": 2
-  },
-]
-
-{
-  ...,
-"characteristics": {
-    "Fit": {
-        "id": 36825,
-        "value": "3.3571428571428571"
-    },
-    "Length": {
-        "id": 36826,
-        "value": "3.1428571428571429"
-    },
-    "Comfort": {
-        "id": 36827,
-        "value": "3.6428571428571429"
-    },
-    "Quality": {
-        "id": 36828,
-        "value": "3.7647058823529412"
-    }
-  }
-}
-*/
-
-/* When fetching data from /reviews/meta, the product_id should be included in the request as params
-
-Example request:
-
-axios.get('/reviews/meta', {
-  params: {
-    product_id: 11002
-  }
-})
-  .then(({data}) => {
-    console.log(data)
-  })
-  .catch((err) => {
-    console.log(err);
-  })
-}
-*/
-
 
 module.exports = router;