Merge branch 'main' of https://github.com/mage-os/mirror-commerce-dat…

…a-export
mage-os · Nov 1, 2023 · 1ee0a2b · 1ee0a2b
2 parents 1e78c97 + 21dad30
commit 1ee0a2b
Show file tree

Hide file tree

Showing 9 changed files with 79 additions and 9 deletions.
diff --git a/CatalogDataExporter/etc/db_schema.xml b/CatalogDataExporter/etc/db_schema.xml
@@ -77,6 +77,10 @@
             <column name="modified_at"/>
             <column name="id"/>
         </index>
+        <index referenceId="catalog_data_exporter_products_modified_at_status" indexType="btree">
+            <column name="modified_at"/>
+            <column name="status"/>
+        </index>
     </table>
 
     <table name="catalog_data_exporter_categories" resource="default" engine="innodb" comment="Category Feed Storage">

diff --git a/CatalogDataExporter/etc/db_schema_whitelist.json b/CatalogDataExporter/etc/db_schema_whitelist.json
@@ -1,7 +1,8 @@
 {
     "catalog_data_exporter_products": {
         "index": {
-            "CATALOG_DATA_EXPORTER_PRODUCTS_MODIFIED_AT": true
+            "CATALOG_DATA_EXPORTER_PRODUCTS_MODIFIED_AT": true,
+            "CATALOG_DATA_EXPORTER_PRODUCTS_MODIFIED_AT_STATUS": true
         },
         "constraint": {
             "PRIMARY": true

diff --git a/DataExporter/Model/FeedHashBuilder.php b/DataExporter/Model/FeedHashBuilder.php
@@ -8,6 +8,7 @@
 namespace Magento\DataExporter\Model;
 
 use Magento\DataExporter\Model\Indexer\FeedIndexMetadata;
+use Magento\DataExporter\Model\Logging\CommerceDataExportLoggerInterface;
 use Magento\Framework\App\ResourceConnection;
 use Magento\Framework\Serialize\SerializerInterface;
 
@@ -18,17 +19,21 @@ class FeedHashBuilder
 {
     private SerializerInterface $serializer;
     private ResourceConnection $resourceConnection;
+    private CommerceDataExportLoggerInterface $logger;
 
     /**
      * @param SerializerInterface $serializer
      * @param ResourceConnection $resourceConnection
+     * @param CommerceDataExportLoggerInterface $logger
      */
     public function __construct(
         SerializerInterface $serializer,
-        ResourceConnection $resourceConnection
+        ResourceConnection $resourceConnection,
+        CommerceDataExportLoggerInterface $logger
     ) {
         $this->serializer = $serializer;
         $this->resourceConnection = $resourceConnection;
+        $this->logger = $logger;
     }
 
     /**
@@ -71,7 +76,15 @@ public function buildIdentifierFromFeedItem(array $feedItem, FeedIndexMetadata $
     {
         $identifier = [];
         foreach ($metadata->getFeedIdentifierMappingFields() as $field) {
-            $this->addValue($identifier, array_key_exists($field, $feedItem) ? (string)$feedItem[$field] : '');
+            $value = array_key_exists($field, $feedItem) ? (string)$feedItem[$field] : null;
+            if ($value === null) {
+                $this->logger->error(
+                    "Cannot build identifier for '$field' from feed item: " . var_export($feedItem, true)
+                );
+                continue;
+            }
+
+            $this->addValue($identifier, $value);
         }
         return $this->convertToString($identifier);
     }
@@ -87,7 +100,14 @@ public function buildIdentifierFromFeedTableRow(array $row, FeedIndexMetadata $m
     {
         $identifier = [];
         foreach (array_keys($metadata->getFeedIdentifierMappingFields()) as $columnName) {
-            $this->addValue($identifier, array_key_exists($columnName, $row) ? (string)$row[$columnName] : '');
+            $value = array_key_exists($columnName, $row) ? (string)$row[$columnName] : null;
+            if ($value === null) {
+                $this->logger->error(
+                    "Cannot build identifier for '$columnName' from feed table: " . var_export($row, true)
+                );
+                continue;
+            }
+            $this->addValue($identifier, $value);
         }
         return $this->convertToString($identifier);
     }
@@ -114,4 +134,4 @@ private function convertToString(array $identifier): string
     {
         return implode(',', $identifier);
     }
-}
+}
diff --git a/DataExporter/Model/Indexer/FeedIndexProcessorCreateUpdate.php b/DataExporter/Model/Indexer/FeedIndexProcessorCreateUpdate.php
@@ -190,6 +190,9 @@ private function truncateIndexTable(FeedIndexMetadata $metadata): void
      */
     private function filterFeedItems(array $feedItems, FeedIndexMetadata $metadata, &$processedHashes = null) : array
     {
+        if (empty($feedItems)) {
+            return [];
+        }
         $connection = $this->resourceConnection->getConnection();
         $primaryKeyFields = $this->getFeedTablePrimaryKey($metadata);
         $primaryKeys = \array_keys($feedItems);
@@ -235,6 +238,7 @@ private function addHashes(array $data, FeedIndexMetadata $metadata, bool $delet
             if ($deleted) {
                 if (!isset($row[FeedIndexMetadata::FEED_TABLE_FIELD_FEED_HASH])) {
                     $this->logger->error("Feed hash is not set for the product id: ". $row['productId']);
+                    unset($data[$key]);
                     continue ;
                 }
                 $identifier = $this->hashBuilder->buildIdentifierFromFeedTableRow($row, $metadata);
@@ -247,7 +251,16 @@ private function addHashes(array $data, FeedIndexMetadata $metadata, bool $delet
                 $identifier = $this->hashBuilder->buildIdentifierFromFeedItem($row, $metadata);
             }
             unset($data[$key]);
-
+            if (empty($identifier)) {
+                $this->logger->error(
+                    'Identifier for feed item is empty. Skip sync for entity',
+                    [
+                        'feed' => $metadata->getFeedName(),
+                        'item' => var_export($row, true)
+                    ]
+                );
+                continue;
+            }
             $hash = $this->hashBuilder->buildHash($row, $metadata);
             $this->addModifiedAtField($row, $metadata);
             $data[$identifier] = [

diff --git a/ProductPriceDataExporter/Model/Query/CustomerGroupPricesQuery.php b/ProductPriceDataExporter/Model/Query/CustomerGroupPricesQuery.php
@@ -171,7 +171,8 @@ private function getCatalogRulePricesSelect(array $productIds): Select
             ->joinLeft(
                 ['tier' => $this->resourceConnection->getTableName('catalog_product_entity_tier_price')],
                 \sprintf('product.%1$s = tier.%1$s', $this->getLinkField()) .
-                ' AND tier.qty=1 AND tier.website_id in (0, product_website.website_id)',
+                ' AND tier.qty=1 AND tier.all_groups = 0 AND tier.customer_group_id = rule.customer_group_id '
+                . 'AND tier.website_id in (0, product_website.website_id)',
                 []
             )
             ->columns([

diff --git a/ProductPriceDataExporter/Model/Query/ProductPricesQuery.php b/ProductPriceDataExporter/Model/Query/ProductPricesQuery.php
@@ -8,7 +8,6 @@
 namespace Magento\ProductPriceDataExporter\Model\Query;
 
 use Magento\Catalog\Api\Data\ProductInterface;
-use Magento\Catalog\Model\Product\Type;
 use Magento\ConfigurableProduct\Model\Product\Type\Configurable;
 use Magento\Framework\App\ResourceConnection;
 use Magento\Framework\DB\Select;
@@ -131,6 +130,8 @@ public function getQuery(array $productIds, array $priceAttributes = []): Select
             )
             ->where('product.entity_id IN (?)', $productIds)
             ->where('product.type_id NOT IN (?)', self::IGNORED_TYPES)
+            // exclude "admin" website
+            ->where('store_website.website_id != ?', 0)
             ->order('product.entity_id')
             ->order('product_website.website_id')
             ->order('eav_store.attribute_id')

diff --git a/ProductPriceDataExporter/etc/db_schema.xml b/ProductPriceDataExporter/etc/db_schema.xml
@@ -78,5 +78,9 @@
             <column name="modified_at"/>
             <column name="product_id"/>
         </index>
+        <index referenceId="catalog_data_exporter_product_prices_modified_at_status" indexType="btree">
+            <column name="modified_at"/>
+            <column name="status"/>
+        </index>
     </table>
 </schema>
diff --git a/ProductPriceDataExporter/etc/db_schema_whitelist.json b/ProductPriceDataExporter/etc/db_schema_whitelist.json
@@ -1,7 +1,8 @@
 {
     "catalog_data_exporter_product_prices": {
         "index": {
-            "CATALOG_DATA_EXPORTER_PRODUCT_PRICES_MODIFIED_AT": true
+            "CATALOG_DATA_EXPORTER_PRODUCT_PRICES_MODIFIED_AT": true,
+            "CATALOG_DATA_EXPORTER_PRODUCT_PRICES_MODIFIED_AT_STATUS": true
         },
         "constraint": {
             "PRIMARY": true

diff --git a/.../testsuite/Magento/Setup/Declaration/_files/ignore_whitelisting/commerce-data-export.json b/.../testsuite/Magento/Setup/Declaration/_files/ignore_whitelisting/commerce-data-export.json
@@ -0,0 +1,25 @@
+{
+  "catalog_data_exporter_products": {
+    "constraint": [
+      "PRIMARY"
+    ],
+    "index": [
+      "CATALOG_DATA_EXPORTER_PRODUCTS_MODIFIED_AT_ID",
+      "CATALOG_DATA_EXPORTER_PRODUCTS_MODIFIED_AT_STATUS"
+    ]
+  },
+  "catalog_data_exporter_product_prices": {
+    "constraint": [
+      "PRIMARY"
+    ],
+    "index": [
+      "CATALOG_DATA_EXPORTER_PRODUCT_PRICES_MODIFIED_AT_PRODUCT_ID",
+      "CATALOG_DATA_EXPORTER_PRODUCT_PRICES_MODIFIED_AT_STATUS"
+    ]
+  },
+  "catalog_data_submitted_hash": {
+    "constraint": [
+      "PRIMARY"
+    ]
+  }
+}