Add melodymae feed (WIP)

app/normalizers/melodymae_normalizer.rb

@@ -0,0 +1,9 @@
+class MelodymaeNormalizer < FeedjiraNormalizer
+  def comments
+    []
+  end
+
+  def attachments
+    [Html.first_image_url(content.content.to_s)]
+  end
+end

config/feeds.yml

@@ -784,13 +784,21 @@
   loader: "http"
   processor: "feedjira"
   normalizer: "elementy"
-  refresh_interval: 3600
-  import_limit: 10
+  refresh_interval: 7200
+  import_limit: 2
 
 - name: elementy-kartinka-dnya
   url: "https://elementy.ru/rss/kartinka_dnya"
   loader: "http"
   processor: "feedjira"
   normalizer: "elementy"
-  refresh_interval: 3600
+  refresh_interval: 7200
+  import_limit: 2
+
+- name: melodymae
+  url: "https://www.melodymae.co.uk/feed/"
+  loader: "http"
+  processor: "feedjira"
+  normalizer: "melodymae"
+  refresh_interval: 14400
   import_limit: 10

scripts/feed_setup.rb

@@ -0,0 +1,77 @@
+# TBD: Refactor script structure if/after the flow will get stabilized
+# :reek:DuplicateMethodCall:
+# :reek:TooManyStatements
+class FeedSetup
+  include ActionView::Helpers::NumberHelper
+
+  def perform
+    feed_name = ARGV[0]
+
+    unless feed_name
+      puts "Specify feed name to setup"
+      exit 1
+    end
+
+    puts "feed name: #{feed_name}"
+
+    config_path = Rails.root.join(FeedsConfiguration::DEFAULT_PATH)
+    config_data = YAML.load_file(config_path).map { FeedSanitizer.call(**_1.symbolize_keys) }
+
+    feed = config_data.find { _1[:name] == feed_name }
+
+    if feed
+      puts "feed configuration:"
+      puts JSON.pretty_generate(feed)
+    else
+      puts "configuration not found"
+      exit 1
+    end
+
+    feed = feed[:attributes]
+
+    touch_or_create_file(subject: "normalizer", path: "app/normalizers/#{feed[:normalizer]}_normalizer.rb")
+    touch_or_create_file(subject: "normalizer spec", path: "spec/normalizers/#{feed[:normalizer]}_normalizer_spec.rb")
+    touch_or_create_file(subject: "processor", path: "app/processors/#{feed[:processor]}_processor.rb")
+    touch_or_create_file(subject: "processor spec", path: "spec/processors/#{feed[:processor]}_processor_spec.rb")
+
+    feed_url = feed[:url]
+
+    if feed_url.present?
+      puts "downloading feed content (#{feed_url})"
+      content = HTTP.get(feed_url).to_s
+      touch_or_create_file(subject: "feed fixture", path: "spec/fixtures/files/feeds/#{feed_name}/feed.xml", content: content)
+    else
+      puts "feed URL not defined, skipping the fixturing"
+    end
+  end
+
+  private
+
+  def touch_or_create_file(subject:, path:, content: nil)
+    if path.blank?
+      puts "#{subject} path undefined"
+      return
+    end
+
+    path = Rails.root.join(path)
+
+    if File.exist?(path)
+      puts "#{subject}: #{path} (exists)"
+    else
+      puts "#{subject}: #{path} (will be created)"
+      FileUtils.mkdir_p(File.dirname(path))
+      FileUtils.touch(path)
+    end
+
+    if content.present?
+      if File.read(path).blank?
+        puts "writing content (length: #{number_to_human_size(content.to_s.length)})"
+        File.write(path, content)
+      else
+        puts "ignoring content (file not empty)"
+      end
+    end
+
+    true
+  end
+end

spec/fixtures/files/feeds/melodymae/normalized.json

@@ -0,0 +1,162 @@
+[
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/plus-size-alternative-retro/",
+    "link": "https://www.melodymae.co.uk/plus-size-alternative-retro/",
+    "published_at": "2022-02-26 12:00:00 +0000",
+    "text": "Plus Size Alternative Retro With Belle Poque - https://www.melodymae.co.uk/plus-size-alternative-retro/",
+    "attachments": [
+      "https://i1.wp.com/www.melodymae.co.uk/wp-content/uploads/2022/02/5372C1A6-11AE-4BCF-B225-DB49D0F928E2_1_201_a.jpeg?resize=819%2C1024&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  },
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/plus-size-date-outfits/",
+    "link": "https://www.melodymae.co.uk/plus-size-date-outfits/",
+    "published_at": "2022-02-12 12:00:00 +0000",
+    "text": "Plus Size Date Outfits with SHEIN! - https://www.melodymae.co.uk/plus-size-date-outfits/",
+    "attachments": [
+      "https://i0.wp.com/www.melodymae.co.uk/wp-content/uploads/2022/02/980262E8-FCF4-4A3E-8AF7-EEA5DACA5A64_1_201_a.jpeg?resize=819%2C1024&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  },
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/handmade-plus-size-style/",
+    "link": "https://www.melodymae.co.uk/handmade-plus-size-style/",
+    "published_at": "2022-01-09 12:00:00 +0000",
+    "text": "Into The Swing of It — Handmade Plus Size Style - https://www.melodymae.co.uk/handmade-plus-size-style/",
+    "attachments": [
+      "https://i2.wp.com/www.melodymae.co.uk/wp-content/uploads/2022/01/21ACAA10-D1CB-4638-8257-DB210C7843B6_1_201_a.jpeg?resize=819%2C1024&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  },
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/2021-in-review/",
+    "link": "https://www.melodymae.co.uk/2021-in-review/",
+    "published_at": "2022-01-02 12:20:52 +0000",
+    "text": "2021 In Review; Turned out 2020 Could Get Worse! - https://www.melodymae.co.uk/2021-in-review/",
+    "attachments": [
+      "https://i2.wp.com/www.melodymae.co.uk/wp-content/uploads/2022/01/3.png?resize=1024%2C576&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  },
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/the-mental-health-struggle/",
+    "link": "https://www.melodymae.co.uk/the-mental-health-struggle/",
+    "published_at": "2021-12-19 12:00:00 +0000",
+    "text": "The Joys of the Mental Health Struggle - https://www.melodymae.co.uk/the-mental-health-struggle/",
+    "attachments": [
+      "https://i1.wp.com/www.melodymae.co.uk/wp-content/uploads/2021/12/B800447B-51A4-45B0-A784-789BD2AE3974_1_201_a.jpeg?resize=1024%2C576&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  },
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/ahead-of-the-curve-by-cashmerette/",
+    "link": "https://www.melodymae.co.uk/ahead-of-the-curve-by-cashmerette/",
+    "published_at": "2021-10-17 08:00:00 +0000",
+    "text": "Ahead of the Curve by Cashmerette; Plus Size Sewing - https://www.melodymae.co.uk/ahead-of-the-curve-by-cashmerette/",
+    "attachments": [
+      "https://i0.wp.com/www.melodymae.co.uk/wp-content/uploads/2021/10/BFA0FDBD-30DC-42A9-808A-85CCCC95C6AD_1_201_a.jpeg?resize=1024%2C576&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  },
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/vendula-london-aw21-collection/",
+    "link": "https://www.melodymae.co.uk/vendula-london-aw21-collection/",
+    "published_at": "2021-10-10 08:00:00 +0000",
+    "text": "Favourites From the Vendula London AW21 Collection! - https://www.melodymae.co.uk/vendula-london-aw21-collection/",
+    "attachments": [
+      "https://i1.wp.com/www.melodymae.co.uk/wp-content/uploads/2021/10/78F3D1AC-C45A-44C1-BCAF-0FC253B8BF06_1_201_a.jpeg?resize=819%2C1024&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  },
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/ww-plus-size-autumn/",
+    "link": "https://www.melodymae.co.uk/ww-plus-size-autumn/",
+    "published_at": "2021-09-30 08:00:00 +0000",
+    "text": "Hello Autumn! Plus Size Autumn Style Wishlist - https://www.melodymae.co.uk/ww-plus-size-autumn/",
+    "attachments": [
+      "https://i0.wp.com/www.melodymae.co.uk/wp-content/uploads/2021/09/2.png?resize=819%2C1024&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  },
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/plus-size-hell-bunny/",
+    "link": "https://www.melodymae.co.uk/plus-size-hell-bunny/",
+    "published_at": "2021-09-25 10:32:55 +0000",
+    "text": "Late Summer Plus Size Hell Bunny Haul! - https://www.melodymae.co.uk/plus-size-hell-bunny/",
+    "attachments": [
+      "https://i1.wp.com/www.melodymae.co.uk/wp-content/uploads/2021/09/39888DD6-D122-46AC-AB9F-CC87EEDE14A9_1_201_a.jpeg?resize=819%2C1024&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  },
+  {
+    "feed_id": null,
+    "uid": "https://www.melodymae.co.uk/plus-size-lyra-dress/",
+    "link": "https://www.melodymae.co.uk/plus-size-lyra-dress/",
+    "published_at": "2021-09-13 08:00:00 +0000",
+    "text": "Another Gingham Dress — Plus Size Lyra Dress - https://www.melodymae.co.uk/plus-size-lyra-dress/",
+    "attachments": [
+      "https://i2.wp.com/www.melodymae.co.uk/wp-content/uploads/2021/09/DSC0090.jpeg?resize=1024%2C576&ssl=1"
+    ],
+    "comments": [
+
+    ],
+    "validation_errors": [
+
+    ]
+  }
+]

spec/normalizers/melodymae_normalizer_spec.rb

@@ -0,0 +1,33 @@
+require "rails_helper"
+
+RSpec.describe MelodymaeNormalizer do
+  subject(:normalizer) { described_class }
+
+  let(:feed) do
+    build(
+      :feed,
+      name: "melodymae",
+      import_limit: 0,
+      processor: "feedjira",
+      normalizer: "melodymae"
+    )
+  end
+
+  let(:feed_content) { file_fixture("feeds/#{feed.name}/feed.xml").read }
+  let(:feed_entities) { feed.processor_class.new(content: feed_content, feed: feed).process }
+
+  let(:expected) do
+    json = file_fixture("feeds/#{feed.name}/normalized.json").read
+    JSON.parse(json).map { _1.merge("feed_id" => feed.id) }
+  end
+
+  it "resolves normalizer class" do
+    expect(feed.normalizer_class).to eq(normalizer)
+  end
+
+  it "matches the expected result" do
+    result = feed_entities.map { normalizer.call(_1).as_json }
+
+    expect(result).to eq(expected)
+  end
+end