Add tags to representation document (#6309)

* Read tags from new field

* Sync analysis tags into search index document

* Update app/commands/exercise/representation/create_search_index_document.rb

Co-authored-by: Jeremy Walker <jez.walker@gmail.com>

* Add memoization

* Fix rubocop

* Add Exercise::Tag and Solution::Tag

* Store exercise and solution tags

* Process tags.json file

* Prevent loading of relationship

* Change order of columns in migration

* Add bugsnags when process tooling job output

* Remove n+1 in updating exercise tags

* Remove n+1 in updating solution tags

* Fix test runner flow tests

---------

Co-authored-by: Jeremy Walker <jez.walker@gmail.com>

app/commands/exercise/representation/create_search_index_document.rb

@@ -25,6 +25,7 @@ def call
       num_solutions: representation.num_published_solutions,
       code: published_iteration.submission.files.map(&:content) || [],
       max_reputation:,
+      tags:,
       exercise: {
         id: solution.exercise.id,
         slug: solution.exercise.slug,
@@ -61,5 +62,20 @@ def max_reputation
     ).maximum(:reputation).to_i
   end
 
+  def tags
+    return [] if last_analyzed_submission_representation.nil?
+
+    last_analyzed_submission_representation.submission.analysis.tags
+  end
+
+  memoize
+  def last_analyzed_submission_representation
+    representation.
+      submission_representations.
+      joins(submission: :analysis).
+      where(submission: { analysis_status: :completed }).
+      last
+  end
+
   attr_reader :solution, :published_iteration
 end

app/commands/exercise/update_tags.rb

@@ -0,0 +1,20 @@
+class Exercise::UpdateTags
+  include Mandate
+
+  initialize_with :exercise
+
+  def call = exercise.update(tags:)
+
+  private
+  def tags
+    solution_tags = Solution::Tag.where(exercise:).distinct.pluck(:tag)
+    existing_tags = Exercise::Tag.where(exercise:).where(tag: solution_tags).select(:id, :tag).to_a
+    exercise_tags = existing_tags.map(&:tag)
+
+    new_tags = (solution_tags - exercise_tags).map do |tag|
+      Exercise::Tag.find_or_create_by!(tag:, exercise:)
+    end
+
+    existing_tags + new_tags
+  end
+end

app/commands/solution/publish_iteration.rb

@@ -6,6 +6,7 @@ class Solution::PublishIteration
   def call
     solution.update!(published_iteration: iteration)
 
+    Solution::UpdateTags.(solution)
     Solution::UpdatePublishedExerciseRepresentation.(solution)
     Solution::UpdateSnippet.(solution)
     Solution::UpdateNumLoc.(solution)

app/commands/solution/unpublish.rb

@@ -5,6 +5,7 @@ class Solution::Unpublish
 
   def call
     solution.update!(published_iteration_id: nil, published_at: nil)
+    Solution::UpdateTags.(solution)
     Solution::UpdatePublishedExerciseRepresentation.(solution)
     Solution::UpdateSnippet.(solution)
     Solution::UpdateNumLoc.(solution)

app/commands/solution/update_tags.rb

@@ -0,0 +1,28 @@
+class Solution::UpdateTags
+  include Mandate
+
+  initialize_with :solution
+
+  def call
+    solution.update(tags:)
+    Exercise::UpdateTags.(solution.exercise)
+  end
+
+  private
+  def tags
+    return [] if latest_analysis.nil?
+
+    analysis_tags = latest_analysis.tags
+    existing_tags = Solution::Tag.where(solution:).where(tag: analysis_tags).select(:id, :tag).to_a
+    solution_tags = existing_tags.map(&:tag)
+
+    new_tags = (analysis_tags - solution_tags).map do |tag|
+      Solution::Tag.find_or_create_by!(tag:, solution:)
+    end
+
+    existing_tags + new_tags
+  end
+
+  memoize
+  def latest_analysis = solution.latest_published_iteration_submission&.analysis
+end

app/commands/submission/analysis/process.rb

@@ -9,7 +9,8 @@ def call
     analysis = submission.create_analysis!(
       tooling_job_id: tooling_job.id,
       ops_status: tooling_job.execution_status.to_i,
-      data:
+      data:,
+      tags_data:
     )
 
     begin
@@ -39,6 +40,7 @@ def handle_ops_error!
 
   def handle_completed!
     submission.analysis_completed!
+    Solution::UpdateTags.(submission.solution)
   end
 
   memoize
@@ -50,7 +52,20 @@ def submission
   def data
     res = JSON.parse(tooling_job.execution_output['analysis.json'])
     res.is_a?(Hash) ? res.symbolize_keys : {}
-  rescue StandardError
+  rescue StandardError => e
+    Bugsnag.notify(e)
+    {}
+  end
+
+  memoize
+  def tags_data
+    tags_json = tooling_job.execution_output['tags.json']
+    return {} if tags_json.empty?
+
+    res = JSON.parse(tags_json)
+    res.is_a?(Hash) ? res.symbolize_keys : {}
+  rescue StandardError => e
+    Bugsnag.notify(e)
     {}
   end
 end

app/commands/submission/representation/process.rb

@@ -32,23 +32,29 @@ def submission
   memoize
   def ast
     tooling_job.execution_output['representation.txt']
-  rescue StandardError
+  rescue StandardError => e
+    Bugsnag.notify(e)
     nil
   end
 
   memoize
   def mapping
     res = JSON.parse(tooling_job.execution_output['mapping.json'])
     res.is_a?(Hash) ? res.symbolize_keys : {}
-  rescue StandardError
+  rescue StandardError => e
+    Bugsnag.notify(e)
     {}
   end
 
   memoize
   def metadata
-    res = JSON.parse(tooling_job.execution_output['representation.json'])
+    representation_json = tooling_job.execution_output['representation.json']
+    return {} if representation_json.empty?
+
+    res = JSON.parse(representation_json)
     res.is_a?(Hash) ? res.symbolize_keys : {}
-  rescue StandardError
+  rescue StandardError => e
+    Bugsnag.notify(e)
     {}
   end
 

app/commands/submission/test_run/process.rb

@@ -111,7 +111,8 @@ def broadcast!(test_run)
   def results
     res = JSON.parse(tooling_job.execution_output['results.json'], allow_invalid_unicode: true)
     res.is_a?(Hash) ? res.symbolize_keys : {}
-  rescue StandardError
+  rescue StandardError => e
+    Bugsnag.notify(e)
     {}
   end
 

app/models/exercise.rb

@@ -19,6 +19,7 @@ class Exercise < ApplicationRecord
   has_many :representations, dependent: :destroy
   has_many :community_videos, dependent: :destroy
   has_many :site_updates, dependent: :destroy
+  has_many :tags, dependent: :destroy
 
   has_many :approaches,
     class_name: "Exercise::Approach",

app/models/exercise/tag.rb

@@ -0,0 +1,14 @@
+class Exercise::Tag < ApplicationRecord
+  extend Mandate::Memoize
+
+  belongs_to :exercise
+
+  memoize
+  def category = tag.split(':').first
+
+  memoize
+  def name = tag.split(':').second
+
+  memoize
+  def to_s = "#{category.titleize}: #{name.titleize}"
+end

app/models/solution.rb

@@ -40,6 +40,8 @@ class Solution < ApplicationRecord
   has_many :mentor_discussions, class_name: "Mentor::Discussion", dependent: :destroy
   has_many :mentors, through: :mentor_discussions
 
+  has_many :tags, dependent: :destroy
+
   scope :completed, -> { where.not(completed_at: nil) }
   scope :not_completed, -> { where(completed_at: nil) }
 

app/models/solution/tag.rb

@@ -0,0 +1,21 @@
+class Solution::Tag < ApplicationRecord
+  extend Mandate::Memoize
+
+  belongs_to :solution
+  belongs_to :exercise
+  belongs_to :user
+
+  before_validation on: :create do
+    self.exercise_id = solution.exercise_id unless exercise_id
+    self.user_id = solution.user_id unless user_id
+  end
+
+  memoize
+  def category = tag.split(':').first
+
+  memoize
+  def name = tag.split(':').second
+
+  memoize
+  def to_s = "#{category.titleize}: #{name.titleize}"
+end

app/models/submission/analysis.rb

@@ -3,6 +3,7 @@ class Submission::Analysis < ApplicationRecord
   include HasToolingJob
 
   serialize :data, JSON
+  serialize :tags_data, JSON
 
   belongs_to :submission
   belongs_to :track
@@ -54,9 +55,11 @@ def num_comments_by_type
     end
   end
 
-  def summary
-    data[:summary].presence
-  end
+  memoize
+  def summary = data[:summary].presence
+
+  memoize
+  def tags = tags_data[:tags].to_a
 
   memoize
   def comments
@@ -96,9 +99,10 @@ def comment_blocks
   end
 
   memoize
-  def data
-    HashWithIndifferentAccess.new(super)
-  end
+  def data = HashWithIndifferentAccess.new(super)
+
+  memoize
+  def tags_data = HashWithIndifferentAccess.new(super)
 
   def analyzer_repo = "#{submission.track.slug}-analyzer"
 

db/migrate/20231013123032_add_tags_to_submission_analyses.rb

@@ -0,0 +1,7 @@
+class AddTagsToSubmissionAnalyses < ActiveRecord::Migration[7.0]
+  def change
+    return if Rails.env.production?
+
+    add_column :submission_analyses, :tags_data, :text, null: true
+  end
+end

db/migrate/20231017072611_create_solution_tags.rb

@@ -0,0 +1,17 @@
+class CreateSolutionTags < ActiveRecord::Migration[7.0]
+  def change
+    return if Rails.env.production?
+
+    create_table :solution_tags do |t|
+      t.references :solution, null: false, foreign_key: true
+      t.references :exercise, null: false, foreign_key: true
+      t.references :user, null: false, foreign_key: true
+
+      t.string :tag, null: false
+
+      t.index %i[solution_id tag], unique: true
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20231017101049_create_exercise_tags.rb

@@ -0,0 +1,16 @@
+class CreateExerciseTags < ActiveRecord::Migration[7.0]
+  def change
+    return if Rails.env.production?
+
+    create_table :exercise_tags do |t|
+      t.references :exercise, null: false, foreign_key: true
+
+      t.string :tag, null: false
+      t.boolean :filterable, null: false, default: true
+
+      t.index %i[exercise_id tag], unique: true
+
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_10_04_120817) do
+ActiveRecord::Schema[7.0].define(version: 2023_10_17_101049) do
   create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.string "name", null: false
     t.string "record_type", null: false
@@ -368,6 +368,16 @@
     t.index ["uuid"], name: "index_exercise_representations_on_uuid", unique: true
   end
 
+  create_table "exercise_tags", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
+    t.string "tag", null: false
+    t.boolean "filterable", default: true, null: false
+    t.bigint "exercise_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["exercise_id", "tag"], name: "index_exercise_tags_on_exercise_id_and_tag", unique: true
+    t.index ["exercise_id"], name: "index_exercise_tags_on_exercise_id"
+  end
+
   create_table "exercise_taught_concepts", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.bigint "exercise_id", null: false
     t.bigint "track_concept_id", null: false
@@ -852,6 +862,19 @@
     t.index ["user_id"], name: "index_solution_stars_on_user_id"
   end
 
+  create_table "solution_tags", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
+    t.string "tag", null: false
+    t.bigint "solution_id", null: false
+    t.bigint "exercise_id", null: false
+    t.bigint "user_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["exercise_id"], name: "index_solution_tags_on_exercise_id"
+    t.index ["solution_id", "tag"], name: "index_solution_tags_on_solution_id_and_tag", unique: true
+    t.index ["solution_id"], name: "index_solution_tags_on_solution_id"
+    t.index ["user_id"], name: "index_solution_tags_on_user_id"
+  end
+
   create_table "solutions", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.bigint "user_id", null: false
     t.string "uuid", null: false
@@ -940,6 +963,7 @@
     t.datetime "updated_at", null: false
     t.integer "num_comments", limit: 1, default: 0, null: false
     t.bigint "track_id"
+    t.text "tags_data"
     t.index ["submission_id"], name: "index_submission_analyses_on_submission_id"
     t.index ["track_id", "id"], name: "index_submission_analyses_on_track_id_and_id"
     t.index ["track_id", "num_comments"], name: "index_submission_analyses_on_track_id_and_num_comments"
@@ -1541,6 +1565,7 @@
   add_foreign_key "exercise_representations", "tracks"
   add_foreign_key "exercise_representations", "users", column: "feedback_author_id"
   add_foreign_key "exercise_representations", "users", column: "feedback_editor_id"
+  add_foreign_key "exercise_tags", "exercises"
   add_foreign_key "exercise_taught_concepts", "exercises"
   add_foreign_key "exercise_taught_concepts", "track_concepts"
   add_foreign_key "exercises", "tracks"
@@ -1574,6 +1599,9 @@
   add_foreign_key "site_updates", "tracks"
   add_foreign_key "site_updates", "users", column: "author_id"
   add_foreign_key "solution_comments", "solutions"
+  add_foreign_key "solution_tags", "exercises"
+  add_foreign_key "solution_tags", "solutions"
+  add_foreign_key "solution_tags", "users"
   add_foreign_key "solutions", "exercise_representations", column: "published_exercise_representation_id"
   add_foreign_key "solutions", "exercises"
   add_foreign_key "solutions", "iterations", column: "published_iteration_id"