bundle_monitoring.yml

resources:
  jobs:
    credit-default-monitor-update:
      name: credit-default-monitor-update-workflow
      schedule:
        quartz_cron_expression: "0 0 6 ? * MON"
        timezone_id: "Europe/Amsterdam"
        pause_status: ${var.schedule_pause_status}
      tags:
        project_name: "credit-default"
      # job_clusters:
      #   - job_cluster_key: "credit-default-cluster"
      #     new_cluster:
      #       spark_version: "15.4.x-scala2.12"
      #       data_security_mode: "SINGLE_USER"
      #       node_type_id: "i3.xlarge"
      #       driver_node_type_id: "i3.xlarge"
      #       autoscale:
      #         min_workers: 1
      #         max_workers: 1

      tasks:
        - task_key: "refresh_monitor_table"
          # job_cluster_key: "credit-default-cluster"
          existing_cluster_id: 1109-205408-dob4qyuc
          spark_python_task:
            python_file: "workflows/refresh_monitor.py"
            parameters:
              - "--root_path"
              - ${var.root_path}
          libraries:
           - whl: ./dist/*.whl