config.template.toml

[environment]
scripts = ./scripts
configs = ./configs
job_dir = ./jobs
node_prefix = calabash
admin_name = admin
admin_pass = SECRET:___
ss_pass    = SECRET:___
rsync_pass = SECRET:___

[iaas]
providers = [linode]
#providers = [linode, ucloud]

  [iaas.linode]
  token = SECRET:___
  docker_image = approach0/linode-cli
  # docker mirror alternatives: https://hub-mirror.c.163.com
  docker_mirror = https://docker.io # for docker daemon system-wise config
  docker_registry_url = https://index.docker.io/v1/ # For login
  docker_registry_user = SECRET:___
  docker_registry_pass = SECRET:___

    [iaas.linode.config_1]
    region = eu-central # ap-northeast
    specs = 'g6-nanode-1'
    distro = linode/debian10
    ssh_port = SECRET:___
    password = SECRET:___

    [iaas.linode.config_2]
    region = eu-central # us-west
    specs = 'g6-standard-2'
    distro = linode/debian10
    ssh_port = SECRET:___
    password = SECRET:___

  [iaas.ucloud]
  pubkey = ___
  prikey = ___
  docker_image = approach0/ucloud-cli
  docker_mirror = https://uhub.service.ucloud.cn
  docker_registry_url = uhub.service.ucloud.cn
  docker_registry_user = ___
  docker_registry_pass = ___

    [iaas.ucloud.config_1]
    region = cn-gd
    specs = 1cpu-1gb-1mb
    distro = 'Debian_9'
    ssh_port = ___
    password = ___

    [iaas.ucloud.config_2]
    region = cn-gd
    specs = 1cpu-2gb-1mb
    distro = 'Debian_9'
    ssh_port = ___
    password = ___

[network]

  [network.calabash_net]
  driver = overlay

[github]
  # open_PAT will be used by webpage (UI) to request Github
  # workflow changes, so it is public token. However, we still
  # need to prepend a SECRET prefix here otherwise Github will
  # detect our pushed token and revoke it.
  open_PAT = NOPUSH:___
  webhook_key = SECRET:___
  workflows = [
    approach0/gateway,
    approach0/ui-calabash,
    approach0/calabash,
    approach0/lattice,
    approach0/ui-login,

    approach0/docs,
    approach0/guide,
    approach0/a0-crawlers,
    approach0/a0-relay,
    approach0/a0-stats,
    approach0/ui-approach0,
    #approach0/docker-postgres13
  ]

[node_usage]
  [node_usage.persistent]
  labels = [
    'host_corpus=true',
    'host_persistent=true'
  ]

  [node_usage.indexer]
  labels = [
    'host_crawler=true',
    'host_indexer=true'
  ]
  install_hooks = [
    'install_fuse',
    'vdisk_producer_daemon 6K' # 6 GiB
  ]

  [node_usage.searchd]
  labels = [
    'host_crawler=true',
    'host_searchd=true'
  ]
  install_hooks = [
    'install_fuse',
    'vdisk_consume_daemon'
  ]

[loop_task]
  [loop_task.1]
  goal = 'iaas:list-nodes'
  reborn = 16100

  [loop_task.2]
  goal = 'swarm:list-nodes?format=json'
  reborn = 9200

  [loop_task.3]
  goal = 'swarm:list-services?format=json'
  reborn = 9300

  [loop_task.4]
  goal = 'swarm:list-tasks'
  reborn = 10100

[service]
bootstrap_services = [
  gateway_bootstrap,
  usersdb, lattice, # lattice should not start until usersdb service is ready
  calabash, ui_calabash
]

bootstrap_config_path = '/tmp/bootstrap.toml'

# See https://docs.docker.com/engine/reference/commandline/service_create
print_arguments = [
  mesh_replicas,
  mesh_sharding,
  max_per_node,
  restart_condition,
  stop_signal,
  stop_grace_period,
  limit_memory,
  service_labels,
  constraints,
  mounts,
  environments,
  configs,
  user,
  publish_ports,
  network,
  docker_image,
  docker_exec
]

  ###
  # Core Services
  ###

  [service.gateway_bootstrap]
  max_per_node = 1
  network = calabash_net
  publish = ['published=8080,target=443,mode=host']
  constraints = [
    'node.role==manager',
    'node.labels.host_persistent==true'
  ]
  mounts = [
    'type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock'
  ]
  docker_image = approach0/gateway

  [service.gateway]
  max_per_node = 1
  network = calabash_net
  constraints = [
    'node.role==manager',
    'node.labels.dns_pin==true',
    'node.labels.host_persistent==true'
  ]
  mounts = [
    'type=volume,src=gateway_acme_vol,dst=/root/.acme.sh',
    'type=volume,src=gateway_keys_vol,dst=/root/keys',
    'type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock'
  ]
  docker_image = approach0/gateway
  publish = [
    'published=443,target=443,mode=host',
    'published=80,target=80,mode=host'
  ]
  docker_exec = ./entrypoint.sh $domain_name

  [service.calabash]
  network = calabash_net
  labels = [
    'gateway.route=calabash',
    'gateway.protect=/runjob,/del',
    'gateway.port=8964'
  ]
  limit_memory = 300MB
  constraints = [
    'node.role==manager',
    'node.labels.host_persistent==true'
  ]
  docker_image = approach0/calabash
  config_bind = [
    'path:calabash_config:/code/calabash/config.toml'
  ]
  config_source = [
    '$service_bootstrap_config_path',
  ]
  mounts = [
    'type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock'
  ]
  docker_exec = node ./jobd/jobd.js --config ./config.toml

  [service.usersdb]
  network = calabash_net
  labels = [
    'gateway.route=usersdb',
    'gateway.port=80',
    'gateway.protect=/'
  ]
  constraints = [
    'node.role==manager', # to keep at one place for easy backup
    'node.labels.host_persistent==true'
  ]
  mounts = [
    'type=volume,src=usersdb_vol,dst=/postgres/data',
  ]
  docker_image = approach0/postgres13

  [service.usersdb_syncd]
  publish = ['published=8873,target=873,mode=host']
  mounts = [
    'type=volume,src=usersdb_vol,dst=/data',
  ]
  constraints = [
    'node.role==manager', # to keep at one place for easy backup
    'node.labels.host_persistent==true'
  ]
  config_bind = [
    'text:userdb_syncd_pass:/tmp/rsyncd.secret,mode=0600'
  ]
  config_source = [
    'rsyncclient:INJECT:rsync_pass',
  ]
  docker_image = approach0/rsyncd

  [service.lattice]
  network = calabash_net
  env = [
    'LATTICE_DATABASE_HOST=usersdb'
  ]
  constraints = [
    'node.labels.host_persistent==true'
  ]
  config_bind = [
    'text:lattice_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '''
    node db.js --init --password INJECT:admin_pass
    node ./authd.js
    '''
  ]
  labels = [
    'gateway.route=auth',
    'gateway.port=19721',
    'gateway.jwt_port=64264',
    'gateway.protect=/forbidden'
  ]
  docker_image = approach0/lattice
  docker_exec = bash /tmp/entrypoint.sh

  [service.ui_calabash]
  network = calabash_net
  labels = [
    'gateway.route=backend',
    'gateway.port=19985'
  ]
  limit_memory = 300MB
  constraints = [
    'node.labels.host_persistent==true'
  ]
  docker_image = approach0/ui-calabash

  [service.ui_login]
  network = calabash_net
  constraints = [
    'node.labels.host_persistent==true'
  ]
  labels = [
    'gateway.route=login',
    'gateway.port=19985'
  ]
  docker_image = approach0/ui-login

  [service.monitor]
  network = calabash_net
  constraints = [
    'node.role==manager',
    'node.labels.host_persistent==true'
  ]
  mounts = [
    'type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock',
    'type=volume,src=prometheus_vol,dst=/prometheus'
  ]
  user = 0:0 # default user is nobody, cannot access docker.sock
  ## FOR DEBUG ##
  #labels = [
  #  'gateway.route=_root_',
  #  'gateway.port=9090'
  #]
  #docker_image = prom/prometheus@sha256:60190123eb28250f9e013df55b7d58e04e476011911219f5cedac3c73a8b74e6
  docker_image = prom/prometheus
  config_bind = [
    'path:prometheus_conf:/tmp/prometheus.yml'
    'text:prometheus_log_rotate:/etc/logrotate.d/prometheus',
    'text:prometheus_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '$configs/prometheus.yml',
    '''
    /prometheus/query.log {
      daily
      rotate 7
      compress
      delaycompress
      postrotate
      killall -HUP prometheus
      endscript
    }
    ''',
    '''
    /bin/prometheus \
        --storage.tsdb.retention.time=3d \
        --storage.tsdb.path=/prometheus \
        --config.file=/tmp/prometheus.yml
    '''
  ]
  docker_exec = sh /tmp/entrypoint.sh

  [service.grafana]
  network = calabash_net
  constraints = [
    'node.labels.host_persistent==true'
  ]
  labels = [
    'gateway.route=grafana',
    'gateway.port=3000'
  ]
  env = [
    'GF_PATHS_DATA=/data',
    'GF_SERVER_SERVE_FROM_SUB_PATH=true',
    'GF_SERVER_ROOT_URL=/grafana',
    'GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/admin_password'
  ]
  config_bind = [
    'text:grafana_password:/run/secrets/admin_password'
  ]
  config_source = ['INJECT:admin_pass']
  mounts = [
    'type=volume,src=grafana_vol,dst=/data',
  ]
  user = 0:0
  #docker_image = approach0/grafana
  docker_image = grafana/grafana

  ###
  # Peripheral Services
  ###

  [service.hello]
  ## servcie mesh with only shard#1 publishing port
  # publish = ['published=3389,target=8080,mode=ingress']
  # mesh_replicas = 2
  # mesh_sharding = 2

  ## different IP different shard
  # publish = ['published=3389,target=8080,mode=host']
  # mesh_replicas=1
  # mesh_sharding=2

  ## round robin-ing hosts
  # publish = ['published=3389,target=8080,mode=ingress']
  mesh_replicas=2
  mesh_sharding=1

  docker_image = ga6840/hello-httpd

  config_bind = [
    'text:hello_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '''
    node /code/hello.js $(hostname) shard#$@
    '''
  ]
  docker_exec = sh /tmp/entrypoint.sh $shard

  [service.ui_search]
  network = calabash_net
  limit_memory = 600MB
  mesh_replicas = 2 # let's scale up it later
  docker_image = approach0/ui-approach0
  constraints = [
    'node.labels.host_persistent==true'
  ]
  labels = [
    'gateway.route=search',
    'gateway.port=19985'
  ]

  [service.ui_404]
  network = calabash_net
  constraints = [
    'node.labels.host_persistent==true'
  ]
  labels = [
    'gateway.route=_root_',
    'gateway.port=8080'
  ]
  docker_image = 'approach0/ui-404'

  [service.docs]
  network = calabash_net
  docker_image = approach0/docs
  constraints = [
    'node.labels.host_persistent==true'
  ]
  labels = [
    'gateway.route=docs',
    'gateway.port=8080'
  ]
  config_bind = [
    'text:docs_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '''
    export WEBHOOKSECRET=INJECT:github_webhook_key
    node main.js serve
    '''
  ]
  docker_exec = sh /tmp/entrypoint.sh

  [service.guide]
  network = calabash_net
  docker_image = approach0/guide
  constraints = [
    'node.labels.host_persistent==true'
  ]
  labels = [
    'gateway.route=guide',
    'gateway.port=8080'
  ]
  config_bind = [
    'text:guide_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '''
    export WEBHOOKSECRET=INJECT:github_webhook_key
    node main.js serve
    '''
  ]
  docker_exec = sh /tmp/entrypoint.sh

  [service.stats]
  network = calabash_net
  docker_image = approach0/a0-stats
  constraints = [
    'node.labels.host_persistent==true'
  ]
  labels = [
    'gateway.route=stats',
    'gateway.port=3207'
  ]
  config_bind = [
    'text:stats_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '''
    export IP_XOR_SECRET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 10)
    export PG_HOST=usersdb
    node statsd.js
    '''
  ]
  docker_exec = sh /tmp/entrypoint.sh

  [service.corpus_syncd]
  network = calabash_net
  publish = ['published=873,target=873,mode=host']
  mounts = [
    'type=volume,src=corpus_vol,dst=/data',
  ]
  constraints = [
    'node.labels.host_corpus==true'
  ]
  docker_image = approach0/rsyncd
  config_bind = [
    'text:corpus_syncd_pass:/tmp/rsyncd.secret,mode=0600',
    'text:corpus_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    'rsyncclient:INJECT:rsync_pass',
    '''
    rsync --daemon --verbose --config=/tmp/rsyncd.conf

    while true; do
      echo '=== Corpus Directory ==='
      df -h
      set -x
      find /data/ -name '*.json' | wc -l
      set +x

      sleep 32
    done
    '''
  ]
  docker_exec = bash /tmp/entrypoint.sh

  [service.crawler_sync]
  network = calabash_net
  max_per_node = 1
  mesh_replicas = 10
  mounts = [
    'type=volume,src=crawler_vol,dst=/data',
  ]
  constraints = [
    'node.labels.host_crawler==true'
  ]
  docker_image = approach0/rsyncd
  config_bind = [
    'text:crawler_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '''
    while true; do
      echo "Copying harvest data to corpus ..."
      export RSYNC_PASSWORD=INJECT:rsync_pass
      rsync -zauv --progress /data/ rsync://rsyncclient@corpus_syncd/data/

      echo 'Inspecting harvest data ...'
      set -x
      find /data/ -name '*.json' | wc -l
      set +x

      echo 'Deleting old data ...'
      set -x
      find /data/ -name '*.json' -mtime +1 -exec rm -f {} \;
      set +x

      sleep 32
    done
    '''
  ]
  docker_exec = bash /tmp/entrypoint.sh

  [service.crawler]
  max_per_node = 2
  mesh_replicas = 20
  mounts = [
    'type=volume,src=crawler_vol,dst=/data',
  ]
  constraints = [
    'node.labels.host_crawler==true'
  ]
  docker_image = approach0/a0-crawlers
  config_bind = [
    'path:crawler_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '$configs/entrypoint-crawler.sh'
  ]
  docker_exec = bash /tmp/entrypoint.sh

  [service.feeder]
  network = calabash_net
  mounts = [
    'type=volume,src=corpus_vol,dst=/data'
  ]
  constraints = [
    'node.labels.host_corpus==true'
  ]
  docker_image = approach0/a0-crawlers
  restart_condition = none
  config_bind = [
    'path:detect_shards:/tmp/detect-shards.sh',
    'text:feeder_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '$scripts/swarm/detect-shards.sh',
    '''
    source /tmp/detect-shards.sh

    urls=$(for i in $(detect_shards indexer); do echo "--indexd-url http://${i}:8934/index"; done)
    set -x
    feeder.py --corpus-path /data $urls --bye
    set +x
    echo 'Waiting long enough to get better chance that indexers are ready...'
    sleep 3600
    '''
  ]
  docker_exec = bash /tmp/entrypoint.sh

  [service.indexer]
  limit_memory = 300MB
  network = calabash_net
  mesh_sharding = 5
  max_per_node = 1
  mounts = [
    'type=bind,src=/var/tmp/vdisk,dst=/mnt/vdisk'
  ]
  constraints = [
    'node.labels.host_indexer==true'
  ]
  docker_image = approach0/a0
  config_bind = [
    'text:indexer_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '''
    (flock 100;
      # we need to check if the mnt directory inode is changed during docker
      # image boots up, because that is going to mislead our container to
      # the original (unmounted) directory.
      if mount | grep -q vdisk.img; then
        indexer.out -z -o /mnt/vdisk/mnt/;
        sync
        exit 0
      else
        exit 1
      fi
    ) 100>/mnt/vdisk/vdisk.lock

    # fall through
    exit $?
    '''
  ]
  stop_signal = SIGINT
  stop_grace_period = 25s # ensure indexer complete index dumping gracefully after we stops it.
  restart_condition = on-failure # restart to refresh inode only if exit code is non-zero.
  docker_exec = bash /tmp/entrypoint.sh

  [service.index_syncd]
  publish = ['published=8873,target=873,mode=host']
  network = calabash_net
  mesh_sharding = 5
  max_per_node = 1
  mounts = [
    'type=bind,src=/var/tmp/vdisk,dst=/data'
  ]
  constraints = [
    'node.labels.host_indexer==true'
  ]
  docker_image = approach0/rsyncd
  config_bind = [
    'text:index_syncd_pass:/tmp/rsyncd.secret,mode=0600',
    'text:index_syncd_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    'rsyncclient:INJECT:rsync_pass',
    '''
    shard=$1

    rsync --daemon --verbose --config=/tmp/rsyncd.conf

    while true; do
      echo "index-shard#${shard} /data"
      date
      ls -l --block-size=M /data
      df -h
      sleep 5
    done
    '''
  ]
  docker_exec = bash /tmp/entrypoint.sh $shard

  [service.searchd]
  limit_memory = 800MB
  network = calabash_net
  mesh_sharding = 5
  max_per_node = 1
  mounts = [
    'type=bind,src=/var/tmp/vdisk,dst=/mnt/vdisk'
  ]
  constraints = [
    'node.labels.host_searchd==true'
  ]
  publish = ['published=8921,target=8921,mode=host']
  config_bind = [
    'text:searchd_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '''
    shard=$1
    vdisk=/mnt/vdisk

    rm -f $vdisk/vdisk.lock

    echo '[ Inspect vdisk folder ]'
    ls -la $vdisk/ $vdisk/mnt
    du -h $vdisk/vdisk.img
    if [ -e $vdisk/mnt/mstats.bin ]; then
      doc-lookup.out -p $vdisk/mnt
    fi
    tail -50 $vdisk/history_images.log

    last_timestamp=$(cat $vdisk/timestamp.txt 2>/dev/null || echo 0)
    curr_timestamp=$(date +%s)
    echo $curr_timestamp > $vdisk/timestamp.txt
    delta_time=$(($curr_timestamp - $last_timestamp))
    echo "[ Detal time: $delta_time seconds ]"

    if [[ $delta_time -lt 1800 && -e $vdisk/mnt/mstats.bin ]]; then

      echo '[ Index is up to date and mounted, setup SSH daemon ]'
      /usr/sbin/sshd -e -D

    else
      echo '[ Index missing or out of date, sync index image... ]'
      export RSYNC_PASSWORD=INJECT:rsync_pass
      rm -f $vdisk/.vdisk.*
      if [ $shard -eq 1 ]; then
        rsync -zuv --checksum rsync://rsyncclient@index_syncd/data/vdisk.*.img $vdisk/
      else
        rsync -zuv --checksum rsync://rsyncclient@index_syncd-shard${shard}/data/vdisk.*.img $vdisk/
      fi
      (date; ls $vdisk/vdisk.*.img) >> $vdisk/history_images.log

      echo '[ Wait long enough for new vdisk to be mounted ]'
      sleep 16

      echo '[ Fall through and restart ]'
      exit 1
    fi
    '''
  ]
  stop_signal = SIGUSR1
  docker_image = approach0/a0
  docker_exec = bash /tmp/entrypoint.sh $shard

  [service.searchd_mpirun]
  limit_memory = 800MB
  network = calabash_net
  max_per_node = 1 # for multiple searchd replicas
  constraints = [
    'node.labels.shard==1', # let it be around to searchd (shard 1)
    'node.labels.host_searchd==true'
  ]
  docker_image = approach0/a0
  config_bind = [
    'path:detect_shards:/tmp/detect-shards.sh',
    'text:mpi_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '$scripts/swarm/detect-shards.sh',
    '''
    subnet=$1
    service=${2-searchd}
    word_cache=${3-0}
    math_cache=${4-0}

    trap "" PIPE
    source /tmp/detect-shards.sh

    mountat=/mnt/vdisk
    command="searchd.out -i ./mnt -L /mnt/vdisk/vdisk.lock -c $word_cache -C $math_cache"

    hosts=`detect_shards $service`
    np=$(echo $hosts | wc -w)
    hosts=$(echo $hosts | tr ' ', ',' | sed -e "s/$service/localhost/")

    set -x
    ssh root@$service mpirun --allow-run-as-root --tag-output \
      -np $np --rank-by node \
      --host $hosts --wdir $mountat \
      --mca oob_tcp_if_include $subnet --mca btl_tcp_if_include $subnet \
      $command
    set +x
    '''
  ]
  stop_signal = SIGUSR1
  restart_condition = any
  # suggested full index parameters: word_cache=100, math_cache=500
  docker_exec = 'bash /tmp/entrypoint.sh $(swarm_network_space calabash_net) $target_serv $word_cache $math_cache'

  [service.relay]
  network = calabash_net
  max_per_node = 1 # for multiple searchd replicas
  docker_image = approach0/a0-relay
  constraints = [
    'node.labels.shard==1', # let it be around to searchd (shard 1)
    'node.labels.host_searchd==true'
  ]
  labels = [
    'gateway.route=search-relay',
    'gateway.port=8080'
  ]
  config_bind = [
    'text:relay_entrypoint:/tmp/entrypoint2.sh'
  ]
  config_source = [
    '''
    export A0_SEARCHD=${1-searchd}
    export A0_QRYLOGD=stats
    /tmp/entrypoint.sh
    '''
  ]
  docker_exec = bash /tmp/entrypoint2.sh $relay_target

  [service.ss]
  mesh_replicas = 2
  max_per_node = 1
  publish = ['published=80,target=80,mode=host']
  mounts = [
    'type=volume,src=corpus_vol,dst=/data',
  ]
  constraints = [
    'node.labels.host_searchd==true'
  ]
  docker_image = approach0/docker-ss
  config_bind = [
    'text:ss_entrypoint:/tmp/entrypoint.sh'
  ]
  config_source = [
    '''
    SS_PASS=INJECT:ss_pass
    sed -i -e "s/iamforgetful/$SS_PASS/g" /root/ss.json
    /usr/local/bin/ss-server -c /root/ss.json
    '''
  ]
  docker_exec = bash /tmp/entrypoint.sh