Fix more configs

cfgs/default/4m/data/video/mix_mod3_rgb_tok_to_all_a0.5.yaml

@@ -21,7 +21,7 @@ train:
       # Data can either be local or on cloud storage (e.g. S3), see data docs for more info
       # Use braceexpand notation to indicate shard range (e.g. shard-{0000..9999}.tar)
       # Use brackets to indicate multiple modalities (e.g. [modality1,modality2,modality3]) 
-      data_path: '/store/swissai/a08/data/4m/train/[video_rgb,video_rgb_tok]/shard-{00000..00100}.tar' # TODO: need to reformat the data correctly here. 
+      data_path: '/store/swissai/a08/data/4m/splits2/train/[video_rgb,video_rgb_tok]/00000{00000..00100}.tar' # TODO: need to reformat the data correctly here. 
       use_wds: True # Use webdataset
       wds_n_repeats: 4 # Number of repeats for webdataset loader to improve efficiency
       wds_shuffle_buffer_tar: 1_000 # Webdatasets shuffle buffer after loading tar files
@@ -38,7 +38,7 @@ train:
 
   weights: [1.0] # Sampling weights for the training datasets
 
-val:
-  datasets:
-    my_video_dataset: 
-      data_path: '/store/swissai/a08/data/4m/val/[video_rgb,video_rgb_tok]/shard-{00000..00100}.tar'
+# val:
+#   datasets:
+#     my_video_dataset: 
+#       data_path: '/store/swissai/a08/data/4m/val/[video_rgb,video_rgb_tok]/00000{00175..00199}.tar'

cfgs/default/4m/models/video/4m-b_mod3.yaml

@@ -0,0 +1,46 @@
+# Config for DDP
+
+# Arch: SwiGLU No Bias
+# Modalities: Mix of rgb2all and all2all, with alphas=0.5
+# To be run on 64 GPUs for batch size = 8192
+run_name: auto
+
+# Input & output
+num_input_tokens: 128
+num_target_tokens: 128
+loss_type: mod
+
+# Architecture
+model: fm_base_12e_12d_swiglu_nobias
+patch_size: 16
+input_size: 224
+dtype: bfloat16
+tokenizer_path: "fourm/utils/tokenizer/trained/text_tokenizer_4m_wordpiece_30k.json"
+
+# Train
+epochs: -1
+total_tokens: 500 # in billions
+opt: adamw
+blr: 0.0001 # this is base_lr = 1e-4, lr = base_lr * batch_size / 256
+min_blr: 0.
+warmup_epochs: -1
+warmup_tokens: 10 # in billions
+batch_size: 128 # 128 x 64 = 8192
+
+# Data
+
+data_config: "cfgs/default/4m/data/video/mix_mod3_rgb_tok_to_all_a0.5.yaml"
+s3_data_endpoint: null # Change me
+eval_freq: 1
+fixed_eval: True
+epoch_size: 10_000_000 # Number of samples per "epoch"
+
+# Saving
+save_ckpt_freq: 1
+output_dir: 'output/auto'
+
+# Wandb
+log_wandb: False # Set to True to log to Weights & Biases
+wandb_project: '4m-train'
+wandb_entity: null # Change if needed
+wandb_run_name: auto

run_training_4m.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# ython run_training_4m.py --data_config cfgs/default/4m/data/video/mix_mod3_rgb_tok_to_all_a0.5.yaml
+# python run_training_4m.py --data_config cfgs/default/4m/data/video/mix_mod3_rgb_tok_to_all_a0.5.yaml
 import argparse
 import datetime
 import json