# import
import dadt.Pytorh as dadt

  .
  .
  .

def main(unused_argv):
  # at beginning init DADT
  # cycle_duration_ms: background thread do AllReduce periodically, cycle_duration_ms control how many time between every step.
  # broad_cast_executor: broadcast executor, at beginning rank-0 will broadcast weight to other ranks. use broad_cast_executor to decide which broadcast executor will be used. 
  # 'mpi': use MPI do broadcast
  # 'nccl': use CUDA NCCL do broadcast
  # 'mpicuda': use MPI to do broadcast and MPI must be [GPU-aware](https://devblogs.nvidia.com/introduction-cuda-aware-mpi/). 
  # When traing on CPU must choose 'MPI'.
  # all_reduce_executor: same like broad_cast_executor this flag decide whick executor use to do AllReduce.
  # all_reduce_buffer_size: AllReduce fusion buffer, default is 64MB, set to be 0 means do not use buffer.
  # group_buffer_size: used to collect ioslate tensor to a Group.
  dadt.init(cycle_duration_ms=3, 
            broad_cast_executor='nccl', 
            all_reduce_executor='mpi',
            all_reduce_buffer_size=67108864,
            group_buffer_size=67108864)
  .
  .
  .
  # wrapper the optimizer
  optimizer = optim.Adadelta(model.parameters(), lr=args.lr)
  distribte_optimizer = dadt.DistributedOptimizer(optimizer=optimizer)
  .
  .
  .
  # tain like nornaml pytorch
  distribte_optimizer.zero_grad()
  .
  .
  .
  loss.backward()
  distribte_optimizer.step()