########################### ICML 2022 #################################
# MARINA VS ZEROMARINA: the proof of concept
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_libsvm_zero_marina.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset mushrooms --experiments_name compare_icml_marina_mushroms_nof_10_numnodes_5  --num_nodes_list 5 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 21000 --algorithm_names zero_marina marina --function nonconvex  --compressors rand_k  --number_of_coordinates 10 --quality_check_rate 10
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/zero_marina/plot_marina_mushrooms_gradient.py --dumps_paths ~/exepriments/compare_icml_marina_mushroms_nof_10_numnodes_5/ --output_path ~/tmp_launch/compare_icml_marina_mushroms_nof_10_numnodes_5

# VR-MARINA VS ZEROMARINA-PAGE
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_libsvm_zero_marina.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name compare_icml_vr_marina_real-sim_nof_200_numnodes_5  --num_nodes_list 5 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 50000 --algorithm_names zero_marina_page vr_marina --function nonconvex  --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --batch_size 1 --number_of_coordinates 200 --quality_check_rate 100 --oracle minibatch
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/zero_marina/plot_vr-marina_real-sim_mini-batch.py --dumps_paths ~/exepriments/compare_icml_vr_marina_real-sim_nof_100_numnodes_5/ ~/exepriments/compare_icml_vr_marina_real-sim_nof_500_numnodes_5/ ~/exepriments/compare_icml_vr_marina_real-sim_nof_2000_numnodes_5/ --output_path  ~/tmp_launch/compare_icml_vr_marina_real-sim

# STOCHASTIC
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_libsvm_zero_marina.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name compare_icml_stochastic_marina_real-sim_nof_200_numnodes_5_mega_batch_10000_fix  --num_nodes_list 5 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic marina_stochastic  --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 200 --quality_check_rate 100 --oracle stochastic --mega_batch 10000 --batch_size 1 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_libsvm_zero_marina.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name compare_icml_stochastic_marina_real-sim_nof_200_numnodes_5_mega_batch_100_fix  --num_nodes_list 5 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic marina_stochastic  --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 200 --quality_check_rate 100 --oracle stochastic --mega_batch 100 --batch_size 1 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_libsvm_zero_marina.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name compare_icml_stochastic_marina_real-sim_nof_2000_numnodes_5_mega_batch_10000_fix  --num_nodes_list 5 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic marina_stochastic  --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 2000 --quality_check_rate 100 --oracle stochastic --mega_batch 10000 --batch_size 1 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_libsvm_zero_marina.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name compare_icml_stochastic_marina_real-sim_nof_2000_numnodes_5_mega_batch_100_fix  --num_nodes_list 5 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic marina_stochastic  --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 2000 --quality_check_rate 100 --oracle stochastic --mega_batch 100 --batch_size 1 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_libsvm_zero_marina.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name compare_icml_stochastic_marina_real-sim_nof_200_numnodes_5_mega_batch_100000_fix  --num_nodes_list 5 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic marina_stochastic  --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 200 --quality_check_rate 100 --oracle stochastic --mega_batch 100000 --batch_size 1 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_libsvm_zero_marina.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name compare_icml_stochastic_marina_real-sim_nof_2000_numnodes_5_mega_batch_100000_fix  --num_nodes_list 5 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic marina_stochastic  --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 2000 --quality_check_rate 100 --oracle stochastic --mega_batch 100000 --batch_size 1 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001

PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/zero_marina/plot_vr-marina_real-sim_stochastic.py --dumps_paths /home/tyurina/exepriments/compare_icml_stochastic_marina_real-sim_nof_200_numnodes_5_mega_batch_100000_fix_longer /home/tyurina/exepriments/compare_icml_stochastic_marina_real-sim_nof_2000_numnodes_5_mega_batch_100000_fix_longer /home/tyurina/exepriments/compare_icml_stochastic_marina_real-sim_nof_200_numnodes_5_mega_batch_10000_fix /home/tyurina/exepriments/compare_icml_stochastic_marina_real-sim_nof_2000_numnodes_5_mega_batch_10000_fix --output_path ~/tmp_launch/compare_icml_stochastic_marina_real-sim

# Deep learning
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_nn.py --dataset_path ~/data --dumps_path /home/tyurina/exepriments/ --experiments_name icml_resnet_relu_zero_marina_stochastic_bz_128_nc_2M_nodes_5 --num_nodes_list 5 --step_size_range 0.05 0.01 0.005 0.001 --number_of_iterations 40000 --algorithm_names zero_marina_stochastic --batch_size 128 --number_of_coordinates 2000000 --quality_check_rate 1000 --noise_momentums 0.5 0.1 0.05 0.01 --devices 2 3 --resnet_activation relu
# for f in /home/tyurina/exepriments/icml_resnet_relu_zero_marina_stochastic_bz_128_nc_2M_nodes_5/source_folder/singularity_*; do sh $f & done
PYTHONPATH=./code python3 code/distributed_optimization_library/experiments/plots/zero_marina/plot_marina_nn.py --dumps_paths ~/exepriments/icml_resnet_relu_zero_marina_stochastic_bz_128_nc_2M_nodes_5_repeat/ ~/exepriments/resnet_relu_stochastic_gradient_descent_bz_128_nodes_5/ ~/exepriments/resnet_relu_marina_stochastic_bz_128_nc_2M_nodes_5/ ~/exepriments/icml_resnet_relu_zero_marina_sync_stochastic_bz_128_nc_2M_nodes_5/ --output_path ~/tmp_launch/resnet_relu --function --cut_plots


# Check bad regimes of MVR
# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr
# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_1000_noise_0_1_scale_initial_point_1_sc_1_0_longer_with_marina_sr
# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_10_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr
# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_100000_noise_10_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr
# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_1000000_noise_10_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr
# compare_icml_stochastic_matrix_tridiagonal_nof_10_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr
# compare_icml_stochastic_matrix_tridiagonal_nof_100_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr
# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr_more

# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_longer_with_marina_sr_more
# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_0_01_longer_with_marina_sr_more
# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_0_1_longer_with_marina_sr_more
# compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_10_0_longer_with_marina_sr_more

PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/zero_marina/config_quadratic_zero_marina.py --dumps_path /home/tyurina/exepriments --experiments_name compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr --num_nodes_list 1 --step_size_range -20 -5 --number_of_seeds 1 --number_of_iterations 5000000 --algorithm_names zero_marina_sync_stochastic zero_marina_stochastic zero_marina_stochastic_underestimated_batch marina_stochastic --compressors rand_k --number_of_coordinates 1 --quality_check_rate 100 --oracle stochastic --mega_batch 10000 --batch_size 1 --dim 10000 --noise 1.0 --type_stochastic stochastic_matrix --scale_initial_point 1 --strongly_convex_constant 1.0 --save_rate 100000

PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/zero_marina/plot_vr-marina_real-sim_stochastic_init_exper_strongly.py --dumps_paths /home/tyurina/exepriments/compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr_more --output_path ~/tmp_launch/compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr_more
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/zero_marina/plot_vr-marina_real-sim_stochastic_init_exper_strongly.py --dumps_paths /home/tyurina/exepriments/compare_icml_stochastic_matrix_tridiagonal_nof_10_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr --output_path ~/tmp_launch/compare_icml_stochastic_matrix_tridiagonal_nof_10_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_1_0_longer_with_marina_sr
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/zero_marina/plot_vr-marina_real-sim_stochastic_init_exper_strongly.py --dumps_paths /home/tyurina/exepriments/compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_1000_noise_0_1_scale_initial_point_1_sc_1_0_longer_with_marina_sr --output_path ~/tmp_launch/compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_1000_noise_0_1_scale_initial_point_1_sc_1_0_longer_with_marina_sr
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/zero_marina/plot_vr-marina_real-sim_stochastic_init_exper_strongly.py --dumps_paths /home/tyurina/exepriments/compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_0_1_longer_with_marina_sr_more --output_path ~/tmp_launch/compare_icml_stochastic_matrix_tridiagonal_nof_1_mega_batch_10000_noise_1_0_scale_initial_point_1_sc_0_1_longer_with_marina_sr_more --dont_show_second_mvr
