Merge pull request #584 from FedML-AI/test/v0.7.0

fedml-alex · web-flow · commit f21374426177 · 2022-09-23T15:41:50.000+08:00
Test/v0.7.0
diff --git a/python/app/fedcv/object_detection/config/bootstrap.bat b/python/app/fedcv/object_detection/config/bootstrap.bat
diff --git a/python/app/fedcv/object_detection/config/bootstrap.sh b/python/app/fedcv/object_detection/config/bootstrap.sh
diff --git a/python/examples/cross_silo/mqtt_s3_fedavg_defense_mnist_lr_example/run_server.sh b/python/examples/cross_silo/mqtt_s3_fedavg_defense_mnist_lr_example/run_server.sh
@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 RUN_ID=$1
-python3 torch_server.py --cf config/krum/fedml_config.yaml --rank 0 --role server --run_id $RUN_ID
+python3 torch_server.py --cf config/foolsgold/fedml_config.yaml --rank 0 --role server --run_id $RUN_ID
diff --git a/python/examples/cross_silo/mqtt_s3_fedavg_mnist_lr_example/custom_data_and_model/config/bootstrap.bat b/python/examples/cross_silo/mqtt_s3_fedavg_mnist_lr_example/custom_data_and_model/config/bootstrap.bat
@@ -0,0 +1,12 @@
+:: ### don't modify this part ###
+:: ##############################
+
+
+:: ### please customize your script in this region ####
+set DATA_PATH=%userprofile%\fedml_data
+mkdir %DATA_PATH%
+
+
+:: ### don't modify this part ###
+echo [FedML]Bootstrap Finished
+:: ##############################
diff --git a/python/examples/cross_silo/mqtt_s3_fedavg_mnist_lr_example/custom_data_and_model/config/bootstrap.sh b/python/examples/cross_silo/mqtt_s3_fedavg_mnist_lr_example/custom_data_and_model/config/bootstrap.sh
diff --git a/python/fedml/__init__.py b/python/fedml/__init__.py
@@ -23,7 +23,7 @@
 _global_training_type = None
 _global_comm_backend = None
 
-__version__ = "0.7.321"
+__version__ = "0.7.326"
 
 
 def init(args=None):
diff --git a/python/fedml/cli/edge_deployment/client_constants.py b/python/fedml/cli/edge_deployment/client_constants.py
@@ -1,5 +1,6 @@
 
 import os
+import platform
 import shutil
 import signal
 import subprocess
@@ -217,9 +218,15 @@ def exit_process(process):
     @staticmethod
     def exec_console_with_script(script_path, should_capture_stdout_err=False):
         if should_capture_stdout_err:
-            script_process = subprocess.Popen(['sh', '-c', script_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            if platform.system() == 'Windows':
+                script_process = subprocess.Popen(script_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            else:
+                script_process = subprocess.Popen(['sh', '-c', script_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         else:
-            script_process = subprocess.Popen(['sh', '-c', script_path], stdout=sys.stdout, stderr=subprocess.PIPE)
+            if platform.system() == 'Windows':
+                script_process = subprocess.Popen(script_path, stdout=sys.stdout, stderr=subprocess.PIPE)
+            else:
+                script_process = subprocess.Popen(['sh', '-c', script_path], stdout=sys.stdout, stderr=subprocess.PIPE)
         return script_process
 
     @staticmethod
diff --git a/python/fedml/cli/edge_deployment/client_runner.py b/python/fedml/cli/edge_deployment/client_runner.py
@@ -252,20 +252,27 @@ def build_dynamic_args(self, run_config, package_conf_object, base_dir):
             if bootstrap_script_path is not None:
                 if os.path.exists(bootstrap_script_path):
                     bootstrap_stat = os.stat(bootstrap_script_path)
-                    os.chmod(bootstrap_script_path, bootstrap_stat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
-                    bootstrap_scripts = "cd {}; ./{}".format(bootstrap_script_dir, os.path.basename(bootstrap_script_file))
+                    if platform.system() == 'Windows':
+                        os.chmod(bootstrap_script_path, bootstrap_stat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+                        bootstrap_scripts = "{}".format(bootstrap_script_path)
+                    else:
+                        os.chmod(bootstrap_script_path, bootstrap_stat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+                        bootstrap_scripts = "cd {}; ./{}".format(bootstrap_script_dir, os.path.basename(bootstrap_script_file))
+                    bootstrap_scripts = str(bootstrap_scripts).replace('\\', os.sep).replace('/', os.sep)
                     logging.info("Bootstrap scripts are being executed...")
                     process = ClientConstants.exec_console_with_script(bootstrap_scripts, should_capture_stdout_err=True)
                     ret_code, out, err = ClientConstants.get_console_pipe_out_err_results(process)
                     if out is not None:
                         out_str = out.decode(encoding="utf-8")
-                        if str(out_str).find(FedMLClientRunner.FEDML_BOOTSTRAP_RUN_OK) == -1:
+                        if str(out_str).find(FedMLClientRunner.FEDML_BOOTSTRAP_RUN_OK) == -1 \
+                                and str(out_str).lstrip(' ').rstrip(' ') != '':
                             logging.error("{}".format(out_str))
                         else:
                             logging.info("{}".format(out_str))
                     if err is not None:
                         err_str = err.decode(encoding="utf-8")
-                        if str(err_str).find(FedMLClientRunner.FEDML_BOOTSTRAP_RUN_OK) == -1:
+                        if str(err_str).find(FedMLClientRunner.FEDML_BOOTSTRAP_RUN_OK) == -1 \
+                                and str(err_str).lstrip(' ').rstrip(' ') != '':
                             logging.error("{}".format(err_str))
                         else:
                             logging.info("{}".format(err_str))
diff --git a/python/fedml/cli/server_deployment/server_constants.py b/python/fedml/cli/server_deployment/server_constants.py
@@ -1,6 +1,7 @@
 
 import json
 import os
+import platform
 import signal
 import subprocess
 import sys
@@ -193,9 +194,15 @@ def exit_process(process):
     @staticmethod
     def exec_console_with_script(script_path, should_capture_stdout_err=False):
         if should_capture_stdout_err:
-            script_process = subprocess.Popen(['sh', '-c', script_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            if platform.system() == 'Windows':
+                script_process = subprocess.Popen(script_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            else:
+                script_process = subprocess.Popen(['sh', '-c', script_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         else:
-            script_process = subprocess.Popen(['sh', '-c', script_path], stdout=sys.stdout, stderr=subprocess.PIPE)
+            if platform.system() == 'Windows':
+                script_process = subprocess.Popen(script_path, stdout=sys.stdout, stderr=subprocess.PIPE)
+            else:
+                script_process = subprocess.Popen(['sh', '-c', script_path], stdout=sys.stdout, stderr=subprocess.PIPE)
         return script_process
 
     @staticmethod
diff --git a/python/fedml/cli/server_deployment/server_runner.py b/python/fedml/cli/server_deployment/server_runner.py
@@ -273,22 +273,28 @@ def build_dynamic_args(self, run_config, package_conf_object, base_dir):
             if bootstrap_script_path is not None:
                 if os.path.exists(bootstrap_script_path):
                     bootstrap_stat = os.stat(bootstrap_script_path)
-                    os.chmod(bootstrap_script_path, bootstrap_stat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
-                    bootstrap_scripts = "cd {}; ./{}".format(bootstrap_script_dir,
-                                                             os.path.basename(bootstrap_script_file))
+                    if platform.system() == 'Windows':
+                        os.chmod(bootstrap_script_path, bootstrap_stat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+                        bootstrap_scripts = "{}".format(bootstrap_script_path)
+                    else:
+                        os.chmod(bootstrap_script_path, bootstrap_stat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+                        bootstrap_scripts = "cd {}; ./{}".format(bootstrap_script_dir, os.path.basename(bootstrap_script_file))
+                    bootstrap_scripts = str(bootstrap_scripts).replace('\\', os.sep).replace('/', os.sep)
                     logging.info("Bootstrap scripts are being executed...")
                     process = ServerConstants.exec_console_with_script(bootstrap_scripts,
                                                                        should_capture_stdout_err=True)
                     ret_code, out, err = ServerConstants.get_console_pipe_out_err_results(process)
                     if out is not None:
                         out_str = out.decode(encoding="utf-8")
-                        if str(out_str).find(FedMLServerRunner.FEDML_BOOTSTRAP_RUN_OK) == -1:
+                        if str(out_str).find(FedMLServerRunner.FEDML_BOOTSTRAP_RUN_OK) == -1 \
+                                and str(out_str).lstrip(' ').rstrip(' ') != '':
                             logging.error("{}".format(out_str))
                         else:
                             logging.info("{}".format(out_str))
                     if err is not None:
                         err_str = err.decode(encoding="utf-8")
-                        if str(err_str).find(FedMLServerRunner.FEDML_BOOTSTRAP_RUN_OK) == -1:
+                        if str(err_str).find(FedMLServerRunner.FEDML_BOOTSTRAP_RUN_OK) == -1 \
+                                and str(err_str).lstrip(' ').rstrip(' ') != '':
                             logging.error("{}".format(err_str))
                         else:
                             logging.info("{}".format(err_str))
diff --git a/python/fedml/core/security/defense/foolsgold_defense.py b/python/fedml/core/security/defense/foolsgold_defense.py
@@ -1,9 +1,9 @@
-import numpy as np
-from .defense_base import BaseDefenseMethod
 from typing import Callable, List, Tuple, Dict, Any
-from ..common import utils
+
+import numpy as np
 from scipy import spatial
 
+from .defense_base import BaseDefenseMethod
 
 """
 The Limitations of Federated Learning in Sybil Settings.
@@ -17,17 +17,14 @@ class FoolsGoldDefense(BaseDefenseMethod):
     def __init__(self, config):
         self.config = config
         self.memory = None
-        self.use_memory = config.use_memory
 
     def run(
         self,
         raw_client_grad_list: List[Tuple[float, Dict]],
         base_aggregation_func: Callable = None,
         extra_auxiliary_info: Any = None,
     ):
-        new_grad_list = self.defend_before_aggregation(
-            raw_client_grad_list, extra_auxiliary_info
-        )
+        new_grad_list = self.defend_before_aggregation(raw_client_grad_list, extra_auxiliary_info)
         return base_aggregation_func(self.config, new_grad_list)
 
     def defend_before_aggregation(
@@ -36,24 +33,20 @@ def defend_before_aggregation(
         extra_auxiliary_info: Any = None,
     ):
         client_num = len(raw_client_grad_list)
-        if self.use_memory:
-            if self.memory is None:
-                self.memory = [grad for num, grad in raw_client_grad_list]
-            else:  # memory: potential bugs: grads in different iterations may be from different clients
-                for i in range(client_num):
-                    (num, grad) = raw_client_grad_list[i]
-                    for k in grad.keys():
-                        self.memory[i][k] += grad[k]
-            alphas = self.fools_gold_score(self.memory)  # Use FG
-        else:
-            grads = [grad for (_, grad) in raw_client_grad_list]
-            alphas = self.fools_gold_score(grads)  # Use FG
+        importance_feature_list = self._get_importance_feature(raw_client_grad_list)
+        print(len(importance_feature_list))
 
+        if self.memory is None:
+            self.memory = importance_feature_list
+        else:  # memory: potential bugs: grads in different iterations may be from different clients
+            for i in range(client_num):
+                self.memory[i] += importance_feature_list[i]
+        alphas = self.fools_gold_score(self.memory)  # Use FG
+
+        print("alphas = {}".format(alphas))
         assert len(alphas) == len(
             raw_client_grad_list
-        ), "len of wv {} is not consistent with len of client_grads {}".format(
-            len(alphas), len(raw_client_grad_list)
-        )
+        ), "len of wv {} is not consistent with len of client_grads {}".format(len(alphas), len(raw_client_grad_list))
         new_grad_list = []
         client_num = len(raw_client_grad_list)
         for i in range(client_num):
@@ -63,35 +56,51 @@ def defend_before_aggregation(
 
     # Takes in grad, compute similarity, get weightings
     @staticmethod
-    def fools_gold_score(grad_list):
-        n_clients = len(grad_list)
-        grads = [utils.vectorize_weight(grad) for grad in grad_list]
+    def fools_gold_score(feature_vec_list):
+        n_clients = len(feature_vec_list)
         cs = np.zeros((n_clients, n_clients))
         for i in range(n_clients):
             for j in range(n_clients):
-                cs[i][j] = 1 - spatial.distance.cosine(
-                    grads[i].tolist(), grads[j].tolist()
-                )
+                cs[i][j] = 1 - spatial.distance.cosine(feature_vec_list[i], feature_vec_list[j])
         cs -= np.eye(n_clients)
+        # cs = smp.cosine_similarity(feature_vec_list) - np.eye(n_clients)
         maxcs = np.max(cs, axis=1)
         # pardoning
         for i in range(n_clients):
             for j in range(n_clients):
-                if i != j and maxcs[i] < maxcs[j]:
+                if i == j:
+                    continue
+                if maxcs[i] < maxcs[j]:
                     cs[i][j] = cs[i][j] * maxcs[i] / maxcs[j]
         alpha = 1 - (np.max(cs, axis=1))
-        alpha[alpha > 1] = 1
-        alpha[alpha < 0] = 0
+        alpha[alpha > 1.0] = 1.0
+        alpha[alpha <= 0.0] = 1e-15
 
         # Rescale so that max value is alpha
+        print(np.max(alpha))
         alpha = alpha / np.max(alpha)
-        alpha[(alpha == 1)] = 0.99
+        alpha[(alpha == 1.0)] = 0.999999
 
         # Logit function
-        for i in range(len(alpha)):
-            if alpha[i] != 0:
-                alpha[i] = np.log(alpha[i] / (1 - alpha[i])) + 0.5
+        alpha = np.log(alpha / (1 - alpha)) + 0.5
         alpha[(np.isinf(alpha) + alpha > 1)] = 1
         alpha[(alpha < 0)] = 0
 
+        print("alpha = {}".format(alpha))
+
         return alpha
+
+    def _get_importance_feature(self, raw_client_grad_list):
+        # Foolsgold uses the last layer's gradient/weights as the importance feature.
+        ret_feature_vector_list = []
+        for idx in range(len(raw_client_grad_list)):
+            raw_grad = raw_client_grad_list[idx]
+            (p, grads) = raw_grad
+
+            # Get last key-value tuple
+            (weight_name, importance_feature) = list(grads.items())[-2]
+            print(importance_feature)
+            feature_len = np.array(importance_feature.data.detach().numpy().shape).prod()
+            feature_vector = np.reshape(importance_feature.cpu().data.detach().numpy(), feature_len)
+            ret_feature_vector_list.append(feature_vector)
+        return ret_feature_vector_list
diff --git a/python/setup.py b/python/setup.py
@@ -73,7 +73,7 @@ def finalize_options(self):
 
 setup(
     name="fedml",
-    version="0.7.321",
+    version="0.7.326",
     author="FedML Team",
     author_email="ch@fedml.ai",
     description="A research and production integrated edge-cloud library for "
diff --git a/python/tests/security/defense/test_foolsgold_defense.py b/python/tests/security/defense/test_foolsgold_defense.py
@@ -1,35 +1,35 @@
 import argparse
+
+import torch
+
 from fedml.core.security.defense.foolsgold_defense import FoolsGoldDefense
 from fedml.ml.aggregator.agg_operator import FedMLAggOperator
-from utils import create_fake_model_list_MNIST, create_fake_model_list
 
 
-def add_args(use_memory):
+def add_args():
     parser = argparse.ArgumentParser(description="FedML")
     parser.add_argument(
-        "--yaml_config_file", "--cf", help="yaml configuration file", type=str, default="",
+        "--yaml_config_file",
+        "--cf",
+        help="yaml configuration file",
+        type=str,
+        default="",
     )
     parser.add_argument("--federated_optimizer", type=str, default="FedAvg")
-    parser.add_argument("--use_memory", type=bool, default=use_memory)
     args, unknown = parser.parse_known_args()
     return args
 
 
-def test_fools_gold_score():
-    model_list = create_fake_model_list(5)
-    print(f"modellist len = {len(model_list)}")
-    grads = [grad for num, grad in model_list]
-    print(f"fools_gold_score={FoolsGoldDefense.fools_gold_score(grads)}")
-
-
 def test_defense():
-    config = add_args(use_memory=True)
-    model_list = create_fake_model_list_MNIST(3)
+    config = add_args()
+    model = torch.hub.load("pytorch/vision:v0.10.0", "vgg11", pretrained=True).state_dict()
+    model_list = [(100, model) for i in range(4)]
+
+    print(f"model_list len = {len(model_list)}")
     defense = FoolsGoldDefense(config)
     aggr_result = defense.run(model_list, base_aggregation_func=FedMLAggOperator.agg)
-    print(f"result = {aggr_result}")
+    # print(f"result = {aggr_result}")
 
 
 if __name__ == "__main__":
-    test_fools_gold_score()
     test_defense()
