Initial commit for throughput upper bound computation!

Author: PooriaNamyar

main_tub.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+
+from utils import utilities
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='compute throughput upper bound (TUB)')
+    parser.add_argument('file_name', metavar='file_name', type=str, help='path to the topology file')
+    args = parser.parse_args()
+    print("=" * 10 + " Computing TUB " + "=" * 10)
+    topology = utilities.get_model(args.file_name)
+    tub = topology.get_tub()
+    print("=" * 10 + " Result " + "=" * 10)
+    print("TUB = ", tub)

metric/tub.py

@@ -0,0 +1,15 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import networkx as nx
+from utils import near_wc_tm
+from typing import List, Dict
+
+
+def get_throughput_upper_bound(topo: nx.Graph, tor_list: List, demand_dict: Dict) -> float:
+    print("** Computing maximal permutation traffic matrix...")
+    _, sum_weight_matching = near_wc_tm.get_longest_matching_traffic_matrix(topo, tor_list, demand_dict)
+    print("** Computing TUB...")
+    tub = len(topo.edges()) * 2.0 / sum_weight_matching
+    return tub

requirements.txt

@@ -0,0 +1,5 @@
+networkx
+python-igraph
+numpy
+scipy
+matplotlib

utils/near_wc_tm.py

@@ -0,0 +1,75 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import gc
+import igraph
+import numpy as np
+
+from networkx import nx
+from utils import shortest_path
+from typing import List, Dict, Tuple
+
+
+def get_longest_matching_traffic_matrix(topology: nx.Graph, tor_list: List, demand_dict: Dict) -> Tuple[Dict, int]:
+    """ Generates maximal permutation traffic matrix that results in near worst-case throughput
+
+            This is a generalization to: Measuring and Understanding Throughput of Network Topologies
+            (Sangeetha Abdu Jyothi, Ankit Singla, P. Brighten Godfrey, Alexandra Kolla), 2016
+
+            Args:
+                topology: a networkx graph representing switch connections
+                tor_list: contains list of switches with directly connected servers
+                demand_dict: a mapping from each ToR to its number of directly connected servers
+            Returns:
+                traffic_matrix: a dictionary (src, dst) --> amount of traffic
+                sum_weight_matching: sum shortest_path_length(src, dst) * traffic_matrix(src, dst) over all pairs
+                                     traffic_matrix
+    """
+
+    initial_tor_list = list(tor_list)
+    num_nodes = len(tor_list)
+    tor_list = list()
+    for node in topology.nodes():
+        if node in initial_tor_list:
+            tor_list.append(node)
+    del initial_tor_list
+
+    print("**** Computing all-pair shortest path lengths...")
+    np_shortest_path = shortest_path.all_pair_shortest_path_length_adjacency_matrix(topology, tor_list)
+
+    print("**** Generating bipartite graph...")
+    bi_graph = igraph.Graph.Full_Bipartite(num_nodes, num_nodes)
+    s = np.array(list(demand_dict.values()), order='F', dtype=np.uint16) * \
+        (np.ones((num_nodes, num_nodes), order='F', dtype=np.uint16) - np.eye(num_nodes, order='F', dtype=np.uint16))
+    minimum = np.min(np.array(list(demand_dict.values())))
+    coefficient = np.minimum(s, np.transpose(s))/minimum
+    weights = np.reshape(np.multiply(np_shortest_path, coefficient, dtype=np.float32, order='F'),
+                         (1, num_nodes * num_nodes), order='F').tolist()[0]
+    del s
+    del coefficient
+    del minimum
+    gc.collect()
+    # bi_graph.es["weights"] = weights
+
+    print("**** Computing maximum weighted matching...")
+    maximal_matching = bi_graph.maximum_bipartite_matching(weights=weights, eps=0.001)
+    traffic_matrix = dict()
+
+    sum_weight_matching = 0
+    for i in range(num_nodes):
+
+        j = maximal_matching.matching[i]
+        if j >= num_nodes > i:
+            j -= num_nodes
+            s0 = tor_list[i]
+            s1 = tor_list[j]
+        elif i >= num_nodes > j:
+            raise ValueError
+        else:
+            raise ValueError
+
+        traffic_matrix[s0, s1] = min(demand_dict[s0], demand_dict[s1])
+        sum_weight_matching += np_shortest_path[i][j] * traffic_matrix[s0, s1]
+
+    return traffic_matrix, sum_weight_matching

utils/shortest_path.py

@@ -0,0 +1,81 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import networkx as nx
+import scipy
+
+from typing import List
+
+
+def sum_all_pair_shortest_path_length_adjacency_matrix(g: nx.Graph) -> int:
+    """ Computes the sum of shortest path length over all the pairs of nodes in g
+
+    Args:
+        g: a networkx graph representing the connection between switches.
+
+    Returns:
+        total_sp: sum of shortest path length over all the pairs of nodes in g
+    """
+
+    num_node = len(g.nodes())
+    A = nx.to_numpy_matrix(g, order='F', dtype=np.float32)
+    B = np.eye(num_node, order='F', dtype=np.float32)
+    total_sp = num_node * (num_node - 1) - num_node
+    C = np.zeros(np.shape(A), order='F', dtype=np.float32)
+    for k in range(num_node - 1):
+        B = scipy.linalg.blas.sgemm(1, B, A)
+        # B = np.matmul(B, A)
+        C = np.add(C, B)
+        num = np.count_nonzero(C == 0)
+        if num == 0:
+            break
+        total_sp += num
+
+    return total_sp
+
+
+def all_pair_shortest_path_length_adjacency_matrix(g: nx.Graph, tor_list: List = None) -> np.array:
+    """ Returns the length of the shortest path between all pairs of ToRs
+
+        Args:
+            g: a networkx graph representing the connection between switches.
+            tor_list: a list of tors such that the output represents the shortest path length among pairs
+                with both ends in tor_list. In case tor_list = None, this function returns the shortest
+                path length between all the pairs.
+        Returns:
+            shortest_path_np_array:
+        """
+
+    num_node = len(g.nodes())
+    A = nx.to_numpy_matrix(g, order='F', dtype=np.float32)
+    B = np.eye(num_node, order='F', dtype=np.float32)
+
+    C = np.eye(num_node, order='F', dtype=np.float32)
+    shortest_path_np_array = np.ones(np.shape(A), order='F', dtype=np.float32)
+
+    for k in range(num_node - 1):
+        B = scipy.linalg.blas.sgemm(1, B, A)
+        C = np.add(C, B)
+        if np.count_nonzero(C == 0) == 0:
+            break
+
+        add_np_array = np.subtract(np.ones(np.shape(C), order='F', dtype=np.float32), C)
+        add_np_array = np.where(add_np_array < 0, 0, add_np_array)
+        shortest_path_np_array = np.add(shortest_path_np_array, add_np_array)
+        del add_np_array
+
+    shortest_path_np_array = np.subtract(shortest_path_np_array, np.eye(num_node, order='F', dtype=np.float32))
+    delete_index_list = list()
+    if tor_list:
+        for index, node in enumerate(g.nodes()):
+            if node not in tor_list:
+                delete_index_list.append(index)
+        shortest_path_np_array = np.delete(shortest_path_np_array, delete_index_list, axis=1)
+        shortest_path_np_array = np.delete(shortest_path_np_array, delete_index_list, axis=0)
+
+    del A
+    del B
+    del C
+    return shortest_path_np_array

utils/utilities.py

@@ -0,0 +1,26 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import pickle
+
+from topo_repo import topology
+
+
+def get_model(filename: str) -> topology.Topology:
+    print("** Loading Topology...")
+    with open(filename, "rb") as file:
+        model = pickle.load(file, encoding="latin1")
+    return model
+
+
+def delete_file(file_path: str) -> None:
+    if os.path.exists(file_path):
+        os.remove(file_path)
+
+
+def store_model(model, file_path: str) -> None:
+    delete_file(file_path)
+    with open(file_path, "wb") as f:
+        pickle.dump(model, f)