SmartNet/linear_topo.py at main · intrig-unicamp/SmartNet · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
import os
import time
import argparse
import subprocess
import re
import itertools


def setting_physical_ports(num_switches):
    # Add physical ports and HPF ports to first and last bridge
    print("\n\nAdding physical and HPF ports to first and last bridges...\n\n")
    os.system("ovs-vsctl add-port ovsbr0 p0")
    os.system("ovs-vsctl add-port ovsbr0 pf0hpf")

    os.system(f"ovs-vsctl add-port ovsbr{num_switches} p1")
    os.system(f"ovs-vsctl add-port ovsbr{num_switches} pf1hpf")

def create_sf(pf_num, sf_num, sf_index, driver_counter, switch):
    os.system(f"/opt/mellanox/iproute2/sbin/mlxdevm port add pci/0000:03:00.{pf_num} flavour pcisf pfnum {pf_num} sfnum {sf_num}")

    time.sleep(2)

    os.system(f"/opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.{pf_num}/{sf_index} hw_addr 00:00:00:00:0{pf_num}:{sf_num} trust on state active")
    print(f"SF Index pci/0000:03:00.{pf_num}/{sf_index} created.")
    time.sleep(2)

    print("Binding drivers...")

    os.system(f"echo mlx5_core.sf.{driver_counter} > /sys/bus/auxiliary/drivers/mlx5_core.sf_cfg/unbind")
    time.sleep(0.5)
    os.system(f"echo mlx5_core.sf.{driver_counter} > /sys/bus/auxiliary/drivers/mlx5_core.sf/bind")
    print(f"Driver binded for SF Index pci/0000:03:00.{pf_num}/{sf_index}")

    time.sleep(0.5)
    os.system(f"ovs-vsctl add-port ovsbr{switch} en3f{pf_num}pf{pf_num}sf{sf_num}")
    print(f"\n\n Created SF Index pci/0000:03:00.{pf_num}/{sf_index}, bounded to driver mlx5_core.sf.{driver_counter} and connected to bridge ovsbr{switch}\n\n")


def run_hairpin(driver1, driver2, hp_num):
    unique_prefix = f"hp{hp_num}_{int(time.time())}"
    cmd = f"./flow_hairpin_vnf_marcelo/build/doca_flow_hairpin_vnf -a auxiliary:mlx5_core.sf.{driver1},dv_flow_en=2 -a auxiliary:mlx5_core.sf.{driver2},dv_flow_en=2 --file-prefix={unique_prefix} &"
    print(cmd)
    subprocess.run(cmd, shell=True, check=True)


def get_ovs_interfaces():
    # Run the ovs-vsctl show command and capture output
    result = subprocess.run("ovs-vsctl show", shell=True, capture_output=True, text=True)
    output = result.stdout

    bridges = {}
    current_bridge = None

    # Process each line
    for line in output.split("\n"):
        line = line.strip()

        # Detect bridge names
        if line.startswith("Bridge "):
            current_bridge = line.split(" ")[1]
            bridges[current_bridge] = []

        # Detect ports that match "p0", "p1", or start with "en"
        elif line.startswith("Port "):
            port_name = line.split(" ")[1]
            if re.match(r"^(p0|p1|en[\w\d]+)$", port_name):
                bridges[current_bridge].append(port_name)

    # Generate pairs of interfaces for each bridge
    bridge_pairs = {}
    for bridge, ports in bridges.items():
        bridge_pairs[bridge] = list(itertools.combinations(ports, 2))  # Generate pairs of interfaces

    return bridge_pairs


def set_bidirectional_flow(switch, port1, port2):
    os.system(f"ovs-ofctl add-flow {switch} \"in_port={port1},actions=output:{port2}\"")
    os.system(f"ovs-ofctl add-flow {switch} \"in_port={port2},actions=output:{port1}\"")

def main():
    parser = argparse.ArgumentParser(description="Create SF switch topology")
    parser.add_argument("--sw", type=int, required=True, help="Number of switches in the topology")
    parser.add_argument("--dir", type=int, required=False, default=0, help="If 0 bidirectional, If 1 direction p0->p1, If 2 direction p1->p0")
    args = parser.parse_args()

    num_switches = args.sw
    directional = args.dir

    if num_switches < 2:
        print("Number of switches must be at least 2")
        return

    # Initialize SF counters
    pci_00_sf_index = 229408
    pci_01_sf_index = 294944
    sf_num_counter = 10
    driver_counter = 2
    hp_num = 0

    # Create bridges if they don't exist
    for i in range(0, num_switches):
        os.system(f"ovs-vsctl --may-exist add-br ovsbr{i}")
        print(f"ovs-vsctl --may-exist add-br ovsbr{i}")

    setting_physical_ports(num_switches-1)

    for i in range(1, num_switches):
        if i < num_switches/2:
            create_sf(0, sf_num_counter, pci_00_sf_index, driver_counter, i-1)

            sf_num_counter += 1
            pci_00_sf_index += 1
            driver_counter += 1
            create_sf(0, sf_num_counter, pci_00_sf_index, driver_counter, i)
            sf_num_counter += 1
            pci_00_sf_index += 1
            driver_counter += 1

            run_hairpin(driver1=driver_counter-2, driver2=driver_counter-1, hp_num=hp_num)
            hp_num += 1

        else:
            create_sf(1, sf_num_counter, pci_01_sf_index, driver_counter, i-1)
            sf_num_counter += 1
            pci_01_sf_index += 1
            driver_counter += 1
            create_sf(1, sf_num_counter, pci_01_sf_index, driver_counter, i)
            sf_num_counter += 1
            pci_01_sf_index += 1
            driver_counter += 1

            run_hairpin(driver1=driver_counter-2, driver2=driver_counter-1, hp_num=hp_num)
            hp_num += 1

    # Run function and print result
    bridge_interface_pairs = get_ovs_interfaces()
    for bridge, pairs in bridge_interface_pairs.items():
        for pair in pairs:
            set_bidirectional_flow(bridge, pair[0], pair[1])


    exit(0)

if __name__ == "__main__":
    main()