Binary-Classification-using-Machine-Learning-and-Data-parallelism/data_separation.py at main · Sujith013/Binary-Classification-using-Machine-Learning-and-Data-parallelism · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
# mpiexec -np 4 python data_separation.py

from mpi4py import MPI
import os
from imutils import paths
import shutil

comm = MPI.COMM_WORLD
NODES = comm.Get_size()
RANK = comm.Get_rank()

DIR = "augmented"
OUTPUT_DIR = "data"

classes = os.listdir(DIR)
len_arr = []
qt = []
rem = []

for c in classes:
    l = len(list(paths.list_images(DIR+'/'+c)))

    qt.append(l//NODES)
    rem.append(l%NODES)
    len_arr.append(l)

print(qt,rem)

if RANK==0:
    os.mkdir(OUTPUT_DIR)

comm.Barrier()

for i in range(NODES):
    if i==RANK:
        print("Rank: ",RANK)
        dir1 = OUTPUT_DIR+'/data'+str(i+1)
        os.mkdir(dir1)

        for j in range(len(classes)):
            start = i*qt[j]
            end = start+qt[j]

            img_paths = list(paths.list_images(DIR+'/'+classes[j])) #source

            dest = dir1+'/'+classes[j]
            os.mkdir(dest)

            for k in range(start,end):
                src = img_paths[k]
                shutil.copy(src, dest)

            #adding remaining images to data1 folder
            if i==0:
                for k in range(-rem[j],0):
                    src = img_paths[k]
                    shutil.copy(src,dest)