SignLanguageRecognizer/process_data.py at main · srinidhipappu/SignLanguageRecognizer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
import numpy as np
import os
from sklearn.model_selection import train_test_split
from sklearn.utils import class_weight
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout
from tensorflow.keras.callbacks import TensorBoard
from keras.optimizers import Adam


actions = np.array(['hello', 'thanks', 'iloveyou', 'yes', 'no', 'about', 'again', 'bad', 'boy', 'but', 'i', 'you', 'we', 'want', 'have', 'where'])
label_map = {label: num for num, label in enumerate(actions)}

DATA_PATH = os.path.join('NEWMP_Data')
no_sequences = 90
sequence_length = 30


sequences, labels = [], []
for action in actions:
    sequence_names = [f for f in os.listdir(os.path.join(DATA_PATH, action)) if f.isdigit()]
    for sequence in np.array(sequence_names).astype(int):
        window = []
        for frame_num in range(sequence_length):
            res = np.load(os.path.join(DATA_PATH, action, str(sequence), f"{frame_num}.npy"))
            window.append(res)
        sequences.append(window)
        labels.append(label_map[action])

X = np.array(sequences)
labels_np = np.array(labels)
y = to_categorical(labels_np).astype(int)


class_weights_array = class_weight.compute_class_weight('balanced', classes=np.unique(labels_np), y=labels_np)
class_weights_dict = dict(enumerate(class_weights_array))

class_weights_dict[label_map['hello']] = 4.0
class_weights_dict[label_map['iloveyou']] = 5.5
class_weights_dict[label_map['yes']] = 5.0
class_weights_dict[label_map['no']] = 4.5
class_weights_dict[label_map['again']] = 4.5
class_weights_dict[label_map['we']] = 5.5
class_weights_dict[label_map['want']] = 5.0


class_weights_dict[label_map['boy']] = 4.0
class_weights_dict[label_map['bad']] = 4.0
class_weights_dict[label_map['i']] = 4.5
class_weights_dict[label_map['you']] = 4.5
class_weights_dict[label_map['have']] = 3.5
class_weights_dict[label_map['thanks']] = 3.5
class_weights_dict[label_map['where']] = 4.5


class_weights_dict[label_map['about']] = 3.0
class_weights_dict[label_map['but']] = 3.0


print("Class weights with manual overrides:", class_weights_dict)


X_train, X_val, y_train, y_val = train_test_split(
    X, y, test_size=0.2, stratify=labels_np, random_state=42
)


log_dir = os.path.join('Logs')
tb_callback = TensorBoard(log_dir=log_dir)


model = Sequential()
model.add(LSTM(64, return_sequences=True, activation='relu', input_shape=(sequence_length, 1662)))
model.add(Dropout(0.3))
model.add(LSTM(128, return_sequences=True, activation='relu'))
model.add(Dropout(0.3))
model.add(LSTM(64, return_sequences=False, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(actions.shape[0], activation='softmax'))


model.compile(optimizer=Adam(learning_rate=0.0001),loss='categorical_crossentropy', metrics=['accuracy'])


try:
    model.fit(
        X_train, y_train,
        validation_data=(X_val, y_val),
        epochs=2000,
        callbacks=[tb_callback],
        class_weight=class_weights_dict
    )
except KeyboardInterrupt:
    print("Training interrupted by user.")
finally:
    model.save('action.h5')
    print("Model saved.")