DeepLearningArchitectures/activation.py at main · ada-k/DeepLearningArchitectures · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# hidden layers activation functions
#  ReLU
import matplotlib.pyplot as plt

def relu(input):
    return max(0.0, input)

x = [i for i in range(-10, 10)]
y = [relu(i) for i in x]

plt.plot(x,y)
plt.show()

# sigmoid/logistic
from math import exp

def sigmoid(x):
    return 1/(1 + exp(-x))

x = [i for i in range(-10, 10)]
y = [sigmoid(i) for i in x]

plt.plot(x,y)
plt.show()


# tanh
def tanh(x):
    return (exp(x) - exp(-x)) / (exp(x) + exp(-x))

x = [i for i in range(-10, 10)]
y = [tanh(i) for i in x]

plt.plot(x,y)
plt.show()


# output layer activation functions

# linear/no activation/identity
def linear(x):
    return x


# softmax
def softmax(x):
    return exp(x) / exp(x).sum()

x = [1,2,3]
print(softmax(x).sum())