Merge branch 'master' of https://github.com/octaveguinebretiere/complexPyTorch into octaveguinebretiere-master

Author: Sebastien M. Popoff

complexPyTorch/complexFunctions.py

@@ -40,6 +40,23 @@ def complex_normalize(input):
 def complex_relu(input):
     return relu(input.real).type(torch.complex64)+1j*relu(input.imag).type(torch.complex64)
 
+def complex_relu(input):
+    return relu(input.real).type(torch.complex64)+1j*relu(input.imag).type(torch.complex64)
+
+def complex_sigmoid(input):
+    return sigmoid(input.real).type(torch.complex64)+1j*sigmoid(input.imag).type(torch.complex64)
+
+def complex_tanh(input):
+    return tanh(input.real).type(torch.complex64)+1j*tanh(input.imag).type(torch.complex64)
+
+def complex_opposite(input):
+    return -(input.real).type(torch.complex64)+1j*(-(input.imag).type(torch.complex64))
+
+def complex_stack(input, dim):
+    input_real = [x.real for x in input]
+    input_imag = [x.imag for x in input]
+    return torch.stack(input_real, dim).type(torch.complex64)+1j*torch.stack(input_imag, dim).type(torch.complex64)
+
 def _retrieve_elements_from_indices(tensor, indices):
     flattened_tensor = tensor.flatten(start_dim=-2)
     output = flattened_tensor.gather(dim=-1, index=indices.flatten(start_dim=-2)).view_as(indices)

complexPyTorch/complexLayers.py

@@ -13,8 +13,21 @@
 from torch.nn import Module, Parameter, init
 from torch.nn import Conv2d, Linear, BatchNorm1d, BatchNorm2d
 from torch.nn import ConvTranspose2d
+<<<<<<< HEAD:complexPyTorch/complexLayers.py
 from .complexFunctions import complex_relu, complex_max_pool2d, complex_avg_pool2d
 from .complexFunctions import complex_dropout, complex_dropout2d
+=======
+from complexFunctions import ( 
+    complex_relu,
+    complex_tanh,
+    complex_sigmoid,
+    complex_max_pool2d,
+    complex_avg_pool2d,
+    complex_dropout,
+    complex_dropout2d,
+    complex_opposite,
+)
+>>>>>>> d00e50c5d93386dd85b363a39c24cf783b7914aa:complexLayers.py
 
 def apply_complex(fr, fi, input, dtype = torch.complex64):
     return (fr(input.real)-fi(input.imag)).type(dtype) \
@@ -83,6 +96,16 @@ class ComplexReLU(Module):
 
      def forward(self,input):
          return complex_relu(input)
+    
+class ComplexSigmoid(Module):
+
+     def forward(self,input):
+         return complex_sigmoid(input)
+    
+class ComplexTanh(Module):
+
+     def forward(self,input):
+         return complex_tanh(input)
 
 class ComplexConvTranspose2d(Module):
 
@@ -342,3 +365,243 @@ def forward(self, input):
 
         del Crr, Cri, Cii, Rrr, Rii, Rri, det, s, t
         return input
+
+# class complexGruCell(Module):
+    
+#     def __init__(self):
+
+class ComplexGruCell(Module):
+    """
+    A GRU cell for complex-valued inputs
+    """
+
+    def __init__(self, input_length=10, hidden_length=20):
+        super(ComplexGruCell, self).__init__()
+        self.input_length = input_length
+        self.hidden_length = hidden_length
+
+        # reset gate components
+        self.linear_reset_w1 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_reset_r1 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        self.linear_reset_w2 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_reset_r2 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        # update gate components
+        self.linear_gate_w3 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_gate_r3 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        self.activation_gate = ComplexSigmoid()
+        self.activation_candidate = ComplexTanh()
+
+    def reset_gate(self, x, h):
+        x_1 = self.linear_reset_w1(x)
+        h_1 = self.linear_reset_r1(h)
+        # gate update
+        reset = self.activation_gate(x_1 + h_1)
+        return reset
+
+    def update_gate(self, x, h):
+        x_2 = self.linear_reset_w2(x)
+        h_2 = self.linear_reset_r2(h)
+        z = self.activation_gate(h_2 + x_2)
+        return z
+
+    def update_component(self, x, h, r):
+        x_3 = self.linear_gate_w3(x)
+        h_3 = r * self.linear_gate_r3(h) # element-wise multiplication
+        gate_update = self.activation_candidate(x_3 + h_3)
+        return gate_update
+
+    def forward(self, x, h):
+        # Equation 1. reset gate vector
+        r = self.reset_gate(x, h)
+
+        # Equation 2: the update gate - the shared update gate vector z
+        z = self.update_gate(x, h)
+
+        # Equation 3: The almost output component
+        n = self.update_component(x, h, r)
+
+        # Equation 4: the new hidden state
+        h_new = (1 + complex_opposite(z)) * n + z * h # element-wise multiplication
+
+        return h_new
+    
+class ComplexBNGruCell(Module):
+    """
+    A BN-GRU cell for complex-valued inputs
+    """
+
+    def __init__(self, input_length=10, hidden_length=20):
+        super(ComplexBNGruCell, self).__init__()
+        self.input_length = input_length
+        self.hidden_length = hidden_length
+
+        # reset gate components
+        self.linear_reset_w1 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_reset_r1 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        self.linear_reset_w2 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_reset_r2 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        # update gate components
+        self.linear_gate_w3 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_gate_r3 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        self.activation_gate = ComplexSigmoid()
+        self.activation_candidate = ComplexTanh()
+
+        self.bn = ComplexBatchNorm2d(1)
+        
+    def reset_gate(self, x, h):
+        x_1 = self.linear_reset_w1(x)
+        h_1 = self.linear_reset_r1(h)
+        # gate update
+        reset = self.activation_gate(self.bn(x_1) + self.bn(h_1))
+        return reset
+
+    def update_gate(self, x, h):
+        x_2 = self.linear_reset_w2(x)
+        h_2 = self.linear_reset_r2(h)
+        z = self.activation_gate(self.bn(h_2) + self.bn(x_2))
+        return z
+
+    def update_component(self, x, h, r):
+        x_3 = self.linear_gate_w3(x)
+        h_3 = r * self.bn(self.linear_gate_r3(h)) # element-wise multiplication
+        gate_update = self.activation_candidate(self.bn(self.bn(x_3) + h_3))
+        return gate_update
+
+    def forward(self, x, h):
+        # Equation 1. reset gate vector
+        r = self.reset_gate(x, h)
+
+        # Equation 2: the update gate - the shared update gate vector z
+        z = self.update_gate(x, h)
+
+        # Equation 3: The almost output component
+        n = self.update_component(x, h, r)
+
+        # Equation 4: the new hidden state
+        h_new = (1 + complex_opposite(z)) * n + z * h # element-wise multiplication
+
+        return h_new
+
+    class ComplexGRUCell(Module):
+    """
+    A GRU cell for complex-valued inputs
+    """
+
+    def __init__(self, input_length=10, hidden_length=20):
+        super(ComplexGRUCell, self).__init__()
+        self.input_length = input_length
+        self.hidden_length = hidden_length
+
+        # reset gate components
+        self.linear_reset_w1 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_reset_r1 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        self.linear_reset_w2 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_reset_r2 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        # update gate components
+        self.linear_gate_w3 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_gate_r3 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        self.activation_gate = ComplexSigmoid()
+        self.activation_candidate = ComplexTanh()
+
+    def reset_gate(self, x, h):
+        x_1 = self.linear_reset_w1(x)
+        h_1 = self.linear_reset_r1(h)
+        # gate update
+        reset = self.activation_gate(x_1 + h_1)
+        return reset
+
+    def update_gate(self, x, h):
+        x_2 = self.linear_reset_w2(x)
+        h_2 = self.linear_reset_r2(h)
+        z = self.activation_gate(h_2 + x_2)
+        return z
+
+    def update_component(self, x, h, r):
+        x_3 = self.linear_gate_w3(x)
+        h_3 = r * self.linear_gate_r3(h) # element-wise multiplication
+        gate_update = self.activation_candidate(x_3 + h_3)
+        return gate_update
+
+    def forward(self, x, h):
+        # Equation 1. reset gate vector
+        r = self.reset_gate(x, h)
+
+        # Equation 2: the update gate - the shared update gate vector z
+        z = self.update_gate(x, h)
+
+        # Equation 3: The almost output component
+        n = self.update_component(x, h, r)
+
+        # Equation 4: the new hidden state
+        h_new = (1 + complex_opposite(z)) * n + z * h # element-wise multiplication
+
+        return h_new
+    
+class ComplexBNGRUCell(Module):
+    """
+    A BN-GRU cell for complex-valued inputs
+    """
+
+    def __init__(self, input_length=10, hidden_length=20):
+        super(ComplexBNGRUCell, self).__init__()
+        self.input_length = input_length
+        self.hidden_length = hidden_length
+
+        # reset gate components
+        self.linear_reset_w1 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_reset_r1 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        self.linear_reset_w2 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_reset_r2 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        # update gate components
+        self.linear_gate_w3 = ComplexLinear(self.input_length, self.hidden_length)
+        self.linear_gate_r3 = ComplexLinear(self.hidden_length, self.hidden_length)
+
+        self.activation_gate = ComplexSigmoid()
+        self.activation_candidate = ComplexTanh()
+
+        self.bn = ComplexBatchNorm2d(1)
+        
+    def reset_gate(self, x, h):
+        x_1 = self.linear_reset_w1(x)
+        h_1 = self.linear_reset_r1(h)
+        # gate update
+        reset = self.activation_gate(self.bn(x_1) + self.bn(h_1))
+        return reset
+
+    def update_gate(self, x, h):
+        x_2 = self.linear_reset_w2(x)
+        h_2 = self.linear_reset_r2(h)
+        z = self.activation_gate(self.bn(h_2) + self.bn(x_2))
+        return z
+
+    def update_component(self, x, h, r):
+        x_3 = self.linear_gate_w3(x)
+        h_3 = r * self.bn(self.linear_gate_r3(h)) # element-wise multiplication
+        gate_update = self.activation_candidate(self.bn(self.bn(x_3) + h_3))
+        return gate_update
+
+    def forward(self, x, h):
+        # Equation 1. reset gate vector
+        r = self.reset_gate(x, h)
+
+        # Equation 2: the update gate - the shared update gate vector z
+        z = self.update_gate(x, h)
+
+        # Equation 3: The almost output component
+        n = self.update_component(x, h, r)
+
+        # Equation 4: the new hidden state
+        h_new = (1 + complex_opposite(z)) * n + z * h # element-wise multiplication
+
+        return h_new