diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Celu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Celu.cs new file mode 100644 index 00000000..fcbc08ce --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Celu.cs @@ -0,0 +1,50 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a continuously differentiable exponential linear unit (CELU) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a continuously differentiable exponential linear unit (CELU) activation function.")] +[DisplayName("CELU")] +public class Celu +{ + /// + /// The alpha value for the CELU activation function. + /// + [Description("The alpha value for the CELU activation function.")] + public double Alpha { get; set; } = 1D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a continuously differentiable exponential linear unit (CELU) module. + /// + /// + public IObservable Process() + { + return Observable.Return(CELU(Alpha, Inplace)); + } + + /// + /// Creates a continuously differentiable exponential linear unit (CELU) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => CELU(Alpha, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Elu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Elu.cs new file mode 100644 index 00000000..039902cb --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Elu.cs @@ -0,0 +1,50 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates an exponential linear unit (ELU) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates an exponential linear unit (ELU) activation function.")] +[DisplayName("ELU")] +public class Elu +{ + /// + /// The alpha value for the ELU activation function. + /// + [Description("The alpha value for the ELU activation function")] + public double Alpha { get; set; } = 1D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place")] + public bool Inplace { get; set; } = false; + + /// + /// Creates an exponential linear unit (ELU) module. + /// + /// + public IObservable Process() + { + return Observable.Return(ELU(Alpha, Inplace)); + } + + /// + /// Creates an exponential linear unit (ELU) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ELU(Alpha, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Gelu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Gelu.cs new file mode 100644 index 00000000..978c9bf4 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Gelu.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a gaussian error linear unit (GELU) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a gaussian error linear unit (GELU) activation function.")] +[DisplayName("GELU")] +public class Gelu +{ + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool InPlace { get; set; } = false; + + /// + /// Creates a gaussian error linear unit (GELU) module. + /// + /// + public IObservable Process() + { + return Observable.Return(GELU(InPlace)); + } + + /// + /// Creates a gaussian error linear unit (GELU) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => GELU(InPlace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Glu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Glu.cs new file mode 100644 index 00000000..a022ec4d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Glu.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a gated linear unit (GLU) module. +/// +/// +/// See for more information. +/// +[Description("Creates a gated linear unit (GLU) module.")] +[DisplayName("GLU")] +public class Glu +{ + /// + /// The dimension on which to split the input tensor. + /// + [Description("The dimension on which to split the input tensor.")] + public long Dim { get; set; } = -1; + + /// + /// Creates a gated linear unit (GLU) module. + /// + /// + public IObservable Process() + { + return Observable.Return(GLU(Dim)); + } + + /// + /// Creates a gated linear unit (GLU) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => GLU(Dim)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardshrink.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardshrink.cs new file mode 100644 index 00000000..53cb6ff3 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardshrink.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a Hardshrink module. +/// +/// +/// See for more information. +/// +[Description("Creates a Hardshrink module.")] +public class Hardshrink +{ + /// + /// The lambda parameter for the Hardshrink function. + /// + [Description("The lambda parameter for the Hardshrink function")] + public double Lambda { get; set; } = 0.5D; + + /// + /// Creates a Hardshrink module. + /// + /// + public IObservable Process() + { + return Observable.Return(Hardshrink(Lambda)); + } + + /// + /// Creates a Hardshrink module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Hardshrink(Lambda)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardsigmoid.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardsigmoid.cs new file mode 100644 index 00000000..1a5f4156 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardsigmoid.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a Hardsigmoid module. +/// +/// +/// See for more information. +/// +[Description("Creates a Hardsigmoid module.")] +public class Hardsigmoid +{ + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a Hardsigmoid module. + /// + /// + public IObservable Process() + { + return Observable.Return(Hardsigmoid(Inplace)); + } + + /// + /// Creates a Hardsigmoid module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Hardsigmoid(Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardswish.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardswish.cs new file mode 100644 index 00000000..73d2a0c2 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardswish.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a Hardswish module. +/// +/// +/// See for more information. +/// +[Description("Creates a Hardswish module.")] +public class Hardswish +{ + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a Hardswish module. + /// + /// + public IObservable Process() + { + return Observable.Return(Hardswish(Inplace)); + } + + /// + /// Creates a Hardswish module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Hardswish(Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardtanh.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardtanh.cs new file mode 100644 index 00000000..3abbbc81 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Hardtanh.cs @@ -0,0 +1,55 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a Hardtanh module. +/// +/// +/// See for more information. +/// +[Description("Creates a Hardtanh module.")] +public class Hardtanh +{ + /// + /// The minimum value of the linear region range. + /// + [Description("The minimum value of the linear region range.")] + public double MinVal { get; set; } = -1D; + + /// + /// The maximum value of the linear region range. + /// + [Description("The maximum value of the linear region range.")] + public double MaxVal { get; set; } = 1D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a Hardtanh module. + /// + /// + public IObservable Process() + { + return Observable.Return(Hardtanh(MinVal, MaxVal, Inplace)); + } + + /// + /// Creates a Hardtanh module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Hardtanh(MinVal, MaxVal, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/LeakyRelu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/LeakyRelu.cs new file mode 100644 index 00000000..e8fa6cee --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/LeakyRelu.cs @@ -0,0 +1,50 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a leaky rectified linear unit (LeakyReLU) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a leaky rectified linear unit (LeakyReLU) activation function.")] +[DisplayName("LeakyReLU")] +public class LeakyRelu +{ + /// + /// The angle of the negative slope. + /// + [Description("The angle of the negative slope.")] + public double NegativeSlope { get; set; } = 0.01D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a leaky rectified linear unit (LeakyReLU) module. + /// + /// + public IObservable Process() + { + return Observable.Return(LeakyReLU(NegativeSlope, Inplace)); + } + + /// + /// Creates a leaky rectified linear unit (LeakyReLU) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => LeakyReLU(NegativeSlope, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/LogSigmoid.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/LogSigmoid.cs new file mode 100644 index 00000000..99f03b29 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/LogSigmoid.cs @@ -0,0 +1,37 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a log sigmoid module. +/// +/// +/// See for more information. +/// +[Description("Creates a log sigmoid module.")] +public class LogSigmoid +{ + /// + /// Creates a LogSigmoid module. + /// + /// + public IObservable Process() + { + return Observable.Return(LogSigmoid()); + } + + /// + /// Creates a LogSigmoid module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => LogSigmoid()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/LogSoftmax.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/LogSoftmax.cs new file mode 100644 index 00000000..b9d14bb7 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/LogSoftmax.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a log softmax activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a log softmax activation function.")] +public class LogSoftmax +{ + /// + /// The dimension along which LogSoftmax will be computed. + /// + [Description("The dimension along which LogSoftmax will be computed")] + public long Dim { get; set; } + + /// + /// Creates a LogSoftmax module. + /// + /// + public IObservable Process() + { + return Observable.Return(LogSoftmax(Dim)); + } + + /// + /// Creates a LogSoftmax module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => LogSoftmax(Dim)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Mish.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Mish.cs new file mode 100644 index 00000000..1877e79f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Mish.cs @@ -0,0 +1,37 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a mish activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a mish activation function.")] +public class Mish +{ + /// + /// Creates a Mish module. + /// + /// + public IObservable Process() + { + return Observable.Return(Mish()); + } + + /// + /// Creates a Mish module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Mish()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/MultiheadAttention.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/MultiheadAttention.cs new file mode 100644 index 00000000..76370225 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/MultiheadAttention.cs @@ -0,0 +1,85 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a multi-head attention mechanism. +/// +/// +/// See for more information. +/// +[Description("Creates a multi-head attention mechanism.")] +public class MultiheadAttention +{ + /// + /// The dimension of the model. + /// + [Description("The dimension of the model.")] + public long EmbeddedDim { get; set; } + + /// + /// The number of parallel attention heads. + /// + [Description("The number of parallel attention heads.")] + public long NumHeads { get; set; } + + /// + /// The dropout probability on attended weights. + /// + [Description("The dropout probability on attended weights.")] + public double Dropout { get; set; } = 0D; + + /// + /// If true, adds a learnable bias to the input/output projection layers. + /// + [Description("If true, adds a learnable bias to the input/output projection layers.")] + public bool Bias { get; set; } = true; + + /// + /// If true, adds bias to the key and value sequences at dimension 0. + /// + [Description("If true, adds bias to the key and value sequences at dimension 0.")] + public bool AddBiasKeyValue { get; set; } = false; + + /// + /// If true, adds a new batch of zeros to the key and value sequences at dimension 1. + /// + [Description("If true, adds a new batch of zeros to the key and value sequences at dimension 1.")] + public bool AddZeroAttention { get; set; } = false; + + /// + /// The total number of features for keys. + /// + [Description("The total number of features for keys.")] + public long? KeyDim { get; set; } = null; + + /// + /// The total number of features for values. + /// + [Description("The total number of features for values.")] + public long? ValueDim { get; set; } = null; + + /// + /// Creates a Multi-head attention module. + /// + /// + public IObservable Process() + { + return Observable.Return(MultiheadAttention(EmbeddedDim, NumHeads, Dropout, Bias, AddBiasKeyValue, AddZeroAttention, KeyDim, ValueDim)); + } + + /// + /// Creates a Multi-head attention module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MultiheadAttention(EmbeddedDim, NumHeads, Dropout, Bias, AddBiasKeyValue, AddZeroAttention, KeyDim, ValueDim)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Prelu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Prelu.cs new file mode 100644 index 00000000..ea8a0beb --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Prelu.cs @@ -0,0 +1,65 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a parametric rectified linear unit (PReLU) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a parametric rectified linear unit (PReLU) activation function.")] +[DisplayName("PReLU")] +public class Prelu +{ + /// + /// The number of parameters to learn. + /// + [Description("The number of parameters to learn.")] + public long NumParameters { get; set; } + + /// + /// The initial value for the learnable parameters. + /// + [Description("The initial value for the learnable parameters.")] + public double InitialValue { get; set; } = 0.25D; + + /// + /// The desired device of returned tensor. + /// + [XmlIgnore] + [Description("The desired device of returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of returned tensor. + /// + [Description("The desired data type of returned tensor.")] + [TypeConverter(typeof(ScalarTypeConverter))] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a parametric rectified linear unit (PReLU) activation function. + /// + /// + public IObservable Process() + { + return Observable.Return(PReLU(NumParameters, InitialValue, Device, Type)); + } + + /// + /// Creates a parametric rectified linear unit (PReLU) activation function. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => PReLU(NumParameters, InitialValue, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Relu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Relu.cs new file mode 100644 index 00000000..53107768 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Relu.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a rectified linear unit (ReLU) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a rectified linear unit (ReLU) activation function.")] +[DisplayName("ReLU")] +public class Relu +{ + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a rectified linear unit (ReLU) module. + /// + /// + public IObservable Process() + { + return Observable.Return(ReLU(Inplace)); + } + + /// + /// Creates a rectified linear unit (ReLU) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ReLU(Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/ReluBounded.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/ReluBounded.cs new file mode 100644 index 00000000..80ca0788 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/ReluBounded.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a bounded rectified linear unit (ReLU6) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a bounded rectified linear unit (ReLU6) activation function.")] +[DisplayName("ReLU6")] +public class ReluBounded +{ + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a bounded rectified linear unit (ReLU6) module. + /// + /// + public IObservable Process() + { + return Observable.Return(ReLU6(Inplace)); + } + + /// + /// Creates a bounded rectified linear unit (ReLU6) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ReLU6(Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Rrelu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Rrelu.cs new file mode 100644 index 00000000..632b8edc --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Rrelu.cs @@ -0,0 +1,56 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a randomized leaky rectified linear unit (RReLU) module. +/// +/// +/// See for more information. +/// +[Description("Creates a randomized leaky rectified linear unit (RReLU) module.")] +[DisplayName("RReLU")] +public class Rrelu +{ + /// + /// The lower bound of the uniform distribution. + /// + [Description("The lower bound of the uniform distribution.")] + public double Lower { get; set; } = 0.125D; + + /// + /// The upper bound of the uniform distribution. + /// + [Description("The upper bound of the uniform distribution.")] + public double Upper { get; set; } = 0.3333333333333333D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a randomized leaky rectified linear unit (RReLU) module. + /// + /// + public IObservable Process() + { + return Observable.Return(RReLU(Lower, Upper, Inplace)); + } + + /// + /// Creates a randomized leaky rectified linear unit (RReLU) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => RReLU(Lower, Upper, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Selu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Selu.cs new file mode 100644 index 00000000..6d460b9e --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Selu.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a scaled exponential linear unit (SELU) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a scaled exponential linear unit (SELU) activation function.")] +[DisplayName("SeLU")] +public class Selu +{ + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a scaled exponential linear unit (SELU) module. + /// + /// + public IObservable Process() + { + return Observable.Return(SELU(Inplace)); + } + + /// + /// Creates a scaled exponential linear unit (SELU) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => SELU(Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Sigmoid.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Sigmoid.cs new file mode 100644 index 00000000..17460059 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Sigmoid.cs @@ -0,0 +1,37 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a sigmoid activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a sigmoid activation function.")] +public class Sigmoid +{ + /// + /// Creates a sigmoid module. + /// + /// + public IObservable Process() + { + return Observable.Return(Sigmoid()); + } + + /// + /// Creates a sigmoid module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Sigmoid()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Silu.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Silu.cs new file mode 100644 index 00000000..a6f88818 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Silu.cs @@ -0,0 +1,38 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a sigmoid weighted linear unit (SiLU) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a sigmoid weighted linear unit (SiLU) activation function.")] +[DisplayName("SiLU")] +public class Silu +{ + /// + /// Creates a sigmoid weighted linear unit (SiLU) module. + /// + /// + public IObservable Process() + { + return Observable.Return(SiLU()); + } + + /// + /// Creates a sigmoid weighted linear unit (SiLU) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => SiLU()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softmax.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softmax.cs new file mode 100644 index 00000000..a77ba716 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softmax.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a softmax activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a softmax activation function.")] +public class Softmax +{ + /// + /// The dimension along which Softmax will be computed. + /// + [Description("The dimension along which Softmax will be computed.")] + public long Dim { get; set; } + + /// + /// Creates a Softmax module. + /// + /// + public IObservable Process() + { + return Observable.Return(Softmax(Dim)); + } + + /// + /// Creates a Softmax module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Softmax(Dim)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softmax2d.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softmax2d.cs new file mode 100644 index 00000000..55ad3c75 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softmax2d.cs @@ -0,0 +1,37 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a 2d softmax activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a 2d softmax activation function.")] +public class Softmax2d +{ + /// + /// Creates a Softmax2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(Softmax2d()); + } + + /// + /// Creates a Softmax2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Softmax2d()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softmin.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softmin.cs new file mode 100644 index 00000000..2fe4119f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softmin.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a softmin activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a softmin activation function.")] +public class Softmin +{ + /// + /// The dimension along which softmin will be computed. + /// + [Description("The dimension along which softmin will be computed.")] + public long Dim { get; set; } + + /// + /// Creates a Softmin module. + /// + /// + public IObservable Process() + { + return Observable.Return(Softmin(Dim)); + } + + /// + /// Creates a Softmin module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Softmin(Dim)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softplus.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softplus.cs new file mode 100644 index 00000000..2f9a918b --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softplus.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a softplus activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a softplus module.")] +public class Softplus +{ + /// + /// The beta value for the softplus formula. + /// + [Description("The beta value for the softplus formula.")] + public double Beta { get; set; } = 1D; + + /// + /// The threshold value for which values above it use a linear function. + /// + [Description("The threshold value for which values above it use a linear function.")] + public double Threshold { get; set; } = 20D; + + /// + /// Creates a Softplus module. + /// + /// + public IObservable Process() + { + return Observable.Return(Softplus(Beta, Threshold)); + } + + /// + /// Creates a Softplus module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Softplus(Beta, Threshold)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softshrink.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softshrink.cs new file mode 100644 index 00000000..2f00f8fa --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softshrink.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a softshrink activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a softshrink activation function.")] +public class Softshrink +{ + /// + /// The lambda value for the softshrink formula. + /// + [Description("The lambda value for the softshrink formula.")] + public double Lambda { get; set; } = 0.5D; + + /// + /// Creates a Softshrink module. + /// + /// + public IObservable Process() + { + return Observable.Return(Softshrink(Lambda)); + } + + /// + /// Creates a Softshrink module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Softshrink(Lambda)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softsign.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softsign.cs new file mode 100644 index 00000000..1c94bdb1 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Softsign.cs @@ -0,0 +1,37 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a softsign activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a softsign activation function.")] +public class Softsign +{ + /// + /// Creates a Softsign module. + /// + /// + public IObservable Process() + { + return Observable.Return(Softsign()); + } + + /// + /// Creates a Softsign module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Softsign()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Tanh.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Tanh.cs new file mode 100644 index 00000000..851ae14a --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Tanh.cs @@ -0,0 +1,37 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a hyperbolic tangent (tanh) activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a hyperbolic tangent (tanh) activation function.")] +public class Tanh +{ + /// + /// Creates a Tanh module. + /// + /// + public IObservable Process() + { + return Observable.Return(Tanh()); + } + + /// + /// Creates a Tanh module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Tanh()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Tanhshrink.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Tanhshrink.cs new file mode 100644 index 00000000..904f3c41 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Tanhshrink.cs @@ -0,0 +1,37 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a tanhshrink activation function. +/// +/// +/// See for more information. +/// +[Description("Creates a tanhshrink activation function.")] +public class Tanhshrink +{ + /// + /// Creates a Tanhshrink module. + /// + /// + public IObservable Process() + { + return Observable.Return(Tanhshrink()); + } + + /// + /// Creates a Tanhshrink module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Tanhshrink()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Threshold.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Threshold.cs new file mode 100644 index 00000000..1b31c2ca --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunction/Threshold.cs @@ -0,0 +1,55 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.ActivationFunction; + +/// +/// Represents an operator that creates a threshold activation function. +/// +/// +/// See for more information. +/// +[Description("Represents an operator that creates a threshold activation function.")] +public class Threshold +{ + /// + /// The threshold value. + /// + [Description("The threshold value.")] + public double ThresholdValue { get; set; } + + /// + /// The value used to replace values below the threshold. + /// + [Description("The value used to replace values below the threshold.")] + public double FillValue { get; set; } + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a Threshold module. + /// + /// + public IObservable Process() + { + return Observable.Return(Threshold(ThresholdValue, FillValue, Inplace)); + } + + /// + /// Creates a Threshold module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Threshold(ThresholdValue, FillValue, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ActivationFunctionBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunctionBuilder.cs new file mode 100644 index 00000000..6c2bdf5a --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ActivationFunctionBuilder.cs @@ -0,0 +1,69 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates an activation function. +/// +[XmlInclude(typeof(ActivationFunction.Celu))] +[XmlInclude(typeof(ActivationFunction.Elu))] +[XmlInclude(typeof(ActivationFunction.Glu))] +[XmlInclude(typeof(ActivationFunction.Gelu))] +[XmlInclude(typeof(ActivationFunction.Hardshrink))] +[XmlInclude(typeof(ActivationFunction.Hardsigmoid))] +[XmlInclude(typeof(ActivationFunction.Hardswish))] +[XmlInclude(typeof(ActivationFunction.Hardtanh))] +[XmlInclude(typeof(ActivationFunction.LeakyRelu))] +[XmlInclude(typeof(ActivationFunction.LogSigmoid))] +[XmlInclude(typeof(ActivationFunction.LogSoftmax))] +[XmlInclude(typeof(ActivationFunction.Mish))] +[XmlInclude(typeof(ActivationFunction.MultiheadAttention))] +[XmlInclude(typeof(ActivationFunction.Prelu))] +[XmlInclude(typeof(ActivationFunction.Rrelu))] +[XmlInclude(typeof(ActivationFunction.Relu))] +[XmlInclude(typeof(ActivationFunction.ReluBounded))] +[XmlInclude(typeof(ActivationFunction.Selu))] +[XmlInclude(typeof(ActivationFunction.Sigmoid))] +[XmlInclude(typeof(ActivationFunction.Silu))] +[XmlInclude(typeof(ActivationFunction.Softmax))] +[XmlInclude(typeof(ActivationFunction.Softmax2d))] +[XmlInclude(typeof(ActivationFunction.Softmin))] +[XmlInclude(typeof(ActivationFunction.Softplus))] +[XmlInclude(typeof(ActivationFunction.Softshrink))] +[XmlInclude(typeof(ActivationFunction.Softsign))] +[XmlInclude(typeof(ActivationFunction.Tanh))] +[XmlInclude(typeof(ActivationFunction.Tanhshrink))] +[XmlInclude(typeof(ActivationFunction.Threshold))] +[DefaultProperty(nameof(ActivationFunction))] +[Combinator] +[Description("Creates an activation function.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class ActivationFunctionBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "ActivationFunction"; + + /// + /// Initializes a new instance of the class. + /// + public ActivationFunctionBuilder() + { + Module = new ActivationFunction.Relu(); + } + + /// + /// Gets or sets the specific activation function to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific activation function to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object ActivationFunction + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Architecture/AlexNet.cs b/src/Bonsai.ML.Torch/NeuralNets/Architecture/AlexNet.cs new file mode 100644 index 00000000..07047937 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Architecture/AlexNet.cs @@ -0,0 +1,72 @@ +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Architecture; + +/// +/// Modified version of original AlexNet to fix CIFAR10 32x32 images. +/// +internal class AlexNet : Module +{ + private readonly Module features; + private readonly Module avgPool; + private readonly Module classifier; + + /// + /// Constructs a new AlexNet model. + /// + /// + /// + /// + public AlexNet(string name, int numClasses, Device device = null) : base(name) + { + features = Sequential( + ("c1", Conv2d(3, 64, kernel_size: 3, stride: 2, padding: 1)), + ("r1", ReLU(inplace: true)), + ("mp1", MaxPool2d(kernel_size: [ 2, 2 ])), + ("c2", Conv2d(64, 192, kernel_size: 3, padding: 1)), + ("r2", ReLU(inplace: true)), + ("mp2", MaxPool2d(kernel_size: [ 2, 2 ])), + ("c3", Conv2d(192, 384, kernel_size: 3, padding: 1)), + ("r3", ReLU(inplace: true)), + ("c4", Conv2d(384, 256, kernel_size: 3, padding: 1)), + ("r4", ReLU(inplace: true)), + ("c5", Conv2d(256, 256, kernel_size: 3, padding: 1)), + ("r5", ReLU(inplace: true)), + ("mp3", MaxPool2d(kernel_size: [ 2, 2 ]))); + + avgPool = AdaptiveAvgPool2d([ 2, 2 ]); + + classifier = Sequential( + ("d1", nn.Dropout()), + ("l1", nn.Linear(256 * 2 * 2, 4096)), + ("r1", ReLU(inplace: true)), + ("d2", nn.Dropout()), + ("l2", nn.Linear(4096, 4096)), + ("r3", ReLU(inplace: true)), + ("d3", nn.Dropout()), + ("l3", nn.Linear(4096, numClasses)) + ); + + RegisterComponents(); + + if (device != null && device.type != DeviceType.CPU) + this.to(device); + } + + /// + /// Forward pass of the AlexNet model. + /// + /// + /// + public override Tensor forward(Tensor input) + { + var f = features.forward(input); + var avg = avgPool.forward(f); + + var x = avg.view([ avg.shape[0], 256 * 2 * 2 ]); + + return classifier.forward(x); + } +} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Architecture/Mnist.cs b/src/Bonsai.ML.Torch/NeuralNets/Architecture/Mnist.cs new file mode 100644 index 00000000..aca74443 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Architecture/Mnist.cs @@ -0,0 +1,84 @@ +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Architecture; + /// +/// Represents a simple convolutional neural network for the MNIST dataset. +/// +internal class Mnist : Module +{ + private readonly Module conv1; + private readonly Module conv2; + private readonly Module fc1; + private readonly Module fc2; + + private readonly Module pool1; + + private readonly Module relu1; + private readonly Module relu2; + private readonly Module relu3; + + private readonly Module dropout1; + private readonly Module dropout2; + + private readonly Module flatten; + private readonly Module logsm; + + /// + /// Constructs a new Mnist model. + /// + /// + /// + /// + public Mnist(string name, int numClasses, Device device = null) : base(name) + { + conv1 = Conv2d(1, 32, 3); + conv2 = Conv2d(32, 64, 3); + fc1 = nn.Linear(9216, 128); + fc2 = nn.Linear(128, numClasses); + + pool1 = MaxPool2d(kernel_size: [2, 2]); + + relu1 = ReLU(); + relu2 = ReLU(); + relu3 = ReLU(); + + dropout1 = nn.Dropout(0.25); + dropout2 = nn.Dropout(0.5); + + flatten = nn.Flatten(); + logsm = LogSoftmax(1); + + RegisterComponents(); + + if (device != null && device.type != DeviceType.CPU) + this.to(device); + } + + /// + /// Forward pass of the Mnist model. + /// + /// + /// + public override Tensor forward(Tensor input) + { + var l11 = conv1.forward(input); + var l12 = relu1.forward(l11); + + var l21 = conv2.forward(l12); + var l22 = relu2.forward(l21); + var l23 = pool1.forward(l22); + var l24 = dropout1.forward(l23); + + var x = flatten.forward(l24); + + var l31 = fc1.forward(x); + var l32 = relu3.forward(l31); + var l33 = dropout2.forward(l32); + + var l41 = fc2.forward(l33); + + return logsm.forward(l41); + } +} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Architecture/MobileNet.cs b/src/Bonsai.ML.Torch/NeuralNets/Architecture/MobileNet.cs new file mode 100644 index 00000000..60a67d49 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Architecture/MobileNet.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Architecture; + +/// +/// MobileNet model. +/// +internal class MobileNet : Module +{ + private readonly long[] planes = [ 64, 128, 128, 256, 256, 512, 512, 512, 512, 512, 512, 1024, 1024 ]; + private readonly long[] strides = [ 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1 ]; + + private readonly Module layers; + + /// + /// Constructs a new MobileNet model. + /// + /// + /// + /// + /// + public MobileNet(string name, int numClasses, Device device = null) : base(name) + { + if (planes.Length != strides.Length) throw new ArgumentException("'planes' and 'strides' must have the same length."); + + var modules = new List<(string, Module)> + { + ($"conv2d-first", Conv2d(3, 32, kernel_size: 3, stride: 1, padding: 1, bias: false)), + ($"bnrm2d-first", BatchNorm2d(32)), + ($"relu-first", ReLU()) + }; + MakeLayers(modules, 32); + modules.Add(("avgpool", AvgPool2d([2, 2]))); + modules.Add(("flatten", nn.Flatten())); + modules.Add(($"linear", nn.Linear(planes[planes.Length-1], numClasses))); + + layers = Sequential(modules); + + RegisterComponents(); + + if (device != null && device.type != DeviceType.CPU) + this.to(device); + } + + private void MakeLayers(List<(string, Module)> modules, long in_planes) + { + + for (var i = 0; i < strides.Length; i++) { + var out_planes = planes[i]; + var stride = strides[i]; + + modules.Add(($"conv2d-{i}a", Conv2d(in_planes, in_planes, kernel_size: 3, stride: stride, padding: 1, groups: in_planes, bias: false))); + modules.Add(($"bnrm2d-{i}a", BatchNorm2d(in_planes))); + modules.Add(($"relu-{i}a", ReLU())); + modules.Add(($"conv2d-{i}b", Conv2d(in_planes, out_planes, kernel_size: 1L, stride: 1L, padding: 0L, bias: false))); + modules.Add(($"bnrm2d-{i}b", BatchNorm2d(out_planes))); + modules.Add(($"relu-{i}b", ReLU())); + + in_planes = out_planes; + } + } + + /// + /// Forward pass of the MobileNet model. + /// + /// + /// + public override Tensor forward(Tensor input) + { + return layers.forward(input); + } +} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Architecture/ModelArchitecture.cs b/src/Bonsai.ML.Torch/NeuralNets/Architecture/ModelArchitecture.cs new file mode 100644 index 00000000..1a1b2552 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Architecture/ModelArchitecture.cs @@ -0,0 +1,22 @@ +namespace Bonsai.ML.Torch.NeuralNets.Architecture; + +/// +/// Represents canonical model architectures. +/// +public enum ModelArchitecture +{ + /// + /// The AlexNet model architecture. + /// + AlexNet, + + /// + /// The MobileNet model architecture. + /// + MobileNet, + + /// + /// The Mnist model architecture. + /// + Mnist +} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Backward.cs b/src/Bonsai.ML.Torch/NeuralNets/Backward.cs index 52257d68..7468a210 100644 --- a/src/Bonsai.ML.Torch/NeuralNets/Backward.cs +++ b/src/Bonsai.ML.Torch/NeuralNets/Backward.cs @@ -1,75 +1,25 @@ using System; using System.ComponentModel; using System.Reactive.Linq; -using System.Xml.Serialization; using static TorchSharp.torch; -using static TorchSharp.torch.nn; -using static TorchSharp.torch.optim; -namespace Bonsai.ML.Torch.NeuralNets +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that computes backward on the input tensor. +/// +[Combinator] +[Description("Computes backward on the input tensor.")] +[WorkflowElementCategory(ElementCategory.Sink)] +public class Backward { /// - /// Trains a model using backpropagation. + /// Computes backward on the input tensor. /// - [Combinator] - [ResetCombinator] - [Description("Trains a model using backpropagation.")] - [WorkflowElementCategory(ElementCategory.Transform)] - public class Backward + /// + /// + public IObservable Process(IObservable source) { - /// - /// The optimizer to use for training. - /// - public Optimizer Optimizer { get; set; } - - /// - /// The model to train. - /// - [XmlIgnore] - public ITorchModule Model { get; set; } - - /// - /// The loss function to use for training. - /// - public Loss Loss { get; set; } - - /// - /// Trains the model using backpropagation. - /// - /// - /// - public IObservable Process(IObservable> source) - { - optim.Optimizer optimizer = Optimizer switch - { - Optimizer.Adam => Adam(Model.Module.parameters()), - _ => throw new ArgumentException($"Selected optimizer, {Optimizer} is currently not supported.") - }; - - Module loss = Loss switch - { - Loss.NegativeLogLikelihood => NLLLoss(), - _ => throw new ArgumentException($"Selected loss, {Loss} is currently not supported.") - }; - - var scheduler = lr_scheduler.StepLR(optimizer, 1, 0.7); - Model.Module.train(); - - return source.Select((input) => { - var (data, target) = input; - using (_ = NewDisposeScope()) - { - optimizer.zero_grad(); - - var prediction = Model.Forward(data); - var output = loss.forward(prediction, target); - - output.backward(); - - optimizer.step(); - return output.MoveToOuterDisposeScope(); - } - }); - } + return source.Do(input => input.backward()); } } \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/CollectParameters.cs b/src/Bonsai.ML.Torch/NeuralNets/CollectParameters.cs new file mode 100644 index 00000000..3b33577f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/CollectParameters.cs @@ -0,0 +1,34 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Linq; +using System.Collections.Generic; +using TorchSharp.Modules; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that collects the parameters of torch modules into a collection. +/// +[Combinator] +[Description("Collects the parameters from torch modules into a collection.")] +[WorkflowElementCategory(ElementCategory.Combinator)] +public class CollectParameters +{ + /// + /// Collects the parameters from torch modules into a collection. + /// + /// + /// + public IObservable> Process(params IObservable[] sources) + { + return Observable + .Concat(sources.Select(source => + source.Take(1))) + .SelectMany(module => + { + return module.parameters(recurse: true); + }).ToList(); + } +} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Container/Sequential.cs b/src/Bonsai.ML.Torch/NeuralNets/Container/Sequential.cs new file mode 100644 index 00000000..5bf4676c --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Container/Sequential.cs @@ -0,0 +1,28 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; +using System.Linq; +using System.Collections.Generic; + +namespace Bonsai.ML.Torch.NeuralNets.Container; + +/// +/// Represents an operator that creates a sequential container. +/// +/// +/// See for more information. +/// +[Description("Creates a sequential container.")] +public class Sequential +{ + /// + /// Creates a sequential container from the input modules. + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable> + { + return source.Select(modules => Sequential(modules)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ContainerBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/ContainerBuilder.cs new file mode 100644 index 00000000..cbd23287 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ContainerBuilder.cs @@ -0,0 +1,44 @@ +using System.ComponentModel; +using System.Xml.Serialization; +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a torch module for convolution operations. +/// +[XmlInclude(typeof(Container.Sequential))] +[DefaultProperty(nameof(ContainerModule))] +[Combinator] +[Description("Creates a sequential container for torch modules.")] +[WorkflowElementCategory(ElementCategory.Transform)] +public class ContainerBuilder : ModuleCombinatorBuilder, INamedElement +{ + /// + public override Range ArgumentRange => Range.Create(1, 1); + + /// + internal override string BuilderName => "Container"; + + /// + /// Initializes a new instance of the class. + /// + public ContainerBuilder() + { + Module = new Container.Sequential(); + } + + /// + /// Gets or sets the specific container module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific container module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object ContainerModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv1d.cs new file mode 100644 index 00000000..fdf76173 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv1d.cs @@ -0,0 +1,106 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Convolution; + +/// +/// Represents an operator that creates a 1D convolution module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D convolution module.")] +public class Conv1d +{ + /// + /// The number of input channels in the input tensor. + /// + [Description("The number of input channels in the input tensor.")] + public long InChannels { get; set; } + + /// + /// The number of output channels produced by the convolution. + /// + [Description("The number of output channels produced by the convolution.")] + public long OutChannels { get; set; } + + /// + /// The size of the convolution kernel. + /// + [Description("The size of the convolution kernel.")] + public long KernelSize { get; set; } + + /// + /// The stride of the convolution. + /// + [Description("The stride of the convolution.")] + public long Stride { get; set; } = 1; + + /// + /// The padding added to both sides of the input. + /// + [Description("The padding added to both sides of the input.")] + public long Padding { get; set; } = 0; + + /// + /// The spacing between kernel elements. + /// + [Description("The spacing between kernel elements.")] + public long Dilation { get; set; } = 1; + + /// + /// The mode of padding. + /// + [Description("The mode of padding.")] + public PaddingModes PaddingMode { get; set; } = PaddingModes.Zeros; + + /// + /// The number of blocked connections from input channels to output channels. + /// + [Description("The number of blocked connections from input channels to output channels.")] + public long Groups { get; set; } = 1; + + /// + /// If true, adds a learnable bias to the output. + /// + [Description("If true, adds a learnable bias to the output")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a Conv1d module. + /// + /// + public IObservable Process() + { + return Observable.Return(Conv1d(InChannels, OutChannels, KernelSize, Stride, Padding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } + + /// + /// Creates a Conv1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Conv1d(InChannels, OutChannels, KernelSize, Stride, Padding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv2d.cs new file mode 100644 index 00000000..c426b5e7 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv2d.cs @@ -0,0 +1,110 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Convolution; + +/// +/// Represents an operator that creates a 2D convolution module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D convolution module.")] +public class Conv2d +{ + /// + /// The number of input channels in the input tensor. + /// + [Description("The number of input channels in the input tensor.")] + public long InChannels { get; set; } + + /// + /// The number of output channels produced by the convolution. + /// + [Description("The number of output channels produced by the convolution.")] + public long OutChannels { get; set; } + + /// + /// The size of the convolution kernel. + /// + [Description("The size of the convolution kernel.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) KernelSize { get; set; } + + /// + /// The stride of the convolution. + /// + [Description("The stride of the convolution.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Stride { get; set; } = null; + + /// + /// The padding added to all four sides of the input. + /// + [Description("The padding added to all four sides of the input.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Padding { get; set; } = null; + + /// + /// The spacing between kernel elements. + /// + [Description("The spacing between kernel elements.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Dilation { get; set; } = null; + + /// + /// The mode of padding. + /// + [Description("The mode of padding.")] + public PaddingModes PaddingMode { get; set; } = PaddingModes.Zeros; + + /// + /// The number of blocked connections from input channels to output channels. + /// + [Description("The number of blocked connections from input channels to output channels.")] + public long Groups { get; set; } = 1; + + /// + /// If true, adds a learnable bias to the output. + /// + [Description("If true, adds a learnable bias to the output")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a Conv2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(Conv2d(InChannels, OutChannels, KernelSize, Stride, Padding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } + + /// + /// Creates a Conv2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Conv2d(InChannels, OutChannels, KernelSize, Stride, Padding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv3d.cs new file mode 100644 index 00000000..c02e30d6 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Conv3d.cs @@ -0,0 +1,110 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Convolution; + +/// +/// Represents an operator that creates a 3D convolution module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D convolution module.")] +public class Conv3d +{ + /// + /// The number of input channels in the input tensor. + /// + [Description("The number of input channels in the input tensor.")] + public long InChannels { get; set; } + + /// + /// The number of output channels produced by the convolution. + /// + [Description("The number of output channels produced by the convolution.")] + public long OutChannels { get; set; } + + /// + /// The size of the convolution kernel. + /// + [Description("The size of the convolution kernel.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long) KernelSize { get; set; } + + /// + /// The stride of the convolution. + /// + [Description("The stride of the convolution.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Stride { get; set; } = null; + + /// + /// The padding to add to all six sides of the input. + /// + [Description("The padding to add to all six sides of the input.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Padding { get; set; } = null; + + /// + /// The spacing between kernel elements. + /// + [Description("The spacing between kernel elements.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Dilation { get; set; } = null; + + /// + /// The mode of padding. + /// + [Description("The mode of padding.")] + public PaddingModes PaddingMode { get; set; } = PaddingModes.Zeros; + + /// + /// The number of blocked connections from input channels to output channels. + /// + [Description("The number of blocked connections from input channels to output channels.")] + public long Groups { get; set; } = 1; + + /// + /// If true, adds a learnable bias to the output. + /// + [Description("If true, adds a learnable bias to the output")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a Conv3d module. + /// + /// + public IObservable Process() + { + return Observable.Return(Conv3d(InChannels, OutChannels, KernelSize, Stride, Padding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } + + /// + /// Creates a Conv3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Conv3d(InChannels, OutChannels, KernelSize, Stride, Padding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Convolution/ConvTranspose1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Convolution/ConvTranspose1d.cs new file mode 100644 index 00000000..0aee1947 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Convolution/ConvTranspose1d.cs @@ -0,0 +1,112 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Convolution; + +/// +/// Represents an operator that creates a 1D transposed convolution module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D transposed convolution module.")] +public class ConvTranspose1d +{ + /// + /// The number of input channels in the input tensor. + /// + [Description("The number of input channels in the input tensor.")] + public long InChannels { get; set; } + + /// + /// The number of output channels produced by the convolution. + /// + [Description("The number of output channels produced by the convolution.")] + public long OutChannels { get; set; } + + /// + /// The size of the convolution kernel. + /// + [Description("The size of the convolution kernel.")] + public long KernelSize { get; set; } + + /// + /// The stride of the convolution. + /// + [Description("The stride of the convolution.")] + public long Stride { get; set; } = 1; + + /// + /// The padding to add to both sides of the input. + /// + [Description("The padding to add to both sides of the input.")] + public long Padding { get; set; } = 0; + + /// + /// The additional size added to one side of the output shape. + /// + [Description("The additional size added to one side of the output shape.")] + public long OutputPadding { get; set; } = 0; + + /// + /// The spacing between kernel elements. + /// + [Description("The spacing between kernel elements.")] + public long Dilation { get; set; } = 1; + + /// + /// The mode of padding. + /// + [Description("The mode of padding.")] + public PaddingModes PaddingMode { get; set; } = PaddingModes.Zeros; + + /// + /// The number of blocked connections from input channels to output channels. + /// + [Description("The number of blocked connections from input channels to output channels.")] + public long Groups { get; set; } = 1; + + /// + /// If true, adds a learnable bias to the output. + /// + [Description("If true, adds a learnable bias to the output")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a ConvTranspose1d module. + /// + /// + public IObservable Process() + { + return Observable.Return(ConvTranspose1d(InChannels, OutChannels, KernelSize, Stride, Padding, OutputPadding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } + + /// + /// Creates a ConvTranspose1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ConvTranspose1d(InChannels, OutChannels, KernelSize, Stride, Padding, OutputPadding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Convolution/ConvTranspose2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Convolution/ConvTranspose2d.cs new file mode 100644 index 00000000..6307420a --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Convolution/ConvTranspose2d.cs @@ -0,0 +1,117 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Convolution; + +/// +/// Represents an operator that creates a 2D transposed convolution module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D transposed convolution module.")] +public class ConvTranspose2d +{ + /// + /// The number of input channels in the input tensor. + /// + [Description("The number of input channels in the input tensor.")] + public long InChannels { get; set; } + + /// + /// The number of output channels produced by the convolution. + /// + [Description("The number of output channels produced by the convolution.")] + public long OutChannels { get; set; } + + /// + /// The size of the convolution kernel. + /// + [Description("The size of the convolution kernel.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) KernelSize { get; set; } + + /// + /// The stride of the convolution. + /// + [Description("The stride of the convolution.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Stride { get; set; } = null; + + /// + /// The zero-padding added to both sides of each dimension in the input. + /// + [Description("Zero-padding added to both sides of each dimension in the input.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Padding { get; set; } = null; + + /// + /// The additional size added to one side of each dimension in the output shape. + /// + [Description("The additional size added to one side of each dimension in the output shape.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? OutputPadding { get; set; } = null; + + /// + /// The spacing between kernel elements. + /// + [Description("The dilation parameter for the Conv1d module")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Dilation { get; set; } = null; + + /// + /// The mode of padding. + /// + [Description("The mode of padding.")] + public PaddingModes PaddingMode { get; set; } = PaddingModes.Zeros; + + /// + /// The number of blocked connections from input channels to output channels. + /// + [Description("The number of blocked connections from input channels to output channels.")] + public long Groups { get; set; } = 1; + + /// + /// If true, adds a learnable bias to the output. + /// + [Description("If true, adds a learnable bias to the output")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a ConvTranspose2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(ConvTranspose2d(InChannels, OutChannels, KernelSize, Stride, Padding, OutputPadding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } + + /// + /// Creates a ConvTranspose2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ConvTranspose2d(InChannels, OutChannels, KernelSize, Stride, Padding, OutputPadding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Convolution/ConvTranspose3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Convolution/ConvTranspose3d.cs new file mode 100644 index 00000000..c8388a76 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Convolution/ConvTranspose3d.cs @@ -0,0 +1,117 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Convolution; + +/// +/// Represents an operator that creates a 3D transposed convolution module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D transposed convolution module.")] +public class ConvTranspose3d +{ + /// + /// The number of input channels in the input tensor. + /// + [Description("The number of input channels in the input tensor.")] + public long InChannels { get; set; } + + /// + /// The number of output channels produced by the convolution. + /// + [Description("The number of output channels produced by the convolution.")] + public long OutChannels { get; set; } + + /// + /// The size of the convolution kernel. + /// + [Description("The size of the convolution kernel.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long) KernelSize { get; set; } + + /// + /// The stride of the convolution. + /// + [Description("The stride of the convolution.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Stride { get; set; } = null; + + /// + /// The zero-padding added to both sides of each dimension in the input. + /// + [Description("The zero-padding added to both sides of each dimension in the input.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Padding { get; set; } = null; + + /// + /// The additional size added to one side of each dimension in the output shape. + /// + [Description("The additional size added to one side of each dimension in the output shape.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? OutputPadding { get; set; } = null; + + /// + /// The spacing between kernel elements. + /// + [Description("The spacing between kernel elements.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Dilation { get; set; } = null; + + /// + /// The mode of padding. + /// + [Description("The mode of padding.")] + public PaddingModes PaddingMode { get; set; } = PaddingModes.Zeros; + + /// + /// The number of blocked connections from input channels to output channels. + /// + [Description("The number of blocked connections from input channels to output channels.")] + public long Groups { get; set; } = 1; + + /// + /// If true, adds a learnable bias to the output. + /// + [Description("If true, adds a learnable bias to the output")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a ConvTranspose3d module. + /// + /// + public IObservable Process() + { + return Observable.Return(ConvTranspose3d(InChannels, OutChannels, KernelSize, Stride, Padding, OutputPadding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } + + /// + /// Creates a ConvTranspose3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ConvTranspose3d(InChannels, OutChannels, KernelSize, Stride, Padding, OutputPadding, Dilation, PaddingMode, Groups, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Convolution/Fold.cs b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Fold.cs new file mode 100644 index 00000000..113c8ccd --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Fold.cs @@ -0,0 +1,72 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Convolution; + +/// +/// Represents an operator that creates a fold module. +/// +/// +/// See for more information. +/// +[Description("Creates a fold module.")] +public class Fold +{ + /// + /// The shape of the spatial dimensions of the output tensor. + /// + [Description("The shape of the spatial dimensions of the output tensor.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) OutputSize { get; set; } + + /// + /// The size of the sliding blocks. + /// + [Description("The size of the sliding blocks.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) KernelSize { get; set; } + + /// + /// The stride of elements within the neighborhood. + /// + [Description("The stride of elements within the neighborhood.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Dilation { get; set; } = null; + + /// + /// The implicit zero-padding to be added on both sides of input. + /// + [Description("The implicit zero-padding to be added on both sides of input.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Padding { get; set; } = null; + + /// + /// The stride of the sliding blocks in the input tensor. + /// + [Description("The stride of the sliding blocks in the input tensor.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Stride { get; set; } = null; + + /// + /// Creates a Fold module. + /// + /// + public IObservable Process() + { + return Observable.Return(Fold(OutputSize, KernelSize, Dilation, Padding, Stride)); + } + + /// + /// Creates a Fold module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Fold(OutputSize, KernelSize, Dilation, Padding, Stride)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Convolution/Unfold.cs b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Unfold.cs new file mode 100644 index 00000000..d32aa810 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Convolution/Unfold.cs @@ -0,0 +1,65 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Convolution; + +/// +/// Represents an operator that creates an unfold module. +/// +/// +/// See for more information. +/// +[Description("Creates an unfold module.")] +public class Unfold +{ + /// + /// The size of the sliding blocks. + /// + [Description("The size of the sliding blocks.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) KernelSize { get; set; } + + /// + /// The stride of elements within the neighborhood. + /// + [Description("The stride of elements within the neighborhood.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Dilation { get; set; } = null; + + /// + /// The implicit zero-padding to be added on both sides of input. + /// + [Description("The implicit zero-padding to be added on both sides of input.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Padding { get; set; } = null; + + /// + /// The stride of the sliding blocks in the input tensor. + /// + [Description("The stride of the sliding blocks in the input tensor.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Stride { get; set; } = null; + + /// + /// Creates an Unfold module. + /// + /// + public IObservable Process() + { + return Observable.Return(Unfold(KernelSize, Dilation, Padding, Stride)); + } + + /// + /// Creates an Unfold module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Unfold(KernelSize, Dilation, Padding, Stride)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ConvolutionModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/ConvolutionModuleBuilder.cs new file mode 100644 index 00000000..ec9e4ae5 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ConvolutionModuleBuilder.cs @@ -0,0 +1,48 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a torch module for convolution operations. +/// +[XmlInclude(typeof(Convolution.Conv1d))] +[XmlInclude(typeof(Convolution.Conv2d))] +[XmlInclude(typeof(Convolution.Conv3d))] +[XmlInclude(typeof(Convolution.ConvTranspose1d))] +[XmlInclude(typeof(Convolution.ConvTranspose2d))] +[XmlInclude(typeof(Convolution.ConvTranspose3d))] +[XmlInclude(typeof(Convolution.Fold))] +[XmlInclude(typeof(Convolution.Unfold))] +[DefaultProperty(nameof(ConvolutionModule))] +[Combinator] +[Description("Creates a torch module for convolution operations.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class ConvolutionModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "ConvolutionModule"; + + /// + /// Initializes a new instance of the class. + /// + public ConvolutionModuleBuilder() + { + Module = new Convolution.Conv1d(); + } + + /// + /// Gets or sets the specific convolution module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific convolution module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object ConvolutionModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Distance/CosineSimilarity.cs b/src/Bonsai.ML.Torch/NeuralNets/Distance/CosineSimilarity.cs new file mode 100644 index 00000000..4e1de349 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Distance/CosineSimilarity.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Distance; + +/// +/// Represents an operator that creates a cosine similarity module. +/// +/// +/// See for more information. +/// +[Description("Creates a cosine similarity module.")] +public class CosineSimilarity +{ + /// + /// The dimension where cosine similarity is computed. + /// + [Description("The dimension where cosine similarity is computed.")] + public long Dim { get; set; } = 1; + + /// + /// The value added to the denominator to avoid division by zero. + /// + [Description("The value added to the denominator to avoid division by zero.")] + public double Eps { get; set; } = 1E-08D; + + /// + /// Creates a CosineSimilarity module. + /// + /// + public IObservable Process() + { + return Observable.Return(CosineSimilarity(Dim, Eps)); + } + + /// + /// Creates a CosineSimilarity module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => CosineSimilarity(Dim, Eps)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Distance/PairwiseDistance.cs b/src/Bonsai.ML.Torch/NeuralNets/Distance/PairwiseDistance.cs new file mode 100644 index 00000000..b269e2e2 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Distance/PairwiseDistance.cs @@ -0,0 +1,55 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Distance; + +/// +/// Represents an operator that creates a pairwise distance module. +/// +/// +/// See for more information. +/// +[Description("Creates a pairwise distance module.")] +public class PairwiseDistance +{ + /// + /// The norm degree which can be positive or negative. + /// + [Description("The norm degree which can be positive or negative.")] + public double P { get; set; } = 2D; + + /// + /// The value added to the denominator to avoid division by zero. + /// + [Description("The value added to the denominator to avoid division by zero.")] + public double Eps { get; set; } = 1E-06D; + + /// + /// Determines whether or not to keep the vector dimension. + /// + [Description("Determines whether or not to keep the vector dimension.")] + public bool KeepDim { get; set; } = false; + + /// + /// Creates a PairwiseDistance module. + /// + /// + public IObservable Process() + { + return Observable.Return(PairwiseDistance(P, Eps, KeepDim)); + } + + /// + /// Creates a PairwiseDistance module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => PairwiseDistance(P, Eps, KeepDim)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/DistanceModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/DistanceModuleBuilder.cs new file mode 100644 index 00000000..aae8f651 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/DistanceModuleBuilder.cs @@ -0,0 +1,42 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a module for distance computations. +/// +[XmlInclude(typeof(Distance.CosineSimilarity))] +[XmlInclude(typeof(Distance.PairwiseDistance))] +[DefaultProperty(nameof(DistanceModule))] +[Combinator] +[Description("Creates a module for distance computations.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class DistanceModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "DistanceModule"; + + /// + /// Initializes a new instance of the class. + /// + public DistanceModuleBuilder() + { + Module = new Distance.CosineSimilarity(); + } + + /// + /// Gets or sets the specific distance module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific distance module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object DistanceModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Dropout/AlphaDropout.cs b/src/Bonsai.ML.Torch/NeuralNets/Dropout/AlphaDropout.cs new file mode 100644 index 00000000..85b52e95 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Dropout/AlphaDropout.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Dropout; + +/// +/// Represents an operator that creates an alpha dropout module. +/// +/// +/// See for more information. +/// +[Description("Creates an alpha dropout module.")] +public class AlphaDropout +{ + /// + /// The probability of an element to be dropped. + /// + [Description("The probability of an element to be dropped.")] + public double Probability { get; set; } = 0.5D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place")] + public bool Inplace { get; set; } = false; + + /// + /// Creates an AlphaDropout module. + /// + /// + public IObservable Process() + { + return Observable.Return(AlphaDropout(Probability, Inplace)); + } + + /// + /// Creates an AlphaDropout module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AlphaDropout(Probability, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout.cs b/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout.cs new file mode 100644 index 00000000..cdda8cda --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Dropout; + +/// +/// Represents an operator that creates a dropout module. +/// +/// +/// See for more information. +/// +[Description("Creates a dropout module.")] +public class Dropout +{ + /// + /// The probability of an element to be zeroed. + /// + [Description("The probability of an element to be zeroed.")] + public double Probability { get; set; } = 0.5D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a Dropout module. + /// + /// + public IObservable Process() + { + return Observable.Return(nn.Dropout(Probability, Inplace)); + } + + /// + /// Creates a Dropout module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => nn.Dropout(Probability, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout1d.cs new file mode 100644 index 00000000..899affb6 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout1d.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Dropout; + +/// +/// Represents an operator that creates a 1D dropout module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D dropout module.")] +public class Dropout1d +{ + /// + /// The probability of an element to be zeroed. + /// + [Description("The probability of an element to be zeroed.")] + public double Probability { get; set; } = 0.5D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a Dropout1d module. + /// + /// + public IObservable Process() + { + return Observable.Return(nn.Dropout1d(Probability, Inplace)); + } + + /// + /// Creates a Dropout1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => nn.Dropout1d(Probability, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout2d.cs new file mode 100644 index 00000000..4618e315 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout2d.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Dropout; + +/// +/// Represents an operator that creates a 2D dropout module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D dropout module.")] +public class Dropout2d +{ + /// + /// The probability of an element to be zeroed. + /// + [Description("The probability of an element to be zeroed.")] + public double Probability { get; set; } = 0.5D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a Dropout2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(nn.Dropout2d(Probability, Inplace)); + } + + /// + /// Creates a Dropout2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => nn.Dropout2d(Probability, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout3d.cs new file mode 100644 index 00000000..627a240c --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Dropout/Dropout3d.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Dropout; + +/// +/// Represents an operator that creates a 3D dropout module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D dropout module.")] +public class Dropout3d +{ + /// + /// The probability of an element to be zeroed. + /// + [Description("The probability of an element to be zeroed.")] + public double Probability { get; set; } = 0.5D; + + /// + /// If set to true, will do this operation in-place. + /// + [Description("If set to true, will do this operation in-place.")] + public bool Inplace { get; set; } = false; + + /// + /// Creates a Dropout3d module. + /// + /// + public IObservable Process() + { + return Observable.Return(nn.Dropout3d(Probability, Inplace)); + } + + /// + /// Creates a Dropout3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => nn.Dropout3d(Probability, Inplace)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Dropout/FeatureAlphaDropout.cs b/src/Bonsai.ML.Torch/NeuralNets/Dropout/FeatureAlphaDropout.cs new file mode 100644 index 00000000..dd827a83 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Dropout/FeatureAlphaDropout.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Dropout; + +/// +/// Represents an operator that creates a feature alpha dropout module. +/// +/// +/// See for more information. +/// +[Description("Creates a feature alpha dropout module.")] +public class FeatureAlphaDropout +{ + /// + /// The probability of an element to be zeroed. + /// + [Description("The probability of an element to be zeroed.")] + public double Probability { get; set; } = 0.5D; + + /// + /// Creates a FeatureAlphaDropout module. + /// + /// + public IObservable Process() + { + return Observable.Return(FeatureAlphaDropout(Probability)); + } + + /// + /// Creates a FeatureAlphaDropout module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => FeatureAlphaDropout(Probability)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/DropoutModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/DropoutModuleBuilder.cs new file mode 100644 index 00000000..b111af08 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/DropoutModuleBuilder.cs @@ -0,0 +1,46 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a module for dropout computations. +/// +[XmlInclude(typeof(Dropout.AlphaDropout))] +[XmlInclude(typeof(Dropout.Dropout))] +[XmlInclude(typeof(Dropout.Dropout1d))] +[XmlInclude(typeof(Dropout.Dropout2d))] +[XmlInclude(typeof(Dropout.Dropout3d))] +[XmlInclude(typeof(Dropout.FeatureAlphaDropout))] +[DefaultProperty(nameof(DropoutModule))] +[Combinator] +[Description("Creates a module for dropout computations.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class DropoutModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "DropoutModule"; + + /// + /// Initializes a new instance of the class. + /// + public DropoutModuleBuilder() + { + Module = new Dropout.AlphaDropout(); + } + + /// + /// Gets or sets the specific dropout module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific dropout module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object DropoutModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Flatten/Flatten.cs b/src/Bonsai.ML.Torch/NeuralNets/Flatten/Flatten.cs new file mode 100644 index 00000000..6f343414 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Flatten/Flatten.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Flatten; + +/// +/// Represents an operator that creates a Flatten module. +/// +/// +/// See for more information. +/// +[Description("Creates a Flatten module.")] +public class Flatten +{ + /// + /// The first dimension to flatten. + /// + [Description("The first dimension to flatten.")] + public long StartDim { get; set; } = 1; + + /// + /// The last dimension to flatten. + /// + [Description("The last dimension to flatten.")] + public long EndDim { get; set; } = -1; + + /// + /// Creates a Flatten module. + /// + /// + public IObservable Process() + { + return Observable.Return(nn.Flatten(StartDim, EndDim)); + } + + /// + /// Creates a Flatten module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => nn.Flatten(StartDim, EndDim)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Flatten/Unflatten.cs b/src/Bonsai.ML.Torch/NeuralNets/Flatten/Unflatten.cs new file mode 100644 index 00000000..b00683de --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Flatten/Unflatten.cs @@ -0,0 +1,50 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Flatten; + +/// +/// Creates an Unflatten module. +/// +/// +/// See for more information. +/// +[Description("Creates an Unflatten module.")] +public class Unflatten +{ + /// + /// The dimension to unflatten. + /// + [Description("The dimension to unflatten.")] + public long Dim { get; set; } + + /// + /// The new shape of the unflattened dimension. + /// + [Description("The new shape of the unflattened dimension.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public long[] UnflattenedSize { get; set; } + + /// + /// Creates an Unflatten module. + /// + /// + public IObservable Process() + { + return Observable.Return(Unflatten(Dim, UnflattenedSize)); + } + + /// + /// Creates an Unflatten module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Unflatten(Dim, UnflattenedSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/FlattenModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/FlattenModuleBuilder.cs new file mode 100644 index 00000000..d5ef301c --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/FlattenModuleBuilder.cs @@ -0,0 +1,42 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a module for flattening tensors. +/// +[XmlInclude(typeof(Flatten.Flatten))] +[XmlInclude(typeof(Flatten.Unflatten))] +[DefaultProperty(nameof(FlattenModule))] +[Combinator] +[Description("Creates a module for flattening tensors.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class FlattenModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "FlattenModule"; + + /// + /// Initializes a new instance of the class. + /// + public FlattenModuleBuilder() + { + Module = new Flatten.Flatten(); + } + + /// + /// Gets or sets the specific flatten module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific flatten module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object FlattenModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Forward.cs b/src/Bonsai.ML.Torch/NeuralNets/Forward.cs deleted file mode 100644 index fdab2387..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/Forward.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System; -using System.ComponentModel; -using System.Reactive.Linq; -using static TorchSharp.torch; -using System.Xml.Serialization; - -namespace Bonsai.ML.Torch.NeuralNets -{ - /// - /// Runs forward inference on the input tensor using the specified model. - /// - [Combinator] - [ResetCombinator] - [Description("Runs forward inference on the input tensor using the specified model.")] - [WorkflowElementCategory(ElementCategory.Transform)] - public class Forward - { - /// - /// The model to use for inference. - /// - [XmlIgnore] - public ITorchModule Model { get; set; } - - /// - /// Runs forward inference on the input tensor using the specified model. - /// - /// - /// - public IObservable Process(IObservable source) - { - Model.Module.eval(); - return source.Select(Model.Forward); - } - } -} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/ForwardBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/ForwardBuilder.cs new file mode 100644 index 00000000..79074611 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ForwardBuilder.cs @@ -0,0 +1,122 @@ +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Collections.Generic; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Reflection; +using static TorchSharp.torch; +using static TorchSharp.torch.jit; +using Bonsai.Expressions; +using TorchSharp; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that runs forward inference on the input using the specified module. +/// +[Combinator] +[Description("Runs forward inference on the input using the specified module.")] +[WorkflowElementCategory(ElementCategory.Transform)] +public class ForwardBuilder : SingleArgumentExpressionBuilder +{ + /// + public override Expression Build(IEnumerable arguments) + { + var argumentList = arguments.First().Type.GetGenericArguments()[0]; + var inputArgs = argumentList.GetGenericArguments()[0]; + var moduleArg = argumentList.GetGenericArguments()[1]; + + var sourceType = typeof(Tuple<,>).MakeGenericType(inputArgs, moduleArg); + + var selectMethod = typeof(Observable) + .GetMethods() + .First(m => + m.Name == nameof(Observable.Select) && + m.IsGenericMethodDefinition && + m.GetParameters().Length == 2); + + System.Type? moduleType = null; + Expression? selectExpression = null; + System.Type? resultType = null; + + for (var t = moduleArg; t != null && t != typeof(object); t = t.BaseType) + { + if (t.IsGenericType && t.GetGenericTypeDefinition().Name.StartsWith("Module`")) + { + moduleType = t; + + resultType = typeof(Tuple<,>).MakeGenericType(inputArgs, moduleType); + + var tuple1 = Expression.Parameter(sourceType, "t"); + var tuple1Item1 = Expression.Property(tuple1, "Item1"); + var tuple1Item2 = Expression.Property(tuple1, "Item2"); + + var moduleConversionExpression = Expression.Convert(tuple1Item2, moduleType); + var tupleCreateMethod = typeof(Tuple) + .GetMethods() + .Single(m => + m.Name == nameof(Tuple.Create) && + m.IsGenericMethodDefinition && + m.GetGenericArguments().Length == 2 && + m.GetParameters().Length == 2) + .MakeGenericMethod(inputArgs, moduleType); + + var newTuple = Expression.Call(tupleCreateMethod, tuple1Item1, moduleConversionExpression); + + var selector = Expression.Lambda(newTuple, tuple1); + + selectExpression = Expression.Call(selectMethod.MakeGenericMethod(sourceType, resultType), arguments.First(), selector); + break; + } + + else if (t.IsGenericType && t.GetGenericTypeDefinition().Name.StartsWith("ScriptModule`")) + { + moduleType = t; + resultType = sourceType; + selectExpression = arguments.First(); + break; + } + } + + if (moduleType is null) + throw new InvalidOperationException("The specified module type is not a valid TorchSharp module."); + + var tuple = Expression.Parameter(resultType, "t"); + var item1 = Expression.Property(tuple, "Item1"); + var item2 = Expression.Property(tuple, "Item2"); + + List forwardCallArgs = []; + + if (inputArgs.IsGenericType && inputArgs.GetGenericTypeDefinition().Name.StartsWith("Tuple`")) + { + var inputArgsTypes = inputArgs.GetGenericArguments(); + for (int i = 0; i < inputArgsTypes.Length; i++) + { + var itemN = Expression.Property(item1, $"Item{i + 1}"); + forwardCallArgs.Add(itemN); + } + } + else + { + forwardCallArgs.Add(item1); + } + + var moduleMethods = moduleType + .GetMethods(BindingFlags.Instance | BindingFlags.Public) + .Where(m => m.Name == "forward" && + m.IsPublic && + m.GetParameters().Length == forwardCallArgs.Count && + m.GetParameters().Select(p => p.ParameterType).SequenceEqual(forwardCallArgs.Select(a => a.Type))); + + if (!moduleMethods.Any()) + throw new InvalidOperationException("The module does not contain a matching forward method."); + + var forwardMethod = moduleMethods.First(); + + var forwardCall = Expression.Call(item2, forwardMethod, forwardCallArgs); + var forwardLambda = Expression.Lambda(forwardCall, tuple); + + return Expression.Call(selectMethod.MakeGenericMethod(resultType, forwardMethod.ReturnType), selectExpression, forwardLambda); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ITorchModule.cs b/src/Bonsai.ML.Torch/NeuralNets/ITorchModule.cs deleted file mode 100644 index 5cde6f73..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/ITorchModule.cs +++ /dev/null @@ -1,22 +0,0 @@ -using static TorchSharp.torch; - -namespace Bonsai.ML.Torch.NeuralNets -{ - /// - /// Represents an interface for a Torch module. - /// - public interface ITorchModule - { - /// - /// The module. - /// - public nn.Module Module { get; } - - /// - /// Runs forward inference on the input tensor using the specified model. - /// - /// - /// - public Tensor Forward(Tensor tensor); - } -} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Constant.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Constant.cs new file mode 100644 index 00000000..84fae13b --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Constant.cs @@ -0,0 +1,52 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets.LearningRateScheduler; + +/// +/// Represents an operator that creates a constant learning rate scheduler. +/// +/// +/// See for more information. +/// +[Description("Creates a constant learning rate scheduler.")] +public class Constant +{ + /// + /// The number that the learning rate will be multiplied by until the milestone. + /// + [Description("The number that the learning rate will be multiplied by until the milestone.")] + public double Factor { get; set; } = 0.3333333333333333D; + + /// + /// The number of steps that the scheduler multiplies the learning rate by the factor. + /// + [Description("The number of steps that the scheduler multiplies the learning rate by the factor.")] + public int TotalIters { get; set; } = 5; + + /// + /// The index of the last epoch. + /// + [Description("The index of the last epoch.")] + public int LastEpoch { get; set; } = -1; + + /// + /// Determines whether to write a message to stdout for each update. + /// + [Description("Determines whether to write a message to stdout for each update.")] + public bool Verbose { get; set; } = false; + + /// + /// Creates a ConstantLR scheduler for the input optimizer. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Select(optimizer => ConstantLR(optimizer, Factor, TotalIters, LastEpoch, Verbose)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/CosineAnnealing.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/CosineAnnealing.cs new file mode 100644 index 00000000..37457bc5 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/CosineAnnealing.cs @@ -0,0 +1,52 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets.LearningRateScheduler; + +/// +/// Represents an operator that creates a cosine annealing learning rate scheduler. +/// +/// +/// See for more information. +/// +[Description("Creates a cosine annealing learning rate scheduler.")] +public class CosineAnnealing +{ + /// + /// The maximum number of iterations. + /// + [Description("The maximum number of iterations.")] + public double TMax { get; set; } + + /// + /// The minimum learning rate. + /// + [Description("The minimum learning rate.")] + public double EtaMin { get; set; } = 0D; + + /// + /// The index of the last epoch. + /// + [Description("The index of the last epoch.")] + public int LastEpoch { get; set; } = -1; + + /// + /// Determines whether to write a message to stdout for each update. + /// + [Description("Determines whether to write a message to stdout for each update.")] + public bool Verbose { get; set; } = false; + + /// + /// Creates a CosineAnnealingLR scheduler. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Select(optimizer => CosineAnnealingLR(optimizer, TMax, EtaMin, LastEpoch, Verbose)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Exponential.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Exponential.cs new file mode 100644 index 00000000..817f8f93 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Exponential.cs @@ -0,0 +1,46 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets.LearningRateScheduler; + +/// +/// Represents an operator that creates an exponential learning rate scheduler. +/// +/// +/// See for more information. +/// +[Description("Creates an exponential learning rate scheduler.")] +public class Exponential +{ + /// + /// The multiplicative factor of learning rate decay. + /// + [Description("The multiplicative factor of learning rate decay.")] + public double Gamma { get; set; } = 0.1D; + + /// + /// The index of the last epoch. + /// + [Description("The index of the last epoch.")] + public int LastEpoch { get; set; } = -1; + + /// + /// Determines whether to write a message to stdout for each update. + /// + [Description("Determines whether to write a message to stdout for each update.")] + public bool Verbose { get; set; } = false; + + /// + /// Creates an ExponentialLR scheduler. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Select(optimizer => ExponentialLR(optimizer, Gamma, LastEpoch, Verbose)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Linear.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Linear.cs new file mode 100644 index 00000000..14901212 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Linear.cs @@ -0,0 +1,58 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets.LearningRateScheduler; + +/// +/// Represents an operator that creates a linear learning rate scheduler. +/// +/// +/// See for more information. +/// +[Description("Creates a linear learning rate scheduler.")] +public class Linear +{ + /// + /// The number used to multiply the learning rate in the first epoch. + /// + [Description("The number used to multiply the learning rate in the first epoch.")] + public double StartFactor { get; set; } = 0.3333333333333333D; + + /// + /// The number used to multiply the learning rate in the last epoch. + /// + [Description("The number used to multiply the learning rate in the last epoch.")] + public double EndFactor { get; set; } = 5D; + + /// + /// The number of iterations over which the learning rate is adjusted. + /// + [Description("The number of iterations over which the learning rate is adjusted.")] + public int TotalIters { get; set; } = 5; + + /// + /// The index of the last epoch. + /// + [Description("The index of the last epoch.")] + public int LastEpoch { get; set; } = -1; + + /// + /// Determines whether to write a message to stdout for each update. + /// + [Description("Determines whether to write a message to stdout for each update.")] + public bool Verbose { get; set; } = false; + + /// + /// Creates a LinearLR scheduler. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Select(optimizer => LinearLR(optimizer, StartFactor, EndFactor, TotalIters, LastEpoch, Verbose)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/MultiStep.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/MultiStep.cs new file mode 100644 index 00000000..eb487bc9 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/MultiStep.cs @@ -0,0 +1,54 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets.LearningRateScheduler; + +/// +/// Represents an operator that creates a multi-step learning rate scheduler. +/// +/// +/// See for more information. +/// +[Description("Creates a multi-step learning rate scheduler.")] +public class MultiStep +{ + /// + /// The list of epoch indices. + /// + [Description("The list of epoch indices.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public int[] Milestones { get; set; } + + /// + /// The multiplicative factor of learning rate decay. + /// + [Description("The multiplicative factor of learning rate decay.")] + public double Gamma { get; set; } = 0.1D; + + /// + /// The index of the last epoch. + /// + [Description("The index of the last epoch.")] + public int LastEpoch { get; set; } = -1; + + /// + /// Determines whether to write a message to stdout for each update. + /// + [Description("Determines whether to write a message to stdout for each update.")] + public bool Verbose { get; set; } = false; + + /// + /// Creates a MultiStepLR scheduler. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Select(optimizer => MultiStepLR(optimizer, Milestones, Gamma, LastEpoch, Verbose)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/OneCycle.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/OneCycle.cs new file mode 100644 index 00000000..4200d44d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/OneCycle.cs @@ -0,0 +1,115 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets.LearningRateScheduler; + +/// +/// Represents an operator that creates a one cycle learning rate scheduler. +/// +/// +/// See for more information. +/// +[Description("Creates a one cycle learning rate scheduler.")] +public class OneCycle +{ + /// + /// The upper learning rate boundaries in the cycle for each parameter group. + /// + [Description("The upper learning rate boundaries in the cycle for each parameter group.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public double[] MaxLearningRate { get; set; } + + /// + /// The total number of steps in the cycle. + /// + [Description("The total number of steps in the cycle.")] + public int TotalSteps { get; set; } = -1; + + /// + /// The number of epochs to train for. + /// + [Description("The number of epochs to train for.")] + public int Epochs { get; set; } = -1; + + /// + /// The number of steps per epoch. + /// + [Description("The number of steps per epoch.")] + public int StepsPerEpoch { get; set; } = -1; + + /// + /// The percentage of the cycle spent increasing the learning rate. + /// + [Description("The percentage of the cycle spent increasing the learning rate.")] + public double PercentageStart { get; set; } = 0.3D; + + /// + /// The annealing strategy to use. + /// + [Description("The annealing strategy to use.")] + public impl.OneCycleLR.AnnealStrategy AnnealStrategy { get; set; } = impl.OneCycleLR.AnnealStrategy.Cos; + + /// + /// If true, momentum is cycled inversely to learning rate. + /// + [Description("If true, momentum is cycled inversely to learning rate.")] + public bool CycleMomentum { get; set; } = true; + + /// + /// The lower momentum boundaries in the cycle for each parameter group + /// + [Description("The lower momentum boundaries in the cycle for each parameter group")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public double[] BaseMomentum { get; set; } = null; + + /// + /// The upper momentum boundaries in the cycle for each parameter group + /// + [Description("The upper momentum boundaries in the cycle for each parameter group")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public double[] MaxMomentum { get; set; } = null; + + /// + /// Determines the initial learning rate. + /// + [Description("Determines the initial learning rate.")] + public double DivFactor { get; set; } = 25D; + + /// + /// Determines the final learning rate. + /// + [Description("Determines the final learning rate.")] + public double FinalDivFactor { get; set; } = 10000D; + + /// + /// If true, uses a third phase of the schedule to annihilate the learning rate. + /// + [Description("If true, uses a third phase of the schedule to annihilate the learning rate.")] + public bool ThreePhase { get; set; } = false; + + /// + /// The index of the last epoch. + /// + [Description("The index of the last epoch.")] + public int LastEpoch { get; set; } = -1; + + /// + /// Determines whether to write a message to stdout for each update. + /// + [Description("Determines whether to write a message to stdout for each update.")] + public bool Verbose { get; set; } = false; + + /// + /// Creates a OneCycleLR scheduler. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Select(optimizer => OneCycleLR(optimizer, MaxLearningRate, TotalSteps, Epochs, StepsPerEpoch, PercentageStart, AnnealStrategy, CycleMomentum, BaseMomentum, MaxMomentum, DivFactor, FinalDivFactor, ThreePhase, LastEpoch, Verbose)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Polynomial.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Polynomial.cs new file mode 100644 index 00000000..854dc6fe --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Polynomial.cs @@ -0,0 +1,52 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets.LearningRateScheduler; + +/// +/// Represents an operator that creates a polynomial learning rate scheduler. +/// +/// +/// See for more information. +/// +[Description("Creates a polynomial learning rate scheduler.")] +public class Polynomial +{ + /// + /// The total number of steps that the learning rate will be adjusted over. + /// + [Description("The total number of steps that the learning rate will be adjusted over.")] + public int TotalIters { get; set; } = 5; + + /// + /// The power of the polynomial. + /// + [Description("The power of the polynomial.")] + public int Power { get; set; } = 1; + + /// + /// The index of the last epoch. + /// + [Description("The index of the last epoch.")] + public int LastEpoch { get; set; } = -1; + + /// + /// Determines whether to write a message to stdout for each update. + /// + [Description("Determines whether to write a message to stdout for each update.")] + public bool Verbose { get; set; } = false; + + /// + /// Creates a PolynomialLR scheduler. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Select(optimizer => PolynomialLR(optimizer, TotalIters, Power, LastEpoch, Verbose)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/ReduceOnPlateau.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/ReduceOnPlateau.cs new file mode 100644 index 00000000..dea93e26 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/ReduceOnPlateau.cs @@ -0,0 +1,102 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets.LearningRateScheduler; + +/// +/// Represents an operator that creates a scheduler that reduces the learning rate on plateau. +/// +/// +/// See for more information. +/// +[Description("Creates a scheduler that reduces the learning rate on plateau.")] +public class ReduceOnPlateau +{ + /// + /// The mode in which to determine when to reduce the learning rate. + /// + [Description("The mode in which to determine when to reduce the learning rate.")] + public PlateauMode Mode { get; set; } = PlateauMode.Min; + + /// + /// The factor by which the learning rate will be reduced. + /// + [Description("The factor by which the learning rate will be reduced.")] + public double Factor { get; set; } = 0.1D; + + /// + /// The number of allowed epochs with no improvement after which the learning rate will be reduced. + /// + [Description("The number of allowed epochs with no improvement after which the learning rate will be reduced.")] + public int Patience { get; set; } = 10; + + /// + /// The threshold for measuring the new optimum, to only focus on significant changes. + /// + [Description("The threshold for measuring the new optimum, to only focus on significant changes.")] + public double Threshold { get; set; } = 0.0001D; + + /// + /// The mode for the threshold parameter. + /// + [Description("The mode for the threshold parameter.")] + public PlateauThresholdMode ThresholdMode { get; set; } = PlateauThresholdMode.Relative; + + /// + /// The number of epochs to wait before resuming normal operation after the learning rate has been reduced. + /// + [Description("The number of epochs to wait before resuming normal operation after the learning rate has been reduced.")] + public int Cooldown { get; set; } = 0; + + /// + /// The lower bound on the learning rate of all param groups or each group respectively. + /// + [Description("The lower bound on the learning rate of all param groups or each group respectively.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public double[] MinLearningRate { get; set; } = null; + + /// + /// The minimal decay applied to the learning rate. + /// + [Description("The minimal decay applied to the learning rate")] + public double Eps { get; set; } = 1E-08D; + + /// + /// Determines whether to write a message to stdout for each update. + /// + [Description("Determines whether to write a message to stdout for each update.")] + public bool Verbose { get; set; } = false; + +#pragma warning disable // Ignore missing xml doc strings + public enum PlateauMode + { + Min, + Max + } + + public enum PlateauThresholdMode + { + Relative, + Absolute + } + + private string ModeString => Mode == PlateauMode.Min ? "min" : "max"; + + private string ThresholdModeString => ThresholdMode == PlateauThresholdMode.Relative ? "rel" : "abs"; + +#pragma warning restore + + /// + /// Creates a ReduceLROnPlateau scheduler. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Select(optimizer => ReduceLROnPlateau(optimizer, ModeString, Factor, Patience, Threshold, ThresholdModeString, Cooldown, MinLearningRate, Eps, Verbose)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Step.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Step.cs new file mode 100644 index 00000000..f9e5cf56 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateScheduler/Step.cs @@ -0,0 +1,52 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets.LearningRateScheduler; + +/// +/// Represents an operator that creates a step learning rate scheduler. +/// +/// +/// See for more information. +/// +[Description("Creates a step learning rate scheduler.")] +public class Step +{ + /// + /// The period of learning rate decay. + /// + [Description("The period of learning rate decay.")] + public int StepSize { get; set; } + + /// + /// The multiplicative factor of learning rate decay. + /// + [Description("The multiplicative factor of learning rate decay.")] + public double Gamma { get; set; } = 0.1D; + + /// + /// The index of the last epoch. + /// + [Description("The index of the last epoch.")] + public int LastEpoch { get; set; } = -1; + + /// + /// Determines whether to write a message to stdout for each update. + /// + [Description("Determines whether to write a message to stdout for each update.")] + public bool Verbose { get; set; } = false; + + /// + /// Creates a StepLR scheduler. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Select(optimizer => StepLR(optimizer, StepSize, Gamma, LastEpoch, Verbose)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LearningRateSchedulerBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/LearningRateSchedulerBuilder.cs new file mode 100644 index 00000000..95c4c489 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LearningRateSchedulerBuilder.cs @@ -0,0 +1,49 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a learning rate scheduler. +/// +[XmlInclude(typeof(LearningRateScheduler.Constant))] +[XmlInclude(typeof(LearningRateScheduler.CosineAnnealing))] +[XmlInclude(typeof(LearningRateScheduler.Exponential))] +[XmlInclude(typeof(LearningRateScheduler.Linear))] +[XmlInclude(typeof(LearningRateScheduler.MultiStep))] +[XmlInclude(typeof(LearningRateScheduler.OneCycle))] +[XmlInclude(typeof(LearningRateScheduler.Polynomial))] +[XmlInclude(typeof(LearningRateScheduler.ReduceOnPlateau))] +[XmlInclude(typeof(LearningRateScheduler.Step))] +[DefaultProperty(nameof(LearningRateScheduler))] +[Combinator] +[Description("Creates a learning rate scheduler.")] +[WorkflowElementCategory(ElementCategory.Transform)] +public class LearningRateSchedulerBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "LearningRateScheduler"; + + /// + /// Initializes a new instance of the class. + /// + public LearningRateSchedulerBuilder() + { + Module = new LearningRateScheduler.Constant(); + } + + /// + /// Gets or sets the specific learning rate scheduler to create. + /// + [DesignOnly(true)] + [DisplayName("Scheduler")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific learning rate scheduler to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object LearningRateScheduler + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Linear/Bilinear.cs b/src/Bonsai.ML.Torch/NeuralNets/Linear/Bilinear.cs new file mode 100644 index 00000000..5c4665de --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Linear/Bilinear.cs @@ -0,0 +1,75 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Linear; + +/// +/// Represents an operator that creates a Bilinear module. +/// +/// +/// See for more information. +/// +[Description("Creates a Bilinear module.")] +public class Bilinear +{ + /// + /// The size of each first input sample. + /// + [Description("The size of each first input sample.")] + public long In1Features { get; set; } + + /// + /// The size of each second input sample. + /// + [Description("The size of each second input sample.")] + public long In2Features { get; set; } + + /// + /// The size of each output sample. + /// + [Description("The size of each output sample.")] + public long OutputSize { get; set; } + + /// + /// Determines whether the layer will learn an additive bias. + /// + [Description("Determines whether the layer will learn an additive bias.")] + public bool HasBias { get; set; } = true; + + /// + /// The desired device of returned tensor. + /// + [XmlIgnore] + [Description("The desired device of returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of returned tensor. + /// + [Description("The desired data type of returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a Bilinear module. + /// + /// + public IObservable Process() + { + return Observable.Return(Bilinear(In1Features, In2Features, OutputSize, HasBias, Device, Type)); + } + + /// + /// Creates a Bilinear module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Bilinear(In1Features, In2Features, OutputSize, HasBias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Linear/Identity.cs b/src/Bonsai.ML.Torch/NeuralNets/Linear/Identity.cs new file mode 100644 index 00000000..45028095 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Linear/Identity.cs @@ -0,0 +1,37 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Linear; + +/// +/// Represents an operator that creates an Identity module. +/// +/// +/// See for more information. +/// +[Description("Creates an Identity module.")] +public class Identity +{ + /// + /// Creates an Identity module. + /// + /// + public IObservable Process() + { + return Observable.Return(nn.Identity()); + } + + /// + /// Creates an Identity module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => nn.Identity()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Linear/Linear.cs b/src/Bonsai.ML.Torch/NeuralNets/Linear/Linear.cs new file mode 100644 index 00000000..494c8c7f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Linear/Linear.cs @@ -0,0 +1,69 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Linear; + +/// +/// Represents an operator that creates a Linear module. +/// +/// +/// See for more information. +/// +[Description("Creates a Linear module.")] +public class Linear +{ + /// + /// The size of each input sample. + /// + [Description("The size of each input sample.")] + public long InputSize { get; set; } + + /// + /// The size of each output sample. + /// + [Description("The size of each output sample.")] + public long OutputSize { get; set; } + + /// + /// Determines whether the layer will learn an additive bias. + /// + [Description("Determines whether the layer will learn an additive bias.")] + public bool HasBias { get; set; } = true; + + /// + /// The desired device of returned tensor. + /// + [XmlIgnore] + [Description("The desired device of returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of returned tensor. + /// + [Description("The desired data type of returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a Linear module. + /// + /// + public IObservable Process() + { + return Observable.Return(nn.Linear(InputSize, OutputSize, HasBias, Device, Type)); + } + + /// + /// Creates a Linear module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => nn.Linear(InputSize, OutputSize, HasBias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LinearModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/LinearModuleBuilder.cs new file mode 100644 index 00000000..5d9df3a8 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LinearModuleBuilder.cs @@ -0,0 +1,43 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a linear module. +/// +[XmlInclude(typeof(Linear.Bilinear))] +[XmlInclude(typeof(Linear.Identity))] +[XmlInclude(typeof(Linear.Linear))] +[DefaultProperty(nameof(LinearModule))] +[Combinator] +[Description("Creates a linear module.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class LinearModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "LinearModule"; + + /// + /// Initializes a new instance of the class. + /// + public LinearModuleBuilder() + { + Module = new Linear.Linear(); + } + + /// + /// Gets or sets the specific linear module. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific linear module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object LinearModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LoadModuleArchitecture.cs b/src/Bonsai.ML.Torch/NeuralNets/LoadModuleArchitecture.cs new file mode 100644 index 00000000..810d2c2c --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LoadModuleArchitecture.cs @@ -0,0 +1,80 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Loads a neural network module from a specified architecture. +/// +[Combinator] +[ResetCombinator] +[Description("Loads a neural network module from a specified architecture.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class LoadModuleArchitecture +{ + /// + /// The model architecture to load. + /// + [Description("The model architecture to load.")] + public Architecture.ModelArchitecture ModelArchitecture { get; set; } + + /// + /// The device on which to load the model. + /// + [Description("The device on which to load the model.")] + [XmlIgnore] + public Device Device { get; set; } + + /// + /// The optional path to the model weights file. + /// + [Description("The optional path to the model weights file.")] + [Editor("Bonsai.Design.OpenFileNameEditor, Bonsai.Design", DesignTypes.UITypeEditor)] + public string ModelWeightsPath { get; set; } + + private int numClasses = 10; + /// + /// The number of classes in the dataset. + /// + [Description("The number of classes in the dataset.")] + public int NumClasses + { + get => numClasses; + set + { + if (value <= 0) + { + numClasses = 10; + } + else + { + numClasses = value; + } + } + } + + /// + /// Loads the neural network module from the specified architecture. + /// + /// + /// + public IObservable> Process() + { + var device = Device; + Module module = ModelArchitecture switch + { + Architecture.ModelArchitecture.AlexNet => new Architecture.AlexNet("alexnet", numClasses, device), + Architecture.ModelArchitecture.MobileNet => new Architecture.MobileNet("mobilenet", numClasses, device), + Architecture.ModelArchitecture.Mnist => new Architecture.Mnist("mnist", numClasses, device), + _ => throw new ArgumentException($"Model {ModelArchitecture} not supported.") + }; + + if (ModelWeightsPath is not null) module.load(ModelWeightsPath); + + return Observable.Return(module); + } +} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/LoadModuleFromArchitecture.cs b/src/Bonsai.ML.Torch/NeuralNets/LoadModuleFromArchitecture.cs deleted file mode 100644 index b2d35488..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/LoadModuleFromArchitecture.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System; -using System.ComponentModel; -using System.Reactive.Linq; -using static TorchSharp.torch; -using System.Xml.Serialization; - -namespace Bonsai.ML.Torch.NeuralNets -{ - /// - /// Loads a neural network module from a specified architecture. - /// - [Combinator] - [ResetCombinator] - [Description("Loads a neural network module from a specified architecture.")] - [WorkflowElementCategory(ElementCategory.Source)] - public class LoadModuleFromArchitecture - { - /// - /// The model architecture to load. - /// - [Description("The model architecture to load.")] - public Models.ModelArchitecture ModelArchitecture { get; set; } - - /// - /// The device on which to load the model. - /// - [Description("The device on which to load the model.")] - [XmlIgnore] - public Device Device { get; set; } - - /// - /// The optional path to the model weights file. - /// - [Description("The optional path to the model weights file.")] - [Editor("Bonsai.Design.OpenFileNameEditor, Bonsai.Design", DesignTypes.UITypeEditor)] - public string ModelWeightsPath { get; set; } - - private int numClasses = 10; - /// - /// The number of classes in the dataset. - /// - [Description("The number of classes in the dataset.")] - public int NumClasses - { - get => numClasses; - set - { - if (value <= 0) - { - numClasses = 10; - } - else - { - numClasses = value; - } - } - } - - /// - /// Loads the neural network module from the specified architecture. - /// - /// - /// - public IObservable Process() - { - var device = Device; - nn.Module module = ModelArchitecture switch - { - Models.ModelArchitecture.AlexNet => new Models.AlexNet("alexnet", numClasses, device), - Models.ModelArchitecture.MobileNet => new Models.MobileNet("mobilenet", numClasses, device), - Models.ModelArchitecture.Mnist => new Models.Mnist("mnist", numClasses, device), - _ => throw new ArgumentException($"Model {ModelArchitecture} not supported.") - }; - - if (ModelWeightsPath is not null) module.load(ModelWeightsPath); - - var torchModule = new TorchModuleAdapter(module); - return Observable.Return((ITorchModule)torchModule); - } - } -} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/LoadModuleStateBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/LoadModuleStateBuilder.cs new file mode 100644 index 00000000..cba653b8 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LoadModuleStateBuilder.cs @@ -0,0 +1,86 @@ +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Collections.Generic; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Reflection; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; +using static TorchSharp.torch.jit; +using Bonsai.Expressions; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that loads a module's state from a file. +/// +[Combinator] +[Description("Loads a module's state from a file.")] +[WorkflowElementCategory(ElementCategory.Sink)] +public class LoadModuleStateBuilder : SingleArgumentExpressionBuilder +{ + /// + /// The path to the module's saved state. + /// + [Description("The path to the module's saved state.")] + [Editor("Bonsai.Design.OpenFileNameEditor, Bonsai.Design", DesignTypes.UITypeEditor)] + public string ModulePath { get; set; } + + /// + public override Expression Build(IEnumerable args) + { + var inputArg = args.First().Type.GetGenericArguments()[0]; + bool isInputModule = false; + for (var t = inputArg; t != null && t != typeof(object); t = t.BaseType) + { + if (t.GetType() == typeof(nn.Module) || t.IsGenericType && t.GetGenericTypeDefinition().Name.StartsWith("Module`")) + { + isInputModule = true; + break; + } + } + + if (!isInputModule) + throw new InvalidOperationException("The LoadModuleState operator requires a Module type as input."); + + var selectMethod = typeof(Observable) + .GetMethods() + .First(m => + m.Name == nameof(Observable.Select) && + m.IsGenericMethodDefinition && + m.GetParameters().Length == 2); + + var moduleParameter = Expression.Parameter(inputArg, "m"); + + var moduleLoadMethods = typeof(nn.Module) + .GetMethods(BindingFlags.Instance | BindingFlags.Public) + .Where(m => m.Name == "load" && + m.GetParameters().Length == 4 && + m.GetParameters()[0].ParameterType == typeof(string) && + m.GetParameters()[1].ParameterType == typeof(bool) && + m.GetParameters()[2].ParameterType == typeof(IList) && + m.GetParameters()[3].ParameterType == typeof(Dictionary)); + + var moduleLoadMethod = moduleLoadMethods.First(); + + List moduleLoadArgs = + [ + Expression.Constant(ModulePath, typeof(string)), + Expression.Constant(true), + Expression.Constant(null, typeof(IList)), + Expression.Constant(null, typeof(Dictionary)) + ]; + + var loadCall = Expression.Call(moduleParameter, moduleLoadMethod, moduleLoadArgs); + + var convertLoadCall = Expression.Convert(loadCall, inputArg); + var lambda = Expression.Lambda(convertLoadCall, moduleParameter); + + var genericSelectMethod = selectMethod.MakeGenericMethod(inputArg, inputArg); + + return Expression.Call(genericSelectMethod, args.First(), lambda); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LoadScriptModule.cs b/src/Bonsai.ML.Torch/NeuralNets/LoadScriptModule.cs index 2dafa038..fe8d2614 100644 --- a/src/Bonsai.ML.Torch/NeuralNets/LoadScriptModule.cs +++ b/src/Bonsai.ML.Torch/NeuralNets/LoadScriptModule.cs @@ -1,43 +1,50 @@ using System; +using System.Linq; using System.ComponentModel; using System.Reactive.Linq; +using System.Linq.Expressions; +using System.Collections.Generic; +using TorchSharp; using static TorchSharp.torch; +using static TorchSharp.torch.nn; +using static TorchSharp.torch.jit; using System.Xml.Serialization; +using Bonsai.Expressions; -namespace Bonsai.ML.Torch.NeuralNets +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Loads a TorchScript module from the specified file path. +/// +[Combinator] +[Description("Loads a TorchScript module from the specified file path. In order to correctly infer the module type, pass into the operator objects representing the desired ScriptModule generic argument types.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class LoadScriptModule { /// - /// Loads a TorchScript module from the specified file path. + /// The device on which to load the model. /// - [Combinator] - [ResetCombinator] - [Description("Loads a TorchScript module from the specified file path.")] - [WorkflowElementCategory(ElementCategory.Source)] - public class LoadScriptModule - { - /// - /// The device on which to load the model. - /// - [Description("The device on which to load the model.")] - [XmlIgnore] - public Device Device { get; set; } + [Description("The device on which to load the model.")] + [XmlIgnore] + public Device Device { get; set; } - /// - /// The path to the TorchScript model file. - /// - [Description("The path to the TorchScript model file.")] - [Editor("Bonsai.Design.OpenFileNameEditor, Bonsai.Design", DesignTypes.UITypeEditor)] - public string ModelPath { get; set; } + /// + /// The path to the TorchScript module file. + /// + [Description("The path to the TorchScript module file.")] + [Editor("Bonsai.Design.OpenFileNameEditor, Bonsai.Design", DesignTypes.UITypeEditor)] + public string ScriptModulePath { get; set; } - /// - /// Loads the TorchScript module from the specified file path. - /// - /// - public IObservable Process() + /// + /// Loads the scripted module from the specified file path. + /// + /// + public IObservable> Process() + { + return Observable.Defer(() => { - var scriptModule = Device is null ? jit.load(ModelPath) : jit.load(ModelPath, Device); - var torchModule = new TorchModuleAdapter(scriptModule); - return Observable.Return((ITorchModule)torchModule); - } + var scriptModule = Device is null ? load(ScriptModulePath) : load(ScriptModulePath, Device); + return Observable.Return(scriptModule); + }); } -} \ No newline at end of file +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss.cs deleted file mode 100644 index 5752565e..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/Loss.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Bonsai.ML.Torch.NeuralNets -{ - /// - /// Represents a loss function that computes the loss value for a given input and target tensor. - /// - public enum Loss - { - /// - /// Computes the negative log likelihood loss. - /// - NegativeLogLikelihood, - } -} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/BinaryCrossEntropy.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/BinaryCrossEntropy.cs new file mode 100644 index 00000000..74fb378b --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/BinaryCrossEntropy.cs @@ -0,0 +1,72 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a binary cross entropy (BCE) loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a binary cross entropy (BCE) loss module.")] +[TypeConverter(typeof(TensorOperatorConverter))] +[DisplayName("BCE")] +public class BinaryCrossEntropy : IScalarTypeProvider +{ + /// + /// The manually specified rescaling weight given to the loss of each batch element. + /// + [XmlIgnore] + [Description("The manually specified rescaling weight given to the loss of each batch element.")] + [TypeConverter(typeof(TensorConverter))] + public Tensor Weight { get; set; } = null; + + /// + /// The values of the weight tensor in XML string format. + /// + [Browsable(false)] + [XmlElement(nameof(Weight))] + [EditorBrowsable(EditorBrowsableState.Never)] + public string WeightXml + { + get => TensorConverter.ConvertToString(Weight, Type); + set => Weight = TensorConverter.ConvertFromString(value, Type); + } + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// The data type of the tensor elements. + /// + [Description("The data type of the tensor elements.")] + public ScalarType Type { get; set; } = ScalarType.Float32; + + /// + /// Creates a binary cross entropy (BCE) loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(BCELoss(Weight, Reduction)); + } + + /// + /// Creates a binary cross entropy (BCE) loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => BCELoss(Weight, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/BinaryCrossEntropyWithLogits.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/BinaryCrossEntropyWithLogits.cs new file mode 100644 index 00000000..42701344 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/BinaryCrossEntropyWithLogits.cs @@ -0,0 +1,92 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a binary cross entropy with logits (BCEWithLogits) loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a binary cross entropy with logits (BCEWithLogits) loss module.")] +[TypeConverter(typeof(TensorOperatorConverter))] +[DisplayName("BCEWithLogits")] +public class BinaryCrossEntropyWithLogits : IScalarTypeProvider +{ + /// + /// The manually specified rescaling weight given to the loss of each batch element. + /// + [XmlIgnore] + [Description("The manually specified rescaling weight given to the loss of each batch element.")] + [TypeConverter(typeof(TensorConverter))] + public Tensor Weight { get; set; } = null; + + /// + /// The values of the weight tensor in XML string format. + /// + [Browsable(false)] + [XmlElement(nameof(Weight))] + [EditorBrowsable(EditorBrowsableState.Never)] + public string WeightXml + { + get => TensorConverter.ConvertToString(Weight, Type); + set => Weight = TensorConverter.ConvertFromString(value, Type); + } + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// The weight of positive examples to be broadcasted with the target. + /// + [XmlIgnore] + [Description("The weight of positive examples to be broadcasted with the target.")] + [TypeConverter(typeof(TensorConverter))] + public Tensor PosWeights { get; set; } = null; + + /// + /// The values of the pos weights tensor in XML string format. + /// + [Browsable(false)] + [XmlElement(nameof(PosWeights))] + [EditorBrowsable(EditorBrowsableState.Never)] + public string PosWeightsXml + { + get => TensorConverter.ConvertToString(PosWeights, Type); + set => PosWeights = TensorConverter.ConvertFromString(value, Type); + } + + /// + /// The data type of the tensor elements. + /// + [Description("The data type of the tensor elements.")] + public ScalarType Type { get; set; } = ScalarType.Float32; + + /// + /// Creates a binary cross entropy with logits (BCEWithLogits) loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(BCEWithLogitsLoss(Weight, Reduction, PosWeights)); + } + + /// + /// Creates a binary cross entropy with logits (BCEWithLogits) loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => BCEWithLogitsLoss(Weight, Reduction, PosWeights)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/ConnectionistTemporalClassification.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/ConnectionistTemporalClassification.cs new file mode 100644 index 00000000..188aac53 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/ConnectionistTemporalClassification.cs @@ -0,0 +1,56 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a connectionist temporal classification (CTC) loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a connectionist temporal classification (CTC) loss module.")] +[DisplayName("CTC")] +public class ConnectionistTemporalClassification +{ + /// + /// The blank label. + /// + [Description("The blank label.")] + public long Blank { get; set; } = 0; + + /// + /// Determines whether to zero infinite losses and the associated gradients. + /// + [Description("Determines whether to zero infinite losses and the associated gradients.")] + public bool ZeroInfinity { get; set; } = false; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a connectionist temporal classification (CTC) loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(CTCLoss(Blank, ZeroInfinity, Reduction)); + } + + /// + /// Creates a connectionist temporal classification (CTC) loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => CTCLoss(Blank, ZeroInfinity, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/CosineEmbedding.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/CosineEmbedding.cs new file mode 100644 index 00000000..5117b3dd --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/CosineEmbedding.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a cosine embedding loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a cosine embedding loss module.")] +public class CosineEmbedding +{ + /// + /// The margin, which is a value in the range [-1, 1]. + /// + [Description("The margin, which is a value in the range [-1, 1].")] + public double Margin { get; set; } = 0D; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a cosine embedding loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(CosineEmbeddingLoss(Margin, Reduction)); + } + + /// + /// Creates a cosine embedding loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => CosineEmbeddingLoss(Margin, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/CrossEntropy.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/CrossEntropy.cs new file mode 100644 index 00000000..e17434e7 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/CrossEntropy.cs @@ -0,0 +1,77 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a cross entropy loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a cross entropy loss module.")] +[TypeConverter(typeof(TensorOperatorConverter))] +public class CrossEntropy : IScalarTypeProvider +{ + /// + /// The manually specified rescaling weight given to each class. + /// + [XmlIgnore] + [Description("The manually specified rescaling weight given to each class.")] + [TypeConverter(typeof(TensorConverter))] + public Tensor Weight { get; set; } = null; + + /// + /// The values of the weight tensor in XML string format. + /// + [Browsable(false)] + [XmlElement(nameof(Weight))] + [EditorBrowsable(EditorBrowsableState.Never)] + public string WeightXml + { + get => TensorConverter.ConvertToString(Weight, Type); + set => Weight = TensorConverter.ConvertFromString(value, Type); + } + + /// + /// The index to ignore in the target which does not contribute to the input gradient. + /// + [Description("The index to ignore in the target which does not contribute to the input gradient.")] + public long? IgnoreIndex { get; set; } = null; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// The data type of the tensor elements. + /// + [Description("The data type of the tensor elements.")] + public ScalarType Type { get; set; } = ScalarType.Float32; + + /// + /// Creates a cross entropy loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(CrossEntropyLoss(Weight, IgnoreIndex, Reduction)); + } + + /// + /// Creates a cross entropy loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => CrossEntropyLoss(Weight, IgnoreIndex, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/GaussianNegativeLogLikelihood.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/GaussianNegativeLogLikelihood.cs new file mode 100644 index 00000000..3f8379c1 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/GaussianNegativeLogLikelihood.cs @@ -0,0 +1,56 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a gaussian negative log likelihood (GaussianNLL) loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a gaussian negative log likelihood (GaussianNLL) loss module.")] +[DisplayName("GaussianNLL")] +public class GaussianNegativeLogLikelihood +{ + /// + /// Determines whether to include the constant term in the loss. + /// + [Description("Determines whether to include the constant term in the loss.")] + public bool Full { get; set; } = false; + + /// + /// The value used to clamp the variance for numerical stability. + /// + [Description("The value used to clamp the variance for numerical stability")] + public float Eps { get; set; } = 1E-08F; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a gaussian negative log likelihood (GaussianNLL) loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(GaussianNLLLoss(Full, Eps, Reduction)); + } + + /// + /// Creates a gaussian negative log likelihood (GaussianNLL) loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => GaussianNLLLoss(Full, Eps, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/HingeEmbedding.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/HingeEmbedding.cs new file mode 100644 index 00000000..7527a947 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/HingeEmbedding.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a hinge embedding loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a hinge embedding loss module.")] +public class HingeEmbedding +{ + /// + /// The margin, a non-negative float value. + /// + [Description("The margin, a non-negative float value.")] + public double Margin { get; set; } = 1D; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a hinge embedding loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(HingeEmbeddingLoss(Margin, Reduction)); + } + + /// + /// Creates a hinge embedding loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => HingeEmbeddingLoss(Margin, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/Huber.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/Huber.cs new file mode 100644 index 00000000..2538c52f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/Huber.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a Huber loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a Huber loss module.")] +public class Huber +{ + /// + /// The threshold at which to change between delta-scaled L1 and L2 loss. + /// + [Description("The threshold at which to change between delta-scaled L1 and L2 loss.")] + public double Delta { get; set; } = 1D; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a Huber loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(HuberLoss(Delta, Reduction)); + } + + /// + /// Creates a Huber loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => HuberLoss(Delta, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/KullbackLeiblerDivergence.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/KullbackLeiblerDivergence.cs new file mode 100644 index 00000000..f77a3b7e --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/KullbackLeiblerDivergence.cs @@ -0,0 +1,50 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a Kullback-Leibler divergence loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a Kullback-Leibler divergence loss module.")] +[DisplayName("KLDiv")] +public class KullbackLeiblerDivergence +{ + /// + /// Determines whether the target is given as log-probabilities. + /// + [Description("Determines whether the target is given as log-probabilities.")] + public bool LogTarget { get; set; } = true; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a Kullback-Leibler divergence loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(KLDivLoss(LogTarget, Reduction)); + } + + /// + /// Creates a Kullback-Leibler divergence loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => KLDivLoss(LogTarget, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/MarginRanking.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/MarginRanking.cs new file mode 100644 index 00000000..0ae39ebf --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/MarginRanking.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a margin ranking loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a margin ranking loss module.")] +public class MarginRanking +{ + /// + /// The margin, a float value. + /// + [Description("The margin, a float value.")] + public double Margin { get; set; } = 0D; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a margin ranking loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(MarginRankingLoss(Margin, Reduction)); + } + + /// + /// Creates a margin ranking loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MarginRankingLoss(Margin, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/MeanAbsoluteError.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/MeanAbsoluteError.cs new file mode 100644 index 00000000..ad5e0085 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/MeanAbsoluteError.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates an L1 loss (L1Loss) module, known as mean absolute error (MAE). +/// +/// +/// See for more information. +/// +[Description("Creates an L1 loss (L1Loss) module, also known as mean absolute error (MAE).")] +[DisplayName("L1")] +public class MeanAbsoluteError +{ + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates an L1 loss (L1Loss) module. + /// + /// + public IObservable Process() + { + return Observable.Return(L1Loss(Reduction)); + } + + /// + /// Creates an L1 loss (L1Loss) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => L1Loss(Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/MeanSquaredError.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/MeanSquaredError.cs new file mode 100644 index 00000000..f262bc05 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/MeanSquaredError.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a mean squared error (MSE) loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a mean squared error (MSE) loss module.")] +[DisplayName("MSE")] +public class MeanSquaredError +{ + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a mean squared error (MSE) loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(MSELoss(Reduction)); + } + + /// + /// Creates a mean squared error (MSE) loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MSELoss(Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/MultiLabelMargin.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/MultiLabelMargin.cs new file mode 100644 index 00000000..3f02ad8f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/MultiLabelMargin.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a multi-class multi-classification margin loss (MultiLabelMarginLoss) module. +/// +/// +/// See for more information. +/// +[Description("Creates a multi-class multi-classification margin loss (MultiLabelMarginLoss) module.")] +public class MultiLabelMargin +{ + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a multi-label margin loss (MultiLabelMarginLoss) module. + /// + /// + public IObservable Process() + { + return Observable.Return(MultiLabelMarginLoss(Reduction)); + } + + /// + /// Creates a multi-label margin loss (MultiLabelMarginLoss) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MultiLabelMarginLoss(Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/MultiLabelSoftMargin.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/MultiLabelSoftMargin.cs new file mode 100644 index 00000000..58b2d1c7 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/MultiLabelSoftMargin.cs @@ -0,0 +1,71 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a multi-label soft margin loss (MultiLabelSoftMarginLoss) module. +/// +/// +/// See for more information. +/// +[Description("Creates a multi-label soft margin loss (MultiLabelSoftMarginLoss) module.")] +[TypeConverter(typeof(TensorOperatorConverter))] +public class MultiLabelSoftMargin : IScalarTypeProvider +{ + /// + /// The manual rescaling weight given to each class. + /// + [XmlIgnore] + [Description("The manual rescaling weight given to each class.")] + [TypeConverter(typeof(TensorConverter))] + public Tensor Weight { get; set; } = null; + + /// + /// The values of the weight tensor in XML string format. + /// + [Browsable(false)] + [XmlElement(nameof(Weight))] + [EditorBrowsable(EditorBrowsableState.Never)] + public string WeightXml + { + get => TensorConverter.ConvertToString(Weight, Type); + set => Weight = TensorConverter.ConvertFromString(value, Type); + } + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// The data type of the tensor elements. + /// + [Description("The data type of the tensor elements.")] + public ScalarType Type { get; set; } = ScalarType.Float32; + + /// + /// Creates a multi-label soft margin loss (MultiLabelSoftMarginLoss) module. + /// + /// + public IObservable Process() + { + return Observable.Return(MultiLabelSoftMarginLoss(Weight, Reduction)); + } + + /// + /// Creates a multi-label soft margin loss (MultiLabelSoftMarginLoss) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MultiLabelSoftMarginLoss(Weight, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/MultiMargin.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/MultiMargin.cs new file mode 100644 index 00000000..c67ac114 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/MultiMargin.cs @@ -0,0 +1,83 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a multi-class margin loss (MultiMarginLoss) module. +/// +/// +/// See for more information. +/// +[Description("Creates a multi-class margin loss (MultiMarginLoss) module.")] +[TypeConverter(typeof(TensorOperatorConverter))] +public class MultiMargin : IScalarTypeProvider +{ + /// + /// The p parameter, can be either 1 or 2. + /// + [Description("The p parameter, can be either 1 or 2.")] + public int P { get; set; } = 1; + + /// + /// The margin parameter, a float value. + /// + [Description("The margin parameter, a float value.")] + public double Margin { get; set; } = 1D; + + /// + /// The manual rescaling weight given to each class. + /// + [XmlIgnore] + [Description("The manual rescaling weight given to each class.")] + [TypeConverter(typeof(TensorConverter))] + public Tensor Weight { get; set; } = null; + + /// + /// The values of the weight tensor in XML string format. + /// + [Browsable(false)] + [XmlElement(nameof(Weight))] + [EditorBrowsable(EditorBrowsableState.Never)] + public string WeightXml + { + get => TensorConverter.ConvertToString(Weight, Type); + set => Weight = TensorConverter.ConvertFromString(value, Type); + } + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// The data type of the tensor elements. + /// + [Description("The data type of the tensor elements.")] + public ScalarType Type { get; set; } = ScalarType.Float32; + + /// + /// Creates a multi-class margin loss (MultiMarginLoss) module. + /// + /// + public IObservable Process() + { + return Observable.Return(MultiMarginLoss(P, Margin, Weight, Reduction)); + } + + /// + /// Creates a multi-class margin loss (MultiMarginLoss) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MultiMarginLoss(P, Margin, Weight, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/NegativeLogLikelihood.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/NegativeLogLikelihood.cs new file mode 100644 index 00000000..eb65260d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/NegativeLogLikelihood.cs @@ -0,0 +1,72 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a negative log likelihood (NLL) loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a negative log likelihood (NLL) loss module.")] +[TypeConverter(typeof(TensorOperatorConverter))] +[DisplayName("NLL")] +public class NegativeLogLikelihood : IScalarTypeProvider +{ + /// + /// The manual rescaling weight given to each class. + /// + [XmlIgnore] + [Description("The manual rescaling weight given to each class.")] + [TypeConverter(typeof(TensorConverter))] + public Tensor Weight { get; set; } = null; + + /// + /// The values of the weight tensor in XML string format. + /// + [Browsable(false)] + [XmlElement(nameof(Weight))] + [EditorBrowsable(EditorBrowsableState.Never)] + public string WeightXml + { + get => TensorConverter.ConvertToString(Weight, Type); + set => Weight = TensorConverter.ConvertFromString(value, Type); + } + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// The data type of the tensor elements. + /// + [Description("The data type of the tensor elements.")] + public ScalarType Type { get; set; } = ScalarType.Float32; + + /// + /// Creates a negative log likelihood (NLL) loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(NLLLoss(Weight, Reduction)); + } + + /// + /// Creates a negative log likelihood (NLL) loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => NLLLoss(Weight, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/PoissonNegativeLogLikelihood.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/PoissonNegativeLogLikelihood.cs new file mode 100644 index 00000000..65af3083 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/PoissonNegativeLogLikelihood.cs @@ -0,0 +1,62 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a poisson negative log likelihood (PoissonNLL) loss module. +/// +/// +/// See for more information. +/// +[Description("Creates a poisson negative log likelihood (PoissonNLL) loss module.")] +[DisplayName("PoissonNLL")] +public class PoissonNegativeLogLikelihood +{ + /// + /// Determines whether to take the logarithm of the input. + /// + [Description("Determines whether to take the logarithm of the input.")] + public bool LogInput { get; set; } = true; + + /// + /// Determines whether to compute the full loss. + /// + [Description("Determines whether to compute the full loss.")] + public bool Full { get; set; } = false; + + /// + /// The small value added to avoid evaluating log(0) when LogInput is false. + /// + [Description("The small value added to avoid evaluating log(0) when LogInput is false.")] + public float Eps { get; set; } = 1E-08F; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a poisson negative log likelihood (PoissonNLL) loss module. + /// + /// + public IObservable Process() + { + return Observable.Return(PoissonNLLLoss(LogInput, Full, Eps, Reduction)); + } + + /// + /// Creates a poisson negative log likelihood (PoissonNLL) loss module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => PoissonNLLLoss(LogInput, Full, Eps, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/SmoothL1.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/SmoothL1.cs new file mode 100644 index 00000000..f4b3b238 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/SmoothL1.cs @@ -0,0 +1,49 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a smooth L1 loss (SmoothL1Loss) module. +/// +/// +/// See for more information. +/// +[Description("Creates a smooth L1 loss (SmoothL1Loss) module.")] +public class SmoothL1 +{ + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// The threshold at which to change between L1 and L2 loss. + /// + [Description("The threshold at which to change between L1 and L2 loss.")] + public double Beta { get; set; } = 1D; + + /// + /// Creates a smooth L1 loss (SmoothL1Loss) module. + /// + /// + public IObservable Process() + { + return Observable.Return(SmoothL1Loss(Reduction, Beta)); + } + + /// + /// Creates a smooth L1 loss (SmoothL1Loss) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => SmoothL1Loss(Reduction, Beta)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/SoftMargin.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/SoftMargin.cs new file mode 100644 index 00000000..211b74e1 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/SoftMargin.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a soft margin loss (SoftMarginLoss) module. +/// +/// +/// See for more information. +/// +[Description("Creates a soft margin loss (SoftMarginLoss) module.")] +public class SoftMargin +{ + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a soft margin loss (SoftMarginLoss) module. + /// + /// + public IObservable Process() + { + return Observable.Return(SoftMarginLoss(Reduction)); + } + + /// + /// Creates a soft margin loss (SoftMarginLoss) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => SoftMarginLoss(Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Loss/TripletMargin.cs b/src/Bonsai.ML.Torch/NeuralNets/Loss/TripletMargin.cs new file mode 100644 index 00000000..e9403d15 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Loss/TripletMargin.cs @@ -0,0 +1,67 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Loss; + +/// +/// Represents an operator that creates a triplet margin loss (TripletMarginLoss) module. +/// +/// +/// See for more information. +/// +[Description("Creates a triplet margin loss (TripletMarginLoss) module.")] +public class TripletMargin +{ + /// + /// The margin parameter, a float value. + /// + [Description("The margin parameter, a float value.")] + public double Margin { get; set; } = 1D; + + /// + /// The p parameter, the norm degree for pairwise distance. + /// + [Description("The p parameter, the norm degree for pairwise distance.")] + public long P { get; set; } = 2; + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability")] + public double Eps { get; set; } = 1E-06D; + + /// + /// Determines whether to use the swapped distance. + /// + [Description("Determines whether to use the swapped distance.")] + public bool Swap { get; set; } = false; + + /// + /// The reduction type to apply to the output. + /// + [Description("The reduction type to apply to the output.")] + public Reduction Reduction { get; set; } = Reduction.Mean; + + /// + /// Creates a triplet margin loss (TripletMarginLoss) module. + /// + /// + public IObservable Process() + { + return Observable.Return(TripletMarginLoss(Margin, P, Eps, Swap, Reduction)); + } + + /// + /// Creates a triplet margin loss (TripletMarginLoss) module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => TripletMarginLoss(Margin, P, Eps, Swap, Reduction)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/LossBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/LossBuilder.cs new file mode 100644 index 00000000..d7ec3604 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/LossBuilder.cs @@ -0,0 +1,60 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a loss module. +/// +[XmlInclude(typeof(Loss.BinaryCrossEntropy))] +[XmlInclude(typeof(Loss.BinaryCrossEntropyWithLogits))] +[XmlInclude(typeof(Loss.ConnectionistTemporalClassification))] +[XmlInclude(typeof(Loss.CosineEmbedding))] +[XmlInclude(typeof(Loss.CrossEntropy))] +[XmlInclude(typeof(Loss.GaussianNegativeLogLikelihood))] +[XmlInclude(typeof(Loss.HingeEmbedding))] +[XmlInclude(typeof(Loss.Huber))] +[XmlInclude(typeof(Loss.KullbackLeiblerDivergence))] +[XmlInclude(typeof(Loss.MarginRanking))] +[XmlInclude(typeof(Loss.MeanAbsoluteError))] +[XmlInclude(typeof(Loss.MeanSquaredError))] +[XmlInclude(typeof(Loss.MultiMargin))] +[XmlInclude(typeof(Loss.MultiLabelMargin))] +[XmlInclude(typeof(Loss.MultiLabelSoftMargin))] +[XmlInclude(typeof(Loss.NegativeLogLikelihood))] +[XmlInclude(typeof(Loss.PoissonNegativeLogLikelihood))] +[XmlInclude(typeof(Loss.SmoothL1))] +[XmlInclude(typeof(Loss.SoftMargin))] +[XmlInclude(typeof(Loss.TripletMargin))] +[DefaultProperty(nameof(LossModule))] +[Combinator] +[Description("Creates a loss module.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class LossBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "Loss"; + + /// + /// Initializes a new instance of the class. + /// + public LossBuilder() + { + Module = new Loss.GaussianNegativeLogLikelihood(); + } + + /// + /// Gets or sets the specific loss module. + /// + [DesignOnly(true)] + [DisplayName("Loss")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific loss module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object LossModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/MergeModules.cs b/src/Bonsai.ML.Torch/NeuralNets/MergeModules.cs new file mode 100644 index 00000000..1807287a --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/MergeModules.cs @@ -0,0 +1,33 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Linq; +using System.Collections.Generic; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that merges derived modules into a collection of base module types. +/// +[Combinator] +[Description("Merges derived modules into a collection of base module types.")] +[WorkflowElementCategory(ElementCategory.Combinator)] +public class MergeModules +{ + /// + /// Merges the input sequences of modules and casts them to the base module class. + /// + /// + /// + public IObservable>> Process(params IObservable[] sources) + { + return Observable + .Concat(sources.Select(source => + source.Take(1))) + .OfType>() + .ToList() + .Select(modules => modules.AsEnumerable()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Models/AlexNet.cs b/src/Bonsai.ML.Torch/NeuralNets/Models/AlexNet.cs deleted file mode 100644 index c3d19d55..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/Models/AlexNet.cs +++ /dev/null @@ -1,74 +0,0 @@ -using TorchSharp; -using static TorchSharp.torch; -using static TorchSharp.torch.nn; - -namespace Bonsai.ML.Torch.NeuralNets.Models -{ - /// - /// Modified version of original AlexNet to fix CIFAR10 32x32 images. - /// - internal class AlexNet : Module - { - private readonly Module features; - private readonly Module avgPool; - private readonly Module classifier; - - /// - /// Constructs a new AlexNet model. - /// - /// - /// - /// - public AlexNet(string name, int numClasses, Device device = null) : base(name) - { - features = Sequential( - ("c1", Conv2d(3, 64, kernel_size: 3, stride: 2, padding: 1)), - ("r1", ReLU(inplace: true)), - ("mp1", MaxPool2d(kernel_size: [ 2, 2 ])), - ("c2", Conv2d(64, 192, kernel_size: 3, padding: 1)), - ("r2", ReLU(inplace: true)), - ("mp2", MaxPool2d(kernel_size: [ 2, 2 ])), - ("c3", Conv2d(192, 384, kernel_size: 3, padding: 1)), - ("r3", ReLU(inplace: true)), - ("c4", Conv2d(384, 256, kernel_size: 3, padding: 1)), - ("r4", ReLU(inplace: true)), - ("c5", Conv2d(256, 256, kernel_size: 3, padding: 1)), - ("r5", ReLU(inplace: true)), - ("mp3", MaxPool2d(kernel_size: [ 2, 2 ]))); - - avgPool = AdaptiveAvgPool2d([ 2, 2 ]); - - classifier = Sequential( - ("d1", Dropout()), - ("l1", Linear(256 * 2 * 2, 4096)), - ("r1", ReLU(inplace: true)), - ("d2", Dropout()), - ("l2", Linear(4096, 4096)), - ("r3", ReLU(inplace: true)), - ("d3", Dropout()), - ("l3", Linear(4096, numClasses)) - ); - - RegisterComponents(); - - if (device != null && device.type != DeviceType.CPU) - this.to(device); - } - - /// - /// Forward pass of the AlexNet model. - /// - /// - /// - public override Tensor forward(Tensor input) - { - var f = features.forward(input); - var avg = avgPool.forward(f); - - var x = avg.view([ avg.shape[0], 256 * 2 * 2 ]); - - return classifier.forward(x); - } - } - -} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Models/Mnist.cs b/src/Bonsai.ML.Torch/NeuralNets/Models/Mnist.cs deleted file mode 100644 index a1606ddc..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/Models/Mnist.cs +++ /dev/null @@ -1,86 +0,0 @@ -using TorchSharp; -using static TorchSharp.torch; -using static TorchSharp.torch.nn; - -namespace Bonsai.ML.Torch.NeuralNets.Models -{ - /// - /// Represents a simple convolutional neural network for the MNIST dataset. - /// - internal class Mnist : Module - { - private readonly Module conv1; - private readonly Module conv2; - private readonly Module fc1; - private readonly Module fc2; - - private readonly Module pool1; - - private readonly Module relu1; - private readonly Module relu2; - private readonly Module relu3; - - private readonly Module dropout1; - private readonly Module dropout2; - - private readonly Module flatten; - private readonly Module logsm; - - /// - /// Constructs a new Mnist model. - /// - /// - /// - /// - public Mnist(string name, int numClasses, Device device = null) : base(name) - { - conv1 = Conv2d(1, 32, 3); - conv2 = Conv2d(32, 64, 3); - fc1 = Linear(9216, 128); - fc2 = Linear(128, numClasses); - - pool1 = MaxPool2d(kernel_size: [2, 2]); - - relu1 = ReLU(); - relu2 = ReLU(); - relu3 = ReLU(); - - dropout1 = Dropout(0.25); - dropout2 = Dropout(0.5); - - flatten = Flatten(); - logsm = LogSoftmax(1); - - RegisterComponents(); - - if (device != null && device.type != DeviceType.CPU) - this.to(device); - } - - /// - /// Forward pass of the Mnist model. - /// - /// - /// - public override Tensor forward(Tensor input) - { - var l11 = conv1.forward(input); - var l12 = relu1.forward(l11); - - var l21 = conv2.forward(l12); - var l22 = relu2.forward(l21); - var l23 = pool1.forward(l22); - var l24 = dropout1.forward(l23); - - var x = flatten.forward(l24); - - var l31 = fc1.forward(x); - var l32 = relu3.forward(l31); - var l33 = dropout2.forward(l32); - - var l41 = fc2.forward(l33); - - return logsm.forward(l41); - } - } -} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Models/MobileNet.cs b/src/Bonsai.ML.Torch/NeuralNets/Models/MobileNet.cs deleted file mode 100644 index a5f7701a..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/Models/MobileNet.cs +++ /dev/null @@ -1,77 +0,0 @@ -using System; -using System.Collections.Generic; -using TorchSharp; -using static TorchSharp.torch; -using static TorchSharp.torch.nn; - -namespace Bonsai.ML.Torch.NeuralNets.Models -{ - /// - /// MobileNet model. - /// - internal class MobileNet : Module - { - private readonly long[] planes = [ 64, 128, 128, 256, 256, 512, 512, 512, 512, 512, 512, 1024, 1024 ]; - private readonly long[] strides = [ 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1 ]; - - private readonly Module layers; - - /// - /// Constructs a new MobileNet model. - /// - /// - /// - /// - /// - public MobileNet(string name, int numClasses, Device device = null) : base(name) - { - if (planes.Length != strides.Length) throw new ArgumentException("'planes' and 'strides' must have the same length."); - - var modules = new List<(string, Module)> - { - ($"conv2d-first", Conv2d(3, 32, kernel_size: 3, stride: 1, padding: 1, bias: false)), - ($"bnrm2d-first", BatchNorm2d(32)), - ($"relu-first", ReLU()) - }; - MakeLayers(modules, 32); - modules.Add(("avgpool", AvgPool2d([2, 2]))); - modules.Add(("flatten", Flatten())); - modules.Add(($"linear", Linear(planes[planes.Length-1], numClasses))); - - layers = Sequential(modules); - - RegisterComponents(); - - if (device != null && device.type != DeviceType.CPU) - this.to(device); - } - - private void MakeLayers(List<(string, Module)> modules, long in_planes) - { - - for (var i = 0; i < strides.Length; i++) { - var out_planes = planes[i]; - var stride = strides[i]; - - modules.Add(($"conv2d-{i}a", Conv2d(in_planes, in_planes, kernel_size: 3, stride: stride, padding: 1, groups: in_planes, bias: false))); - modules.Add(($"bnrm2d-{i}a", BatchNorm2d(in_planes))); - modules.Add(($"relu-{i}a", ReLU())); - modules.Add(($"conv2d-{i}b", Conv2d(in_planes, out_planes, kernel_size: 1L, stride: 1L, padding: 0L, bias: false))); - modules.Add(($"bnrm2d-{i}b", BatchNorm2d(out_planes))); - modules.Add(($"relu-{i}b", ReLU())); - - in_planes = out_planes; - } - } - - /// - /// Forward pass of the MobileNet model. - /// - /// - /// - public override Tensor forward(Tensor input) - { - return layers.forward(input); - } - } -} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Models/ModelArchitecture.cs b/src/Bonsai.ML.Torch/NeuralNets/Models/ModelArchitecture.cs deleted file mode 100644 index 9d6fe55b..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/Models/ModelArchitecture.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace Bonsai.ML.Torch.NeuralNets.Models -{ - /// - /// Represents the architecture of a neural network model. - /// - public enum ModelArchitecture - { - /// - /// The AlexNet model architecture. - /// - AlexNet, - - /// - /// The MobileNet model architecture. - /// - MobileNet, - - /// - /// The Mnist model architecture. - /// - Mnist - } -} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/ModuleCombinatorBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/ModuleCombinatorBuilder.cs new file mode 100644 index 00000000..c0696ba6 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ModuleCombinatorBuilder.cs @@ -0,0 +1,155 @@ +using Bonsai.Expressions; +using System; +using System.ComponentModel; +using System.Linq; +using System.Linq.Expressions; +using System.Collections.Generic; +using System.Reflection; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents a base class for module combinator builders. +/// +public abstract class ModuleCombinatorBuilder : ExpressionBuilder, ICustomTypeDescriptor, INamedElement +{ + /// + string INamedElement.Name => $"{BuilderName}.{GetElementDisplayName(Module)}"; + + internal abstract string BuilderName { get; } + + internal object Module { get; set; } + + /// + public override Range ArgumentRange => Range.Create(0, 1); + + /// + public override Expression Build(IEnumerable arguments) + { + // We want to return an expression that constructs the module + var module = Module.GetType(); + + // arguments can either be empty or contain a single argument + if (!arguments.Any()) + { + // if empty, we call the non generic Process method + var methodInfo = module.GetMethods(BindingFlags.Public | BindingFlags.Instance).First(m => m.Name == "Process" && !m.IsGenericMethod); + return Expression.Call( + Expression.Constant(Module, module), + methodInfo + ); + } + else + { + // if there is an argument, we call the generic Process method + var argument = arguments.First(); + var argumentType = argument.Type.GetGenericArguments()[0]; + var methodInfo = module.GetMethods(BindingFlags.Public | BindingFlags.Instance).First(m => m.Name == "Process" && m.IsGenericMethodDefinition && m.GetGenericArguments().Length == 1); + var genericMethodInfo = methodInfo.MakeGenericMethod(argumentType); + return Expression.Call( + Expression.Constant(Module, module), + genericMethodInfo, + argument + ); + } + } + + AttributeCollection ICustomTypeDescriptor.GetAttributes() + { + var attributes = TypeDescriptor.GetAttributes(GetType()); + var defaultProperty = TypeDescriptor.GetDefaultProperty(GetType()); + if (defaultProperty != null) + { + var instance = defaultProperty.GetValue(this); + var instanceAttributes = TypeDescriptor.GetAttributes(instance); + if (instanceAttributes[typeof(DescriptionAttribute)] is DescriptionAttribute description) + { + return AttributeCollection.FromExisting(attributes, description); + } + } + + return attributes; + } + + string ICustomTypeDescriptor.GetClassName() + { + return TypeDescriptor.GetClassName(GetType()); + } + + string ICustomTypeDescriptor.GetComponentName() + { + return null; + } + + TypeConverter ICustomTypeDescriptor.GetConverter() + { + return TypeDescriptor.GetConverter(GetType()); + } + + EventDescriptor ICustomTypeDescriptor.GetDefaultEvent() + { + return null; + } + + PropertyDescriptor ICustomTypeDescriptor.GetDefaultProperty() + { + var defaultProperty = TypeDescriptor.GetDefaultProperty(GetType()); + return defaultProperty != null ? new ModuleCombinatorPropertyDescriptor(defaultProperty) : null; + } + + object ICustomTypeDescriptor.GetEditor(Type editorBaseType) + { + return TypeDescriptor.GetEditor(GetType(), editorBaseType); + } + + EventDescriptorCollection ICustomTypeDescriptor.GetEvents() + { + return EventDescriptorCollection.Empty; + } + + EventDescriptorCollection ICustomTypeDescriptor.GetEvents(Attribute[] attributes) + { + return EventDescriptorCollection.Empty; + } + + PropertyDescriptorCollection ICustomTypeDescriptor.GetProperties() + { + return ((ICustomTypeDescriptor)this).GetProperties([]); + } + + PropertyDescriptorCollection ICustomTypeDescriptor.GetProperties(Attribute[] attributes) + { + var baseProperties = TypeDescriptor.GetProperties(GetType(), attributes); + var defaultProperty = TypeDescriptor.GetDefaultProperty(GetType()); + if (defaultProperty != null) + { + var instance = defaultProperty.GetValue(this); + var instanceProperties = TypeDescriptor.GetProperties(instance, attributes); + var properties = new PropertyDescriptor[baseProperties.Count + instanceProperties.Count]; + for (int i = 0; i < baseProperties.Count; i++) + { + var baseProperty = baseProperties[i]; + if (baseProperty == defaultProperty) + { + baseProperty = new ModuleCombinatorPropertyDescriptor(defaultProperty); + } + + properties[i] = baseProperty; + } + + for (int i = 0; i < instanceProperties.Count; i++) + { + var expandedProperty = instanceProperties[i]; + properties[i + baseProperties.Count] = expandedProperty; + } + return new PropertyDescriptorCollection(properties); + } + + return baseProperties; + } + + object ICustomTypeDescriptor.GetPropertyOwner(PropertyDescriptor pd) + { + return pd?.ComponentType.IsAssignableFrom(GetType()) == true ? this : Module; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ModuleCombinatorPropertyDescriptor.cs b/src/Bonsai.ML.Torch/NeuralNets/ModuleCombinatorPropertyDescriptor.cs new file mode 100644 index 00000000..fdf418a7 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ModuleCombinatorPropertyDescriptor.cs @@ -0,0 +1,56 @@ +using System; +using System.ComponentModel; + +namespace Bonsai.ML.Torch.NeuralNets; + +internal class ModuleCombinatorPropertyDescriptor(PropertyDescriptor descriptor) : PropertyDescriptor(descriptor) +{ + private readonly PropertyDescriptor descriptor = descriptor; + + public override Type ComponentType => descriptor.ComponentType; + + public override bool IsReadOnly => false; + + public override Type PropertyType => typeof(Type); + + public override bool CanResetValue(object component) + { + return true; + } + + public override object GetValue(object component) + { + component = descriptor.GetValue(component); + return component?.GetType(); + } + + public override void ResetValue(object component) + { + descriptor.SetValue(component, null); + } + + public override void SetValue(object component, object value) + { + var currentValue = descriptor.GetValue(component); + var newValue = Activator.CreateInstance((Type)value); + + var newProperties = TypeDescriptor.GetProperties(newValue); + var currentProperties = TypeDescriptor.GetProperties(currentValue); + foreach (PropertyDescriptor property in newProperties) + { + var mergeProperty = currentProperties[property.Name]; + if (mergeProperty?.PropertyType == property.PropertyType) + { + var propertyValue = mergeProperty.GetValue(currentValue); + property.SetValue(newValue, propertyValue); + } + } + + descriptor.SetValue(component, newValue); + } + + public override bool ShouldSerializeValue(object component) + { + return true; + } +} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/ModuleTypeConverter.cs b/src/Bonsai.ML.Torch/NeuralNets/ModuleTypeConverter.cs new file mode 100644 index 00000000..a71ad672 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ModuleTypeConverter.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Provides a type converter to convert modules to and from other representations. +/// +public class ModuleTypeConverter : TypeConverter +{ + internal static IEnumerable GetInstanceTypes(ITypeDescriptorContext context) + { + var builderType = context.Instance?.GetType() ?? context.PropertyDescriptor.ComponentType; + var includeAttributes = (XmlIncludeAttribute[])builderType.GetCustomAttributes(typeof(XmlIncludeAttribute), inherit: true); + if (includeAttributes.Length > 0) + { + return includeAttributes.Select(attribute => attribute.Type); + } + + var propertyInfo = builderType.GetProperty(context.PropertyDescriptor.Name); + if (propertyInfo == null) return Enumerable.Empty(); + + var elementAttributes = (XmlElementAttribute[])propertyInfo.GetCustomAttributes(typeof(XmlElementAttribute), inherit: true); + if (elementAttributes.Length > 0) + { + return elementAttributes.Select(attribute => attribute.Type); + } + + return propertyInfo.PropertyType.GetCustomAttributes().Select(attribute => attribute.Type); + } + + static string GetDisplayName(Type type) + { + var displayNameAttribute = (DisplayNameAttribute)type.GetCustomAttribute(typeof(DisplayNameAttribute)); + if (!string.IsNullOrEmpty(displayNameAttribute?.DisplayName)) + { + return displayNameAttribute.DisplayName; + } + + return type.Name; + } + + /// + public override bool CanConvertFrom(ITypeDescriptorContext context, Type sourceType) + { + return sourceType == typeof(string); + } + + /// + public override object ConvertFrom(ITypeDescriptorContext context, CultureInfo culture, object value) + { + if (value is string typeName) + { + return GetInstanceTypes(context).FirstOrDefault( + type => string.Equals(GetDisplayName(type), typeName, StringComparison.OrdinalIgnoreCase)); + } + + return null; + } + + /// + public override object ConvertTo(ITypeDescriptorContext context, CultureInfo culture, object value, Type destinationType) + { + if (destinationType == typeof(string) && value is Type valueType) + { + return GetDisplayName(valueType); + } + + return null; + } + + /// + public override bool GetStandardValuesSupported(ITypeDescriptorContext context) + { + return true; + } + + /// + public override bool GetStandardValuesExclusive(ITypeDescriptorContext context) + { + return true; + } + + /// + public override StandardValuesCollection GetStandardValues(ITypeDescriptorContext context) + { + var includeTypes = GetInstanceTypes(context).ToArray(); + return new StandardValuesCollection(includeTypes); + } +} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Normalization/BatchNorm1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Normalization/BatchNorm1d.cs new file mode 100644 index 00000000..2cfb1a11 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Normalization/BatchNorm1d.cs @@ -0,0 +1,81 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Normalization; + +/// +/// Represents an operator that creates a 1D batch normalization module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D batch normalization module.")] +public class BatchNorm1d +{ + /// + /// The number of features or channels of the input. + /// + [Description("The number of features or channels of the input.")] + public long Features { get; set; } + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-05D; + + /// + /// The value used for computing the running mean and running variance. + /// + [Description("The value used for computing the running mean and running variance.")] + public double Momentum { get; set; } = 0.1D; + + /// + /// If set to true, this module has learnable affine parameters. + /// + [Description("If set to true, this module has learnable affine parameters.")] + public bool Affine { get; set; } = true; + + /// + /// If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics. + /// + [Description("If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics.")] + public bool TrackRunningStats { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a BatchNorm1d module. + /// + /// + public IObservable Process() + { + return Observable.Return(BatchNorm1d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } + + /// + /// Creates a BatchNorm1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => BatchNorm1d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Normalization/BatchNorm2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Normalization/BatchNorm2d.cs new file mode 100644 index 00000000..26cfd3f4 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Normalization/BatchNorm2d.cs @@ -0,0 +1,81 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Normalization; + +/// +/// Represents an operator that creates a 2D batch normalization module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D batch normalization module.")] +public class BatchNorm2d +{ + /// + /// The number of features of the input. + /// + [Description("The number of features of the input.")] + public long Features { get; set; } + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-05D; + + /// + /// The value used for computing the running mean and running variance. + /// + [Description("The value used for computing the running mean and running variance.")] + public double Momentum { get; set; } = 0.1D; + + /// + /// If set to true, this module has learnable affine parameters. + /// + [Description("If set to true, this module has learnable affine parameters")] + public bool Affine { get; set; } = true; + + /// + /// If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics. + /// + [Description("If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics.")] + public bool TrackRunningStats { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a BatchNorm2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(BatchNorm2d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } + + /// + /// Creates a BatchNorm2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => BatchNorm2d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Normalization/BatchNorm3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Normalization/BatchNorm3d.cs new file mode 100644 index 00000000..07e960a7 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Normalization/BatchNorm3d.cs @@ -0,0 +1,81 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Normalization; + +/// +/// Represents an operator that creates a 3D batch normalization module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D batch normalization module.")] +public class BatchNorm3d +{ + /// + /// The number of features of the input. + /// + [Description("The number of features of the input.")] + public long Features { get; set; } + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-05D; + + /// + /// The value used for computing the running mean and running variance. + /// + [Description("The value used for computing the running mean and running variance.")] + public double Momentum { get; set; } = 0.1D; + + /// + /// If set to true, this module has learnable affine parameters. + /// + [Description("If set to true, this module has learnable affine parameters")] + public bool Affine { get; set; } = true; + + /// + /// If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics. + /// + [Description("If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics.")] + public bool TrackRunningStats { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a BatchNorm3d module. + /// + /// + public IObservable Process() + { + return Observable.Return(BatchNorm3d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } + + /// + /// Creates a BatchNorm3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => BatchNorm3d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Normalization/GroupNorm.cs b/src/Bonsai.ML.Torch/NeuralNets/Normalization/GroupNorm.cs new file mode 100644 index 00000000..74bc73e0 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Normalization/GroupNorm.cs @@ -0,0 +1,75 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Normalization; + +/// +/// Represents an operator that creates a group normalization module. +/// +/// +/// See for more information. +/// +[Description("Creates a group normalization module.")] +public class GroupNorm +{ + /// + /// The number of groups to separate the channels into. + /// + [Description("The number of groups to separate the channels into.")] + public long NumGroups { get; set; } + + /// + /// The number of channels in the input. + /// + [Description("The number of channels in the input.")] + public long NumChannels { get; set; } + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-05D; + + /// + /// A boolean value that when set to true, this module has learnable per-channel affine parameters initialized to ones (for weights) and zeros (for biases). + /// + [Description("A boolean value that when set to true, this module has learnable per-channel affine parameters initialized to ones (for weights) and zeros (for biases).")] + public bool Affine { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a GroupNorm module. + /// + /// + public IObservable Process() + { + return Observable.Return(GroupNorm(NumGroups, NumChannels, Eps, Affine, Device, Type)); + } + + /// + /// Creates a GroupNorm module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => GroupNorm(NumGroups, NumChannels, Eps, Affine, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Normalization/InstanceNorm1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Normalization/InstanceNorm1d.cs new file mode 100644 index 00000000..d18a2f27 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Normalization/InstanceNorm1d.cs @@ -0,0 +1,81 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Normalization; + +/// +/// Represents an operator that creates a 1d instance normalization module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1d instance normalization module.")] +public class InstanceNorm1d +{ + /// + /// The number of features or channels in the input tensor. + /// + [Description("The number of features or channels in the input tensor.")] + public long Features { get; set; } + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-05D; + + /// + /// The value used for computing the running mean and running variance. + /// + [Description("The value used for computing the running mean and running variance.")] + public double Momentum { get; set; } = 0.1D; + + /// + /// If set to true, this module has learnable affine parameters. + /// + [Description("If set to true, this module has learnable affine parameters.")] + public bool Affine { get; set; } = false; + + /// + /// If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics. + /// + [Description("If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics.")] + public bool TrackRunningStats { get; set; } = false; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates an InstanceNorm1d module. + /// + /// + public IObservable Process() + { + return Observable.Return(InstanceNorm1d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } + + /// + /// Creates an InstanceNorm1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => InstanceNorm1d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Normalization/InstanceNorm2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Normalization/InstanceNorm2d.cs new file mode 100644 index 00000000..64c6d287 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Normalization/InstanceNorm2d.cs @@ -0,0 +1,81 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Normalization; + +/// +/// Represents an operator that creates a 2d instance normalization module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2d instance normalization module.")] +public class InstanceNorm2d +{ + /// + /// The number of features in the input tensor. + /// + [Description("The number of features in the input tensor.")] + public long Features { get; set; } + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-05D; + + /// + /// The value used for computing the running mean and running variance. + /// + [Description("The value used for computing the running mean and running variance.")] + public double Momentum { get; set; } = 0.1D; + + /// + /// If set to true, this module has learnable affine parameters. + /// + [Description("If set to true, this module has learnable affine parameters.")] + public bool Affine { get; set; } = false; + + /// + /// If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics. + /// + [Description("If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics.")] + public bool TrackRunningStats { get; set; } = false; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates an InstanceNorm2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(InstanceNorm2d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } + + /// + /// Creates an InstanceNorm2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => InstanceNorm2d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Normalization/InstanceNorm3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Normalization/InstanceNorm3d.cs new file mode 100644 index 00000000..b6af2cfc --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Normalization/InstanceNorm3d.cs @@ -0,0 +1,81 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Normalization; + +/// +/// Represents an operator that creates a 3d instance normalization module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3d instance normalization module.")] +public class InstanceNorm3d +{ + /// + /// The number of features in the input tensor. + /// + [Description("The number of features in the input tensor.")] + public long Features { get; set; } + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-05D; + + /// + /// The value used for computing the running mean and running variance. + /// + [Description("The value used for computing the running mean and running variance.")] + public double Momentum { get; set; } = 0.1D; + + /// + /// If set to true, this module has learnable affine parameters. + /// + [Description("If set to true, this module has learnable affine parameters.")] + public bool Affine { get; set; } = false; + + /// + /// If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics. + /// + [Description("If set to true, this module tracks the running mean and variance, otherwise, the module will use batch statistics.")] + public bool TrackRunningStats { get; set; } = false; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates an InstanceNorm3d module. + /// + /// + public IObservable Process() + { + return Observable.Return(InstanceNorm3d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } + + /// + /// Creates an InstanceNorm3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => InstanceNorm3d(Features, Eps, Momentum, Affine, TrackRunningStats, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Normalization/LayerNorm.cs b/src/Bonsai.ML.Torch/NeuralNets/Normalization/LayerNorm.cs new file mode 100644 index 00000000..795830af --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Normalization/LayerNorm.cs @@ -0,0 +1,76 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Normalization; + +/// +/// Represents an operator that creates a layer normalization module. +/// +/// +/// See for more information. +/// +[Description("Creates a layer normalization module.")] +public class LayerNorm +{ + /// + /// The input shape from an expected input of size `[* x normalized_shape[0] x normalized_shape[1] x ... x normalized_shape[-1]]`. + /// + [Description("The input shape from an expected input of size `[* x normalized_shape[0] x normalized_shape[1] x ... x normalized_shape[-1]]`.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public long[] NormalizedShape { get; set; } + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-05D; + + /// + /// If true, this module has learnable per-element affine parameters initialized to ones (for weights) and zeros (for biases). + /// + [Description("If true, this module has learnable per-element affine parameters initialized to ones (for weights) and zeros (for biases).")] + public bool ElementwiseAffine { get; set; } = true; + + /// + /// If true, adds a learnable bias to the output. + /// + [Description("If true, adds a learnable bias to the output.")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a LayerNorm module. + /// + /// + public IObservable Process() + { + return Observable.Return(LayerNorm(NormalizedShape, Eps, ElementwiseAffine, Bias, Device, Type)); + } + + /// + /// Creates a LayerNorm module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => LayerNorm(NormalizedShape, Eps, ElementwiseAffine, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Normalization/LocalResponseNorm.cs b/src/Bonsai.ML.Torch/NeuralNets/Normalization/LocalResponseNorm.cs new file mode 100644 index 00000000..81500870 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Normalization/LocalResponseNorm.cs @@ -0,0 +1,61 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Normalization; + +/// +/// Represents an operator that creates a local response normalization module. +/// +/// +/// See for more information. +/// +[Description("Creates a local response normalization module.")] +public class LocalResponseNorm +{ + /// + /// The number of neighboring channels used for normalization. + /// + [Description("The number of neighboring channels used for normalization.")] + public long Size { get; set; } + + /// + /// The alpha parameter, which serves as a scaling factor. + /// + [Description("The alpha parameter, which serves as a scaling factor.")] + public double Alpha { get; set; } = 0.0001D; + + /// + /// The beta parameter, which serves as an exponent. + /// + [Description("The beta parameter, which serves as an exponent.")] + public double Beta { get; set; } = 0.75D; + + /// + /// The k parameter, which serves as an additive constant. + /// + [Description("The k parameter, which serves as an additive constant.")] + public double K { get; set; } = 1D; + + /// + /// Creates a LocalResponseNorm module. + /// + /// + public IObservable Process() + { + return Observable.Return(LocalResponseNorm(Size, Alpha, Beta, K)); + } + + /// + /// Creates a LocalResponseNorm module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => LocalResponseNorm(Size, Alpha, Beta, K)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/NormalizationModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/NormalizationModuleBuilder.cs new file mode 100644 index 00000000..e925f7fc --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/NormalizationModuleBuilder.cs @@ -0,0 +1,49 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a normalization module. +/// +[XmlInclude(typeof(Normalization.BatchNorm1d))] +[XmlInclude(typeof(Normalization.BatchNorm2d))] +[XmlInclude(typeof(Normalization.BatchNorm3d))] +[XmlInclude(typeof(Normalization.GroupNorm))] +[XmlInclude(typeof(Normalization.InstanceNorm1d))] +[XmlInclude(typeof(Normalization.InstanceNorm2d))] +[XmlInclude(typeof(Normalization.InstanceNorm3d))] +[XmlInclude(typeof(Normalization.LayerNorm))] +[XmlInclude(typeof(Normalization.LocalResponseNorm))] +[DefaultProperty(nameof(NormalizationModule))] +[Combinator] +[Description("Creates a normalization module.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class NormalizationModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "NormalizationModule"; + + /// + /// Initializes a new instance of the class. + /// + public NormalizationModuleBuilder() + { + Module = new Normalization.BatchNorm1d(); + } + + /// + /// Gets or sets the specific normalization module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific normalization module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object NormalizationModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/OptimizationStep.cs b/src/Bonsai.ML.Torch/NeuralNets/OptimizationStep.cs new file mode 100644 index 00000000..77e6377f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/OptimizationStep.cs @@ -0,0 +1,36 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.optim.lr_scheduler; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that performs a single optimization step using the specified optimizer. +/// +[Combinator] +[Description("Performs a single optimization step.")] +[WorkflowElementCategory(ElementCategory.Sink)] +public class OptimizationStep +{ + /// + /// Performs a single step using the specified optimizer. + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Do(input => input.step()); + } + + /// + /// Performs a single step using the specified learning rate scheduler. + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Do(input => input.step()); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer.cs deleted file mode 100644 index 4ab09dbd..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/Optimizer.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Bonsai.ML.Torch.NeuralNets -{ - /// - /// Represents an optimizer that updates the parameters of a neural network. - /// - public enum Optimizer - { - /// - /// Adam optimizer. - /// - Adam, - } -} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adadelta.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adadelta.cs new file mode 100644 index 00000000..d529e57f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adadelta.cs @@ -0,0 +1,60 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates an Adadelta optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates an Adadelta optimizer.")] +public class Adadelta +{ + /// + /// The learning rate coefficient that scales delta before it is applied to the parameters. + /// + [Description("The learning rate coefficient that scales delta before it is applied to the parameters.")] + public double LearningRate { get; set; } = 1D; + + /// + /// The coefficient used to calculate a running average of squared gradients. + /// + [Description("The coefficient used to calculate a running average of squared gradients.")] + public double Rho { get; set; } = 0.9D; + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-06D; + + /// + /// The weight decay coefficient, which adds L2 regularization to the loss. + /// + [Description("The weight decay coefficient, which adds L2 regularization to the loss.")] + public double WeightDecay { get; set; } = 0D; + + /// + /// If set to true, performs maximization instead of minimization of the params based on the objective. + /// + [Description("If set to true, performs maximization instead of minimization of the params based on the objective.")] + public bool Maximize { get; set; } = false; + + /// + /// Creates an Adadelta optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => Adadelta(parameters, LearningRate, Rho, Eps, WeightDecay, Maximize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adagrad.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adagrad.cs new file mode 100644 index 00000000..6e8ff601 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adagrad.cs @@ -0,0 +1,60 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates an Adagrad optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates an Adagrad optimizer.")] +public class Adagrad +{ + /// + /// The learning rate coefficient. + /// + [Description("The learning rate coefficient.")] + public double LearningRate { get; set; } = 0.01D; + + /// + /// The learning rate decay. + /// + [Description("The learning rate decay.")] + public double LearningRateDecay { get; set; } = 0D; + + /// + /// The weight decay coefficient, which adds L2 regularization to the loss. + /// + [Description("The weight decay coefficient, which adds L2 regularization to the loss.")] + public double WeightDecay { get; set; } = 0D; + + /// + /// The initial value of the sum of squares of gradients. + /// + [Description("The initial value of the sum of squares of gradients.")] + public double InitialAccumulatorValue { get; set; } = 0D; + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-10D; + + /// + /// Creates an Adagrad optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => Adagrad(parameters, LearningRate, LearningRateDecay, WeightDecay, InitialAccumulatorValue, Eps)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adam.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adam.cs new file mode 100644 index 00000000..5e2327f7 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adam.cs @@ -0,0 +1,72 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates an Adam optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates an Adam optimizer.")] +public class Adam +{ + /// + /// The learning rate coefficient. + /// + [Description("The learning rate coefficient.")] + public double LearningRate { get; set; } = 0.001D; + + /// + /// The beta coefficient for computing the running average of the gradient. + /// + [Description("The beta coefficient for computing the running average of the gradient.")] + public double BetaGradient { get; set; } = 0.9D; + + /// + /// The beta coefficient for computing the running average of the squared gradient. + /// + [Description("The beta coefficient for computing the running average of the squared gradient.")] + public double BetaSquaredGradient { get; set; } = 0.999D; + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-08D; + + /// + /// The weight decay coefficient, which adds L2 regularization to the loss. + /// + [Description("The weight decay coefficient, which adds L2 regularization to the loss.")] + public double WeightDecay { get; set; } = 0D; + + /// + /// If set to true, uses the AMSGrad variant of this algorithm. + /// + [Description("If set to true, uses the AMSGrad variant of this algorithm.")] + public bool Amsgrad { get; set; } = false; + + /// + /// If set to true, performs maximization instead of minimization of the params based on the objective. + /// + [Description("If set to true, performs maximization instead of minimization of the params based on the objective.")] + public bool Maximize { get; set; } = false; + + /// + /// Creates an Adam optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => Adam(parameters, LearningRate, BetaGradient, BetaSquaredGradient, Eps, WeightDecay, Amsgrad, Maximize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/AdamW.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/AdamW.cs new file mode 100644 index 00000000..044ba766 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/AdamW.cs @@ -0,0 +1,72 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates an AdamW optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates an AdamW optimizer.")] +public class AdamW +{ + /// + /// The learning rate. + /// + [Description("The learning rate.")] + public double LearningRate { get; set; } = 0.001D; + + /// + /// The beta coefficient for computing the running average of the gradient. + /// + [Description("The beta coefficient for computing the running average of the gradient.")] + public double BetaGradient { get; set; } = 0.9D; + + /// + /// The beta coefficient for computing the running average of the squared gradient. + /// + [Description("The beta coefficient for computing the running average of the squared gradient.")] + public double BetaSquaredGradient { get; set; } = 0.999D; + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-08D; + + /// + /// The weight decay coefficient, which adds L2 regularization to the loss. + /// + [Description("The weight decay coefficient, which adds L2 regularization to the loss.")] + public double WeightDecay { get; set; } = 0D; + + /// + /// If set to true, uses the AMSGrad variant of this algorithm. + /// + [Description("If set to true, uses the AMSGrad variant of this algorithm.")] + public bool Amsgrad { get; set; } = false; + + /// + /// If set to true, performs maximization instead of minimization of the params based on the objective. + /// + [Description("If set to true, performs maximization instead of minimization of the params based on the objective.")] + public bool Maximize { get; set; } = false; + + /// + /// Creates an AdamW optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => AdamW(parameters, LearningRate, BetaGradient, BetaSquaredGradient, Eps, WeightDecay, Amsgrad, Maximize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adamax.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adamax.cs new file mode 100644 index 00000000..2b1726e1 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Adamax.cs @@ -0,0 +1,60 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates an Adamax optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates an Adamax optimizer.")] +public class Adamax +{ + /// + /// The learning rate coefficient. + /// + [Description("The learning rate coefficient.")] + public double LearningRate { get; set; } = 0.002D; + + /// + /// The beta coefficient for computing the running average of the gradient. + /// + [Description("The beta coefficient for computing the running average of the gradient.")] + public double BetaGradient { get; set; } = 0.9D; + + /// + /// The beta coefficient for computing the running average of the squared gradient. + /// + [Description("The beta coefficient for computing the running average of the squared gradient.")] + public double BetaSquaredGradient { get; set; } = 0.999D; + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-08D; + + /// + /// The weight decay coefficient, which adds L2 regularization to the loss. + /// + [Description("The weight decay coefficient, which adds L2 regularization to the loss.")] + public double WeightDecay { get; set; } = 0D; + + /// + /// Creates an Adamax optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => Adamax(parameters, LearningRate, BetaGradient, BetaSquaredGradient, Eps, WeightDecay)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/AveragedStochasticGradientDescent.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/AveragedStochasticGradientDescent.cs new file mode 100644 index 00000000..6719f9f6 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/AveragedStochasticGradientDescent.cs @@ -0,0 +1,67 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates an averaged stochastic gradient descent (ASGD) optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates an averaged stochastic gradient descent (ASGD) optimizer.")] +[DisplayName("ASGD")] +public class AveragedStochasticGradientDescent +{ + /// + /// The learning rate. + /// + [Description("The learning rate.")] + public double LearningRate { get; set; } = 0.001D; + + /// + /// The lambda parameter, which determines the decay term. + /// + [Description("The lambda parameter, which determines the decay term.")] + public double Lambda { get; set; } = 0.0001D; + + /// + /// The alpha parameter, which controls the power law decay. + /// + [Description("The alpha parameter, which controls the power law decay.")] + public double Alpha { get; set; } = 0.75D; + + /// + /// The point when averaging starts. + /// + [Description("The point when averaging starts.")] + public double T0 { get; set; } = 1000000D; + + /// + /// The weight decay coefficient, which adds L2 regularization to the loss. + /// + [Description("The weight decay coefficient, which adds L2 regularization to the loss.")] + public double WeightDecay { get; set; } = 0D; + + /// + /// If set to true, performs maximization instead of minimization of the params based on the objective. + /// + [Description("If set to true, performs maximization instead of minimization of the params based on the objective.")] + public bool Maximize { get; set; } = false; + + /// + /// Creates an ASGD optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => ASGD(parameters, LearningRate, Lambda, Alpha, T0, WeightDecay, Maximize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Lbfgs.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Lbfgs.cs new file mode 100644 index 00000000..4446615c --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/Lbfgs.cs @@ -0,0 +1,67 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates a limited-memory Broyden-Fletcher-Goldfarb-Shanno (LBFGS) optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates a limited-memory Broyden-Fletcher-Goldfarb-Shanno (LBFGS) optimizer.")] +[DisplayName("LBFGS")] +public class Lbfgs +{ + /// + /// The learning rate. + /// + [Description("The learning rate.")] + public double LearningRate { get; set; } = 0.01D; + + /// + /// The maximum number of iterations per optimization step. + /// + [Description("The maximum number of iterations per optimization step.")] + public long MaxIter { get; set; } = 20; + + /// + /// The maximum number of function evaluations per optimization step. + /// + [Description("The maximum number of function evaluations per optimization step.")] + public long? MaxEval { get; set; } = null; + + /// + /// The termination criterion that determines when to stop optimizing based on first-order optimality. + /// + [Description("The termination criterion that determines when to stop optimizing based on first-order optimality.")] + public double ToleranceGrad { get; set; } = 1E-05D; + + /// + /// The termination criterion that determines when to stop optimizing based on function value or parameter changes. + /// + [Description("The termination criterion that determines when to stop optimizing based on function value or parameter changes.")] + public double ToleranceChange { get; set; } = 1E-09D; + + /// + /// The update history size. + /// + [Description("The update history size.")] + public long HistorySize { get; set; } = 100; + + /// + /// Creates an LBFGS optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => LBFGS(parameters, LearningRate, MaxIter, MaxEval, ToleranceGrad, ToleranceChange, HistorySize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/ResilientBackpropagation.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/ResilientBackpropagation.cs new file mode 100644 index 00000000..4584fe9d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/ResilientBackpropagation.cs @@ -0,0 +1,67 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates a resilient backpropagation (Rprop) optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates a resilient backpropagation (Rprop) optimizer.")] +[DisplayName("Rprop")] +public class ResilientBackpropagation +{ + /// + /// The learning rate. + /// + [Description("The learning rate.")] + public double LearningRate { get; set; } = 0.01D; + + /// + /// The eta (-) parameter, which serves as a multiplicative factor to decrease the step size. + /// + [Description("The eta (-) parameter, which serves as a multiplicative factor to decrease the step size.")] + public double EtaMinus { get; set; } = 0.5D; + + /// + /// The eta (+) parameter, which serves as a multiplicative factor to increase the step size. + /// + [Description("The eta (+) parameter, which serves as a multiplicative factor to increase the step size.")] + public double EtaPlus { get; set; } = 1.2D; + + /// + /// The minimum allowed step size. + /// + [Description("The minimum allowed step size.")] + public double MinStep { get; set; } = 1E-06D; + + /// + /// The maximum allowed step size. + /// + [Description("The maximum allowed step size.")] + public double MaxStep { get; set; } = 50D; + + /// + /// If set to true, performs maximization instead of minimization of the params based on the objective. + /// + [Description("If set to true, performs maximization instead of minimization of the params based on the objective.")] + public bool Maximize { get; set; } = false; + + /// + /// Creates an Rprop optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => Rprop(parameters, LearningRate, EtaMinus, EtaPlus, MinStep, MaxStep, Maximize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/RootMeanSquarePropagation.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/RootMeanSquarePropagation.cs new file mode 100644 index 00000000..70681e89 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/RootMeanSquarePropagation.cs @@ -0,0 +1,73 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates a root mean square propagation (RMSProp) optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates a root mean square propagation (RMSProp) optimizer.")] +[DisplayName("RMSProp")] +public class RootMeanSquarePropagation +{ + /// + /// The learning rate. + /// + [Description("The learning rate.")] + public double LearningRate { get; set; } = 0.01D; + + /// + /// The alpha parameter, which acts as a smoothing constant. + /// + [Description("The alpha parameter, which acts as a smoothing constant.")] + public double Alpha { get; set; } = 0.99D; + + /// + /// The value added to the denominator for numerical stability. + /// + [Description("The value added to the denominator for numerical stability.")] + public double Eps { get; set; } = 1E-08D; + + /// + /// The weight decay coefficient, which adds L2 regularization to the loss. + /// + [Description("The weight decay coefficient, which adds L2 regularization to the loss.")] + public double WeightDecay { get; set; } = 0D; + + /// + /// The momentum factor, which accelerates learning in the relevant direction. + /// + [Description("The momentum factor, which accelerates learning in the relevant direction.")] + public double Momentum { get; set; } = 0D; + + /// + /// If set to true, the gradient is normalized by an estimation of its variance. + /// + [Description("If set to true, the gradient is normalized by an estimation of its variance.")] + public bool Centered { get; set; } = false; + + /// + /// If set to true, performs maximization instead of minimization of the params based on the objective. + /// + [Description("If set to true, performs maximization instead of minimization of the params based on the objective.")] + public bool Maximize { get; set; } = false; + + /// + /// Creates an RMSProp optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => RMSProp(parameters, LearningRate, Alpha, Eps, WeightDecay, Momentum, Centered, Maximize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Optimizer/StochasticGradientDescent.cs b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/StochasticGradientDescent.cs new file mode 100644 index 00000000..1416d244 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Optimizer/StochasticGradientDescent.cs @@ -0,0 +1,67 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.optim; + +namespace Bonsai.ML.Torch.NeuralNets.Optimizer; + +/// +/// Represents an operator that creates a stochastic gradient descent (SGD) optimizer. +/// +/// +/// See for more information. +/// +[Description("Creates a stochastic gradient descent (SGD) optimizer.")] +[DisplayName("SGD")] +public class StochasticGradientDescent +{ + /// + /// The learning rate. + /// + [Description("The learning rate.")] + public double LearningRate { get; set; } + + /// + /// The momentum factor. + /// + [Description("The momentum factor.")] + public double Momentum { get; set; } = 0D; + + /// + /// The dampening for momentum. + /// + [Description("The dampening for momentum.")] + public double Dampening { get; set; } = 0D; + + /// + /// The weight decay coefficient, which adds L2 regularization to the loss. + /// + [Description("The weight decay coefficient, which adds L2 regularization to the loss.")] + public double WeightDecay { get; set; } = 0D; + + /// + /// If set to true, enables Nesterov momentum when momentum is non-zero. + /// + [Description("If set to true, enables Nesterov momentum when momentum is non-zero.")] + public bool Nesterov { get; set; } = false; + + /// + /// If set to true, performs maximization instead of minimization of the params based on the objective. + /// + [Description("If set to true, performs maximization instead of minimization of the params based on the objective.")] + public bool Maximize { get; set; } = false; + + /// + /// Creates an SGD optimizer from the input parameter collection. + /// + /// + /// + /// + public IObservable Process(IObservable source) where T : IEnumerable + { + return source.Select(parameters => SGD(parameters, LearningRate, Momentum, Dampening, WeightDecay, Nesterov, Maximize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/OptimizerBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/OptimizerBuilder.cs new file mode 100644 index 00000000..59498d3d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/OptimizerBuilder.cs @@ -0,0 +1,50 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates an optimizer. +/// +[XmlInclude(typeof(Optimizer.Adadelta))] +[XmlInclude(typeof(Optimizer.Adagrad))] +[XmlInclude(typeof(Optimizer.Adam))] +[XmlInclude(typeof(Optimizer.Adamax))] +[XmlInclude(typeof(Optimizer.AdamW))] +[XmlInclude(typeof(Optimizer.AveragedStochasticGradientDescent))] +[XmlInclude(typeof(Optimizer.Lbfgs))] +[XmlInclude(typeof(Optimizer.ResilientBackpropagation))] +[XmlInclude(typeof(Optimizer.RootMeanSquarePropagation))] +[XmlInclude(typeof(Optimizer.StochasticGradientDescent))] +[DefaultProperty(nameof(OptimizerModule))] +[Combinator] +[Description("Creates an optimizer.")] +[WorkflowElementCategory(ElementCategory.Transform)] +public class OptimizerBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "Optimizer"; + + /// + /// Initializes a new instance of the class. + /// + public OptimizerBuilder() + { + Module = new Optimizer.Adam(); + } + + /// + /// Gets or sets the specific optimizer to create. + /// + [DesignOnly(true)] + [DisplayName("Optimizer")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific optimizer to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object OptimizerModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ConstantPad1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ConstantPad1d.cs new file mode 100644 index 00000000..104ce716 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ConstantPad1d.cs @@ -0,0 +1,50 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 1D constant padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D constant padding module.")] +public class ConstantPad1d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) PaddingSize { get; set; } + + /// + /// The value to pad with. + /// + [Description("The value to pad with.")] + public double Value { get; set; } + + /// + /// Creates a 1D constant padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ConstantPad1d(PaddingSize, Value)); + } + + /// + /// Creates a 1D constant padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ConstantPad1d(PaddingSize, Value)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ConstantPad2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ConstantPad2d.cs new file mode 100644 index 00000000..47246637 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ConstantPad2d.cs @@ -0,0 +1,50 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 2D constant padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D constant padding module.")] +public class ConstantPad2d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long, long) PaddingSize { get; set; } + + /// + /// The value to pad with. + /// + [Description("The value to pad with.")] + public double Value { get; set; } + + /// + /// Creates a 2D constant padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ConstantPad2d(PaddingSize, Value)); + } + + /// + /// Creates a 2D constant padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ConstantPad2d(PaddingSize, Value)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ConstantPad3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ConstantPad3d.cs new file mode 100644 index 00000000..d22557ec --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ConstantPad3d.cs @@ -0,0 +1,50 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 3D constant padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D constant padding module.")] +public class ConstantPad3d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long, long, long, long) PaddingSize { get; set; } + + /// + /// The value to pad with. + /// + [Description("The value to pad with.")] + public double Value { get; set; } + + /// + /// Creates a 3D constant padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ConstantPad3d(PaddingSize, Value)); + } + + /// + /// Creates a 3D constant padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ConstantPad3d(PaddingSize, Value)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ReflectionPad1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReflectionPad1d.cs new file mode 100644 index 00000000..225087fd --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReflectionPad1d.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 1D reflection padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D reflection padding module.")] +public class ReflectionPad1d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) PaddingSize { get; set; } + + /// + /// Creates a 1D reflection padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ReflectionPad1d(PaddingSize)); + } + + /// + /// Creates a 1D reflection padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ReflectionPad1d(PaddingSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ReflectionPad2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReflectionPad2d.cs new file mode 100644 index 00000000..da3f6a14 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReflectionPad2d.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 2D reflection padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D reflection padding module.")] +public class ReflectionPad2d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long, long) PaddingSize { get; set; } + + /// + /// Creates a 2D reflection padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ReflectionPad2d(PaddingSize)); + } + + /// + /// Creates a 2D reflection padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ReflectionPad2d(PaddingSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ReflectionPad3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReflectionPad3d.cs new file mode 100644 index 00000000..c0677d1d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReflectionPad3d.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 3D reflection padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D reflection padding module.")] +public class ReflectionPad3d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long, long, long, long) PaddingSize { get; set; } + + /// + /// Creates a 3D reflection padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ReflectionPad3d(PaddingSize)); + } + + /// + /// Creates a 3D reflection padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ReflectionPad3d(PaddingSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ReplicationPad1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReplicationPad1d.cs new file mode 100644 index 00000000..fe530720 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReplicationPad1d.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 1D replication padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D replication padding module.")] +public class ReplicationPad1d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) PaddingSize { get; set; } + + /// + /// Creates a 1D replication padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ReplicationPad1d(PaddingSize)); + } + + /// + /// Creates a 1D replication padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ReplicationPad1d(PaddingSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ReplicationPad2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReplicationPad2d.cs new file mode 100644 index 00000000..d434cb2c --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReplicationPad2d.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 2D replication padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D replication padding module.")] +public class ReplicationPad2d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long, long) PaddingSize { get; set; } + + /// + /// Creates a 2D replication padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ReplicationPad2d(PaddingSize)); + } + + /// + /// Creates a 2D replication padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ReplicationPad2d(PaddingSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ReplicationPad3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReplicationPad3d.cs new file mode 100644 index 00000000..9f4f9ef5 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ReplicationPad3d.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 3D replication padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D replication padding module.")] +public class ReplicationPad3d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long, long, long, long) PaddingSize { get; set; } + + /// + /// Creates a 3D replication padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ReplicationPad3d(PaddingSize)); + } + + /// + /// Creates a 3D replication padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ReplicationPad3d(PaddingSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Padding/ZeroPad2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Padding/ZeroPad2d.cs new file mode 100644 index 00000000..fb1f46cc --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Padding/ZeroPad2d.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Padding; + +/// +/// Represents an operator that creates a 2D zero padding module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D zero padding module.")] +public class ZeroPad2d +{ + /// + /// The size of the padding. + /// + [Description("The size of the padding.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long, long) PaddingSize { get; set; } + + /// + /// Creates a 2D zero padding module. + /// + /// + public IObservable Process() + { + return Observable.Return(ZeroPad2d(PaddingSize)); + } + + /// + /// Creates a 2D zero padding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ZeroPad2d(PaddingSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/PaddingModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/PaddingModuleBuilder.cs new file mode 100644 index 00000000..b6b43f15 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/PaddingModuleBuilder.cs @@ -0,0 +1,50 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a padding module. +/// +[XmlInclude(typeof(Padding.ConstantPad1d))] +[XmlInclude(typeof(Padding.ConstantPad2d))] +[XmlInclude(typeof(Padding.ConstantPad3d))] +[XmlInclude(typeof(Padding.ReflectionPad1d))] +[XmlInclude(typeof(Padding.ReflectionPad2d))] +[XmlInclude(typeof(Padding.ReflectionPad3d))] +[XmlInclude(typeof(Padding.ReplicationPad1d))] +[XmlInclude(typeof(Padding.ReplicationPad2d))] +[XmlInclude(typeof(Padding.ReplicationPad3d))] +[XmlInclude(typeof(Padding.ZeroPad2d))] +[DefaultProperty(nameof(PaddingModule))] +[Combinator] +[Description("Creates a padding module.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class PaddingModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "PaddingModule"; + + /// + /// Initializes a new instance of the class. + /// + public PaddingModuleBuilder() + { + Module = new Padding.ConstantPad1d(); + } + + /// + /// Gets or sets the specific padding module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific padding module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object PaddingModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveAvgPool1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveAvgPool1d.cs new file mode 100644 index 00000000..4f1b0b88 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveAvgPool1d.cs @@ -0,0 +1,46 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 1D adaptive average pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D adaptive average pooling module.")] +public class AdaptiveAvgPool1d +{ + /// + /// The output size. + /// + [Description("The output size.")] + public long OutputSize { get; set; } + + /// + /// Creates an AdaptiveAvgPool1d module. + /// + public IObservable Process() + { + return Observable.Return(AdaptiveAvgPool1d(OutputSize)); + } + + /// + /// Creates an AdaptiveAvgPool1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AdaptiveAvgPool1d(OutputSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveAvgPool2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveAvgPool2d.cs new file mode 100644 index 00000000..361c7b2c --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveAvgPool2d.cs @@ -0,0 +1,47 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 2D adaptive average pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D adaptive average pooling module.")] +public class AdaptiveAvgPool2d +{ + /// + /// The output size. + /// + [Description("The output size.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) OutputSize { get; set; } + + /// + /// Creates an AdaptiveAvgPool2d module. + /// + public IObservable Process() + { + return Observable.Return(AdaptiveAvgPool2d(OutputSize)); + } + + /// + /// Creates an AdaptiveAvgPool2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AdaptiveAvgPool2d(OutputSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveAvgPool3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveAvgPool3d.cs new file mode 100644 index 00000000..ac088fb7 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveAvgPool3d.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 3D adaptive average pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D adaptive average pooling module.")] +public class AdaptiveAvgPool3d +{ + /// + /// The output size. + /// + [Description("The output size.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long) OutputSize { get; set; } + + /// + /// Creates an AdaptiveAvgPool3d module. + /// + public IObservable Process() + { + return Observable.Return(AdaptiveAvgPool3d(OutputSize)); + } + + /// + /// Creates an AdaptiveAvgPool3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AdaptiveAvgPool3d(OutputSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveMaxPool1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveMaxPool1d.cs new file mode 100644 index 00000000..473d056f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveMaxPool1d.cs @@ -0,0 +1,42 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 1D adaptive max pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D adaptive max pooling module.")] +public class AdaptiveMaxPool1d +{ + /// + /// The output size. + /// + [Description("The output size.")] + public long OutputSize { get; set; } + + /// + /// Creates an AdaptiveMaxPool1d module. + /// + public IObservable Process() + { + return Observable.Return(AdaptiveMaxPool1d(OutputSize)); + } + + /// + /// Creates an AdaptiveMaxPool1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AdaptiveMaxPool1d(OutputSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveMaxPool2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveMaxPool2d.cs new file mode 100644 index 00000000..93dfa349 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveMaxPool2d.cs @@ -0,0 +1,47 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 2D adaptive max pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D adaptive max pooling module.")] +public class AdaptiveMaxPool2d +{ + /// + /// The output size. + /// + [Description("The output size.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public long[] OutputSize { get; set; } + + /// + /// Creates an AdaptiveMaxPool2d module. + /// + public IObservable Process() + { + return Observable.Return(AdaptiveMaxPool2d(OutputSize)); + } + + /// + /// Creates an AdaptiveMaxPool2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AdaptiveMaxPool2d(OutputSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveMaxPool3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveMaxPool3d.cs new file mode 100644 index 00000000..e30547bb --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AdaptiveMaxPool3d.cs @@ -0,0 +1,47 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 3D adaptive max pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D adaptive max pooling module.")] +public class AdaptiveMaxPool3d +{ + /// + /// The output size. + /// + [Description("The output size.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public long[] OutputSize { get; set; } + + /// + /// Creates an AdaptiveMaxPool3d module. + /// + public IObservable Process() + { + return Observable.Return(AdaptiveMaxPool3d(OutputSize)); + } + + /// + /// Creates an AdaptiveMaxPool3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AdaptiveMaxPool3d(OutputSize)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/AvgPool1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AvgPool1d.cs new file mode 100644 index 00000000..0c765dba --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AvgPool1d.cs @@ -0,0 +1,71 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 1D average pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D average pooling module.")] +public class AvgPool1d +{ + /// + /// The size of the window. + /// + [Description("The size of the window.")] + public long KernelSize { get; set; } + + /// + /// The stride of the window. + /// + [Description("The stride of the window.")] + public long? Stride { get; set; } = null; + + /// + /// The implicit zero padding to be added on both sides. + /// + [Description("The implicit zero padding to be added on both sides.")] + public long Padding { get; set; } = 0; + + /// + /// If set to true, will use ceil instead of floor to compute the output shape. + /// + [Description("If set to true, will use ceil instead of floor to compute the output shape.")] + public bool CeilMode { get; set; } = false; + + /// + /// If set to true, will include the zero-padding in the averaging calculation. + /// + [Description("If set to true, will include the zero-padding in the averaging calculation.")] + public bool CountIncludePad { get; set; } = true; + + /// + /// Creates an AvgPool1d module. + /// + /// + public IObservable Process() + { + return Observable.Return(AvgPool1d(KernelSize, Stride, Padding, CeilMode, CountIncludePad)); + } + + /// + /// Creates an AvgPool1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AvgPool1d(KernelSize, Stride, Padding, CeilMode, CountIncludePad)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/AvgPool2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AvgPool2d.cs new file mode 100644 index 00000000..845a726d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AvgPool2d.cs @@ -0,0 +1,80 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 2D average pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D average pooling module.")] +public class AvgPool2d +{ + /// + /// The size of the window. + /// + [Description("The size of the window.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) KernelSize { get; set; } + + /// + /// The stride of the window. + /// + [Description("The stride of the window.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Stride { get; set; } = null; + + /// + /// The implicit zero padding to be added on both sides. + /// + [Description("The implicit zero padding to be added on both sides.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Padding { get; set; } = null; + + /// + /// If set to true, will use ceil instead of floor to compute the output shape. + /// + [Description("If set to true, will use ceil instead of floor to compute the output shape.")] + public bool CeilMode { get; set; } = false; + + /// + /// If set to true, will include the zero-padding in the averaging calculation. + /// + [Description("If set to true, will include the zero-padding in the averaging calculation.")] + public bool CountIncludePad { get; set; } = true; + + /// + /// If specified, it will be used as divisor, otherwise size of the pooling region will be used. + /// + [Description("If specified, it will be used as divisor, otherwise size of the pooling region will be used.")] + public long? DivisorOverride { get; set; } = null; + + /// + /// Creates an AvgPool2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(AvgPool2d(KernelSize, Stride, Padding, CeilMode, CountIncludePad, DivisorOverride)); + } + + /// + /// Creates an AvgPool2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AvgPool2d(KernelSize, Stride, Padding, CeilMode, CountIncludePad, DivisorOverride)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/AvgPool3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AvgPool3d.cs new file mode 100644 index 00000000..f90d1fb3 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/AvgPool3d.cs @@ -0,0 +1,80 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 3D average pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D average pooling module.")] +public class AvgPool3d +{ + /// + /// The size of the window. + /// + [Description("The size of the window.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long) KernelSize { get; set; } + + /// + /// The stride of the window. + /// + [Description("The stride of the window.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Stride { get; set; } = null; + + /// + /// The implicit zero padding to be added on all three sides. + /// + [Description("The implicit zero padding to be added on all three sides.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Padding { get; set; } = null; + + /// + /// If set to true, will use ceil instead of floor to compute the output shape. + /// + [Description("If set to true, will use ceil instead of floor to compute the output shape.")] + public bool CeilMode { get; set; } = false; + + /// + /// If set to true, will include the zero-padding in the averaging calculation. + /// + [Description("If set to true, will include the zero-padding in the averaging calculation.")] + public bool CountIncludePad { get; set; } = true; + + /// + /// If specified, it will be used as divisor, otherwise size of the pooling region will be used. + /// + [Description("If specified, it will be used as divisor, otherwise size of the pooling region will be used.")] + public long? DivisorOverride { get; set; } = null; + + /// + /// Creates an AvgPool3d module. + /// + /// + public IObservable Process() + { + return Observable.Return(AvgPool3d(KernelSize, Stride, Padding, CeilMode, CountIncludePad, DivisorOverride)); + } + + /// + /// Creates an AvgPool3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => AvgPool3d(KernelSize, Stride, Padding, CeilMode, CountIncludePad, DivisorOverride)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/FractionalMaxPool2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/FractionalMaxPool2d.cs new file mode 100644 index 00000000..51670afe --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/FractionalMaxPool2d.cs @@ -0,0 +1,58 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 2D fractional max pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D fractional max pooling module.")] +public class FractionalMaxPool2d +{ + /// + /// The size of the window to take a max over. + /// + [Description("The size of the window to take a max over.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) KernelSize { get; set; } + + /// + /// The output size. + /// + [Description("The output size.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? OutputSize { get; set; } = null; + + /// + /// Can be used to specify the output size as a ratio of the input size. + /// + [Description("Can be used to specify the output size as a ratio of the input size.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? OutputRatio { get; set; } = null; + + /// + /// Creates a FractionalMaxPool2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(FractionalMaxPool2d(KernelSize, OutputSize, OutputRatio)); + } + + /// + /// Creates a FractionalMaxPool2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => FractionalMaxPool2d(KernelSize, OutputSize, OutputRatio)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/FractionalMaxPool3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/FractionalMaxPool3d.cs new file mode 100644 index 00000000..3b7bd131 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/FractionalMaxPool3d.cs @@ -0,0 +1,58 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 3D fractional max pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D fractional max pooling module.")] +public class FractionalMaxPool3d +{ + /// + /// The size of the window to take a max over. + /// + [Description("The size of the window to take a max over.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long) KernelSize { get; set; } + + /// + /// The output size. + /// + [Description("The output size.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? OutputSize { get; set; } = null; + + /// + /// Can be used to specify the output size as a ratio of the input size. + /// + [Description("Can be used to specify the output size as a ratio of the input size.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? OutputRatio { get; set; } = null; + + /// + /// Creates a FractionalMaxPool3d module. + /// + /// + public IObservable Process() + { + return Observable.Return(FractionalMaxPool3d(KernelSize, OutputSize, OutputRatio)); + } + + /// + /// Creates a FractionalMaxPool3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => FractionalMaxPool3d(KernelSize, OutputSize, OutputRatio)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/LPPool1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/LPPool1d.cs new file mode 100644 index 00000000..b32bc3e3 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/LPPool1d.cs @@ -0,0 +1,65 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 1D power average pooling (LPPool1d) module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D power average pooling (LPPool1d) module.")] +public class LPPool1d +{ + /// + /// The degree of the norm. + /// + [Description("The degree of the norm.")] + public double Norm { get; set; } + + /// + /// The size of the window. + /// + [Description("The size of the window.")] + public long KernelSize { get; set; } + + /// + /// The stride of the window. + /// + [Description("The stride of the window.")] + public long? Stride { get; set; } = null; + + /// + /// If set to true, will use ceil instead of floor to compute the output shape. + /// + [Description("If set to true, will use ceil instead of floor to compute the output shape.")] + public bool CeilMode { get; set; } = false; + + /// + /// Creates a LPPool1d module. + /// + /// + public IObservable Process() + { + return Observable.Return(LPPool1d(Norm, KernelSize, Stride, CeilMode)); + } + + /// + /// Creates a LPPool1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => LPPool1d(Norm, KernelSize, Stride, CeilMode)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/LPPool2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/LPPool2d.cs new file mode 100644 index 00000000..fc235998 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/LPPool2d.cs @@ -0,0 +1,67 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 2D power average pooling (LPPool2d) module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D power average pooling (LPPool2d) module.")] +public class LPPool2d +{ + /// + /// The degree of the norm. + /// + [Description("The degree of the norm.")] + public double Norm { get; set; } + + /// + /// The size of the window. + /// + [Description("The size of the window.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public long[] KernelSize { get; set; } + + /// + /// The stride of the window. + /// + [Description("The stride of the window.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public long[] Stride { get; set; } = null; + + /// + /// If set to true, will use ceil instead of floor to compute the output shape. + /// + [Description("If set to true, will use ceil instead of floor to compute the output shape.")] + public bool CeilMode { get; set; } = false; + + /// + /// Creates a LPPool2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(LPPool2d(Norm, KernelSize, Stride, CeilMode)); + } + + /// + /// Creates a LPPool2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => LPPool2d(Norm, KernelSize, Stride, CeilMode)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxPool1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxPool1d.cs new file mode 100644 index 00000000..c137e2ed --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxPool1d.cs @@ -0,0 +1,71 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 1D max pooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D max pooling module.")] +public class MaxPool1d +{ + /// + /// The size of the sliding window. + /// + [Description("The size of the sliding window.")] + public long KernelSize { get; set; } + + /// + /// The stride of the sliding window. + /// + [Description("The stride of the sliding window.")] + public long? Stride { get; set; } = null; + + /// + /// The implicit negative infinity padding to be added on both sides. + /// + [Description("The implicit negative infinity padding to be added on both sides.")] + public long? Padding { get; set; } = null; + + /// + /// The spacing between kernel elements. + /// + [Description("The spacing between kernel elements.")] + public long? Dilation { get; set; } = null; + + /// + /// If set to true, will use ceil instead of floor to compute the output shape. + /// + [Description("If set to true, will use ceil instead of floor to compute the output shape.")] + public bool CeilMode { get; set; } = false; + + /// + /// Creates a MaxPool1d module. + /// + /// + public IObservable Process() + { + return Observable.Return(MaxPool1d(KernelSize, Stride, Padding, Dilation, CeilMode)); + } + + /// + /// Creates a MaxPool1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MaxPool1d(KernelSize, Stride, Padding, Dilation, CeilMode)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxPool2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxPool2d.cs new file mode 100644 index 00000000..373e86c4 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxPool2d.cs @@ -0,0 +1,71 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 2D max pooling layer. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D max pooling layer.")] +public class MaxPool2d +{ + /// + /// The size of the window to take a max over. + /// + [Description("The size of the window to take a max over.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) KernelSize { get; set; } + + /// + /// The stride of the window. + /// + [Description("The stride of the window.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Stride { get; set; } = null; + + /// + /// The implicit negative infinity padding to be added on both sides. + /// + [Description("The implicit negative infinity padding to be added on both sides.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Padding { get; set; } = null; + + /// + /// The spacing between kernel elements. + /// + [Description("The spacing between kernel elements.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Dilation { get; set; } = null; + + /// + /// If set to true, will use ceil instead of floor to compute the output shape. + /// + [Description("If set to true, will use ceil instead of floor to compute the output shape.")] + public bool CeilMode { get; set; } = false; + + /// + /// Creates a MaxPool2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(MaxPool2d(KernelSize, Stride, Padding, Dilation, CeilMode)); + } + + /// + /// Creates a MaxPool2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MaxPool2d(KernelSize, Stride, Padding, Dilation, CeilMode)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxPool3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxPool3d.cs new file mode 100644 index 00000000..fb0796ab --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxPool3d.cs @@ -0,0 +1,71 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 3D max pooling layer. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D max pooling layer.")] +public class MaxPool3d +{ + /// + /// The size of the window to take a max over. + /// + [Description("The size of the window to take a max over.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long) KernelSize { get; set; } + + /// + /// The stride of the window. + /// + [Description("The stride of the window.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Stride { get; set; } = null; + + /// + /// The implicit negative infinity padding to be added on all three sides. + /// + [Description("The implicit negative infinity padding to be added on all three sides.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Padding { get; set; } = null; + + /// + /// The spacing between kernel elements. + /// + [Description("The spacing between kernel elements.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Dilation { get; set; } = null; + + /// + /// If set to true, will use ceil instead of floor to compute the output shape. + /// + [Description("If set to true, will use ceil instead of floor to compute the output shape.")] + public bool CeilMode { get; set; } = false; + + /// + /// Creates a MaxPool3d module. + /// + /// + public IObservable Process() + { + return Observable.Return(MaxPool3d(KernelSize, Stride, Padding, Dilation, CeilMode)); + } + + /// + /// Creates a MaxPool3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MaxPool3d(KernelSize, Stride, Padding, Dilation, CeilMode)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxUnpool1d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxUnpool1d.cs new file mode 100644 index 00000000..293a4eee --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxUnpool1d.cs @@ -0,0 +1,59 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 1D max unpooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 1D max unpooling module.")] +public class MaxUnpool1d +{ + /// + /// The size of the max pooling window. + /// + [Description("The size of the max pooling window.")] + public long KernelSize { get; set; } + + /// + /// The stride of the max pooling window. + /// + [Description("The stride of the max pooling window.")] + public long? Stride { get; set; } = null; + + /// + /// The padding that was added to the input. + /// + [Description("The padding that was added to the input.")] + public long? Padding { get; set; } = null; + + /// + /// Creates a MaxUnpool1d module. + /// + /// + public IObservable Process() + { + return Observable.Return(MaxUnpool1d(KernelSize, Stride, Padding)); + } + + /// + /// Creates a MaxUnpool1d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MaxUnpool1d(KernelSize, Stride, Padding)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxUnpool2d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxUnpool2d.cs new file mode 100644 index 00000000..8ff1eaad --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxUnpool2d.cs @@ -0,0 +1,62 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 2D max unpooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 2D max unpooling module.")] +public class MaxUnpool2d +{ + /// + /// The size of the max pooling window. + /// + [Description("The size of the max pooling window.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long) KernelSize { get; set; } + + /// + /// The stride of the max pooling window. + /// + [Description("The stride of the max pooling window.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Stride { get; set; } = null; + + /// + /// The padding that was added to the input. + /// + [Description("The padding that was added to the input.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long)? Padding { get; set; } = null; + + /// + /// Creates a MaxUnpool2d module. + /// + /// + public IObservable Process() + { + return Observable.Return(MaxUnpool2d(KernelSize, Stride, Padding)); + } + + /// + /// Creates a MaxUnpool2d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MaxUnpool2d(KernelSize, Stride, Padding)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxUnpool3d.cs b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxUnpool3d.cs new file mode 100644 index 00000000..5a63d783 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Pooling/MaxUnpool3d.cs @@ -0,0 +1,62 @@ +using System; +using System.ComponentModel; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using TorchSharp.Modules; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Pooling; + +/// +/// Represents an operator that creates a 3D max unpooling module. +/// +/// +/// See for more information. +/// +[Description("Creates a 3D max unpooling module.")] +public class MaxUnpool3d +{ + /// + /// The size of the max pooling window. + /// + [Description("The size of the max pooling window.")] + [TypeConverter(typeof(ValueTupleConverter))] + public (long, long, long) KernelSize { get; set; } + + /// + /// The stride of the max pooling window. + /// + [Description("The stride of the max pooling window.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Stride { get; set; } = null; + + /// + /// The padding that was added to the input. + /// + [Description("The padding that was added to the input.")] + [TypeConverter(typeof(NullableValueTupleConverter))] + public (long, long, long)? Padding { get; set; } = null; + + /// + /// Creates a MaxUnpool3d module. + /// + /// + public IObservable Process() + { + return Observable.Return(MaxUnpool3d(KernelSize, Stride, Padding)); + } + + /// + /// Creates a MaxUnpool3d module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => MaxUnpool3d(KernelSize, Stride, Padding)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/PoolingModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/PoolingModuleBuilder.cs new file mode 100644 index 00000000..bd0e9d65 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/PoolingModuleBuilder.cs @@ -0,0 +1,59 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a module for pooling operations. +/// +[XmlInclude(typeof(Pooling.AdaptiveAvgPool1d))] +[XmlInclude(typeof(Pooling.AdaptiveAvgPool2d))] +[XmlInclude(typeof(Pooling.AdaptiveAvgPool3d))] +[XmlInclude(typeof(Pooling.AdaptiveMaxPool1d))] +[XmlInclude(typeof(Pooling.AdaptiveMaxPool2d))] +[XmlInclude(typeof(Pooling.AdaptiveMaxPool3d))] +[XmlInclude(typeof(Pooling.AvgPool1d))] +[XmlInclude(typeof(Pooling.AvgPool2d))] +[XmlInclude(typeof(Pooling.AvgPool3d))] +[XmlInclude(typeof(Pooling.FractionalMaxPool2d))] +[XmlInclude(typeof(Pooling.FractionalMaxPool3d))] +[XmlInclude(typeof(Pooling.LPPool1d))] +[XmlInclude(typeof(Pooling.LPPool2d))] +[XmlInclude(typeof(Pooling.MaxPool1d))] +[XmlInclude(typeof(Pooling.MaxPool2d))] +[XmlInclude(typeof(Pooling.MaxPool3d))] +[XmlInclude(typeof(Pooling.MaxUnpool1d))] +[XmlInclude(typeof(Pooling.MaxUnpool2d))] +[XmlInclude(typeof(Pooling.MaxUnpool3d))] +[DefaultProperty(nameof(PoolingModule))] +[Combinator] +[Description("Creates a module for pooling operations.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class PoolingModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "PoolingModule"; + + /// + /// Initializes a new instance of the class. + /// + public PoolingModuleBuilder() + { + Module = new Pooling.AdaptiveAvgPool1d(); + } + + /// + /// Gets or sets the specific pooling module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific pooling module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object PoolingModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Recurrent/GatedRecurrentUnit.cs b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/GatedRecurrentUnit.cs new file mode 100644 index 00000000..8aaeee87 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/GatedRecurrentUnit.cs @@ -0,0 +1,93 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Recurrent; + +/// +/// Represents an operator that creates a multi-layer gated recurrent unit (GRU) module. +/// +/// +/// See for more information. +/// +[Description("Creates a multi-layer gated recurrent unit (GRU) module.")] +public class GatedRecurrentUnit +{ + /// + /// The number of expected features in the input. + /// + [Description("The number of expected features in the input.")] + public long InputSize { get; set; } + + /// + /// The number of features in the hidden state. + /// + [Description("The number of features in the hidden state.")] + public long HiddenSize { get; set; } + + /// + /// The number of recurrent layers. + /// + [Description("The number of recurrent layers.")] + public long NumLayers { get; set; } = 1; + + /// + /// If set to false, then the layer will not use bias weights. + /// + [Description("If set to false, then the layer will not use bias weights.")] + public bool Bias { get; set; } = true; + + /// + /// If set to true, then the input and output tensors are provided as (batch, seq, feature) instead of (seq, batch, feature). + /// + [Description("If set to true, then the input and output tensors are provided as (batch, seq, feature) instead of (seq, batch, feature).")] + public bool BatchFirst { get; set; } = false; + + /// + /// If non-zero, this introduces a Dropout layer with the provided dropout probability on the outputs of each GRU layer except the last layer. + /// + [Description("If non-zero, this introduces a Dropout layer with the provided dropout probability on the outputs of each GRU layer except the last layer.")] + public double Dropout { get; set; } = 0D; + + /// + /// The set to true, becomes a bidirectional GRU. + /// + [Description("If set to true, becomes a bidirectional GRU.")] + public bool Bidirectional { get; set; } = false; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a GRU module. + /// + /// + public IObservable Process() + { + return Observable.Return(GRU(InputSize, HiddenSize, NumLayers, Bias, BatchFirst, Dropout, Bidirectional, Device, Type)); + } + + /// + /// Creates a GRU module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => GRU(InputSize, HiddenSize, NumLayers, Bias, BatchFirst, Dropout, Bidirectional, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Recurrent/GatedRecurrentUnitCell.cs b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/GatedRecurrentUnitCell.cs new file mode 100644 index 00000000..846a5325 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/GatedRecurrentUnitCell.cs @@ -0,0 +1,69 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Recurrent; + +/// +/// Represents an operator that creates a gated recurrent unit (GRU) cell. +/// +/// +/// See for more information. +/// +[Description("Creates a gated recurrent unit (GRU) cell module.")] +public class GatedRecurrentUnitCell +{ + /// + /// The number of expected features in the input. + /// + [Description("The number of expected features in the input.")] + public long InputSize { get; set; } + + /// + /// The number of features in the hidden state. + /// + [Description("The number of features in the hidden state.")] + public long HiddenSize { get; set; } + + /// + /// If set to false, then the layer will not use bias weights. + /// + [Description("If set to false, then the layer will not use bias weights.")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates a GRUCell module. + /// + /// + public IObservable Process() + { + return Observable.Return(GRUCell(InputSize, HiddenSize, Bias, Device, Type)); + } + + /// + /// Creates a GRUCell module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => GRUCell(InputSize, HiddenSize, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Recurrent/LongShortTermMemory.cs b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/LongShortTermMemory.cs new file mode 100644 index 00000000..43166c0c --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/LongShortTermMemory.cs @@ -0,0 +1,93 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Recurrent; + +/// +/// Represents an operator that creates a multi-layer long short-term memory (LSTM) module. +/// +/// +/// See for more information. +/// +[Description("Creates a multi-layer long short-term memory (LSTM) module.")] +public class LongShortTermMemory +{ + /// + /// The number of expected features in the input. + /// + [Description("The number of expected features in the input.")] + public long InputSize { get; set; } + + /// + /// The number of features in the hidden state. + /// + [Description("The number of features in the hidden state.")] + public long HiddenSize { get; set; } + + /// + /// The number of recurrent layers. + /// + [Description("The number of recurrent layers.")] + public long NumLayers { get; set; } = 1; + + /// + /// If set to false, then the layer will not use bias weights. + /// + [Description("If set to false, then the layer will not use bias weights.")] + public bool Bias { get; set; } = true; + + /// + /// If set to true, then the input and output tensors are provided as (batch, seq, feature) instead of (seq, batch, feature). + /// + [Description("If set to true, then the input and output tensors are provided as (batch, seq, feature) instead of (seq, batch, feature).")] + public bool BatchFirst { get; set; } = false; + + /// + /// If non-zero, this introduces a Dropout layer with the provided dropout probability on the outputs of each LSTM layer except the last layer. + /// + [Description("If non-zero, this introduces a Dropout layer with the provided dropout probability on the outputs of each LSTM layer except the last layer.")] + public double Dropout { get; set; } = 0D; + + /// + /// If set to true, becomes a bidirectional LSTM. + /// + [Description("If set to true, becomes a bidirectional LSTM.")] + public bool Bidirectional { get; set; } = false; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates an LSTM module. + /// + /// + public IObservable Process() + { + return Observable.Return(LSTM(InputSize, HiddenSize, NumLayers, Bias, BatchFirst, Dropout, Bidirectional, Device, Type)); + } + + /// + /// Creates an LSTM module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => LSTM(InputSize, HiddenSize, NumLayers, Bias, BatchFirst, Dropout, Bidirectional, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Recurrent/LongShortTermMemoryCell.cs b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/LongShortTermMemoryCell.cs new file mode 100644 index 00000000..0ce7ce24 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/LongShortTermMemoryCell.cs @@ -0,0 +1,69 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Recurrent; + +/// +/// Represents an operator that creates a long short-term memory (LSTM) cell. +/// +/// +/// See for more information. +/// +[Description("Creates a long short-term memory (LSTM) cell.")] +public class LongShortTermMemoryCell +{ + /// + /// The number of expected features in the input. + /// + [Description("The number of expected features in the input.")] + public long InputSize { get; set; } + + /// + /// The number of features in the hidden state. + /// + [Description("The number of features in the hidden state.")] + public long HiddenSize { get; set; } + + /// + /// If set to false, then the layer will not use bias weights. + /// + [Description("If set to false, then the layer will not use bias weights.")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates an LSTMCell module. + /// + /// + public IObservable Process() + { + return Observable.Return(LSTMCell(InputSize, HiddenSize, Bias, Device, Type)); + } + + /// + /// Creates an LSTMCell module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => LSTMCell(InputSize, HiddenSize, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Recurrent/RecurrentNeuralNetwork.cs b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/RecurrentNeuralNetwork.cs new file mode 100644 index 00000000..e73c5ee0 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/RecurrentNeuralNetwork.cs @@ -0,0 +1,99 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Recurrent; + +/// +/// Represents an operator that creates a multi-layer recurrent neural network (RNN) module. +/// +/// +/// See for more information. +/// +[Description("Creates a multi-layer recurrent neural network (RNN) module.")] +public class RecurrentNeuralNetwork +{ + /// + /// The number of expected features in the input. + /// + [Description("The number of expected features in the input.")] + public long InputSize { get; set; } + + /// + /// The number of features in the hidden state. + /// + [Description("The number of features in the hidden state.")] + public long HiddenSize { get; set; } + + /// + /// The number of recurrent layers. + /// + [Description("The number of recurrent layers.")] + public long NumLayers { get; set; } = 1; + + /// + /// The type of nonlinearity to use. + /// + [Description("The type of nonlinearity to use.")] + public NonLinearities NonLinearity { get; set; } = NonLinearities.Tanh; + + /// + /// If set to false, then the layer will not use bias weights. + /// + [Description("If set to false, then the layer will not use bias weights.")] + public bool Bias { get; set; } = true; + + /// + /// If set to true, then the input and output tensors are provided as (batch, seq, feature) instead of (seq, batch, feature). + /// + [Description("If set to true, then the input and output tensors are provided as (batch, seq, feature) instead of (seq, batch, feature).")] + public bool BatchFirst { get; set; } = false; + + /// + /// If non-zero, this introduces a Dropout layer with the provided dropout probability on the outputs of each RNN layer except the last layer. + /// + [Description("If non-zero, this introduces a Dropout layer with the provided dropout probability on the outputs of each RNN layer except the last layer.")] + public double Dropout { get; set; } = 0D; + + /// + /// If set to true, becomes a bidirectional RNN. + /// + [Description("If set to true, becomes a bidirectional RNN.")] + public bool Bidirectional { get; set; } = false; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates an RNN module. + /// + /// + public IObservable Process() + { + return Observable.Return(RNN(InputSize, HiddenSize, NumLayers, NonLinearity, Bias, BatchFirst, Dropout, Bidirectional, Device, Type)); + } + + /// + /// Creates an RNN module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => RNN(InputSize, HiddenSize, NumLayers, NonLinearity, Bias, BatchFirst, Dropout, Bidirectional, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Recurrent/RecurrentNeuralNetworkCell.cs b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/RecurrentNeuralNetworkCell.cs new file mode 100644 index 00000000..0cfc972d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Recurrent/RecurrentNeuralNetworkCell.cs @@ -0,0 +1,75 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Recurrent; + +/// +/// Represents an operator that creates a recurrent neural network (RNN) cell. +/// +/// +/// See for more information. +/// +[Description("Creates a recurrent neural network (RNN) cell.")] +public class RecurrentNeuralNetworkCell +{ + /// + /// The number of expected features in the input. + /// + [Description("The number of expected features in the input.")] + public long InputSize { get; set; } + + /// + /// The number of features in the hidden state. + /// + [Description("The number of features in the hidden state.")] + public long HiddenSize { get; set; } + + /// + /// The type of non-linearity to use. + /// + [Description("The type of non-linearity to use.")] + public NonLinearities NonLinearity { get; set; } = NonLinearities.Tanh; + + /// + /// If set to false, then the layer will not use bias weights. + /// + [Description("If set to false, then the layer will not use bias weights.")] + public bool Bias { get; set; } = true; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates an RNNCell module. + /// + /// + public IObservable Process() + { + return Observable.Return(RNNCell(InputSize, HiddenSize, NonLinearity, Bias, Device, Type)); + } + + /// + /// Creates an RNNCell module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => RNNCell(InputSize, HiddenSize, NonLinearity, Bias, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/RecurrentModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/RecurrentModuleBuilder.cs new file mode 100644 index 00000000..58fa5bae --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/RecurrentModuleBuilder.cs @@ -0,0 +1,46 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a recurrent neural network module. +/// +[XmlInclude(typeof(Recurrent.GatedRecurrentUnit))] +[XmlInclude(typeof(Recurrent.GatedRecurrentUnitCell))] +[XmlInclude(typeof(Recurrent.LongShortTermMemory))] +[XmlInclude(typeof(Recurrent.LongShortTermMemoryCell))] +[XmlInclude(typeof(Recurrent.RecurrentNeuralNetwork))] +[XmlInclude(typeof(Recurrent.RecurrentNeuralNetworkCell))] +[DefaultProperty(nameof(RecurrentModule))] +[Combinator] +[Description("Creates a recurrent neural network module.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class RecurrentModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "RecurrentModule"; + + /// + /// Initializes a new instance of the class. + /// + public RecurrentModuleBuilder() + { + Module = new Recurrent.RecurrentNeuralNetwork(); + } + + /// + /// Gets or sets the specific recurrent neural network module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific recurrent neural network module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object RecurrentModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/SaveModel.cs b/src/Bonsai.ML.Torch/NeuralNets/SaveModel.cs deleted file mode 100644 index da4a2cd9..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/SaveModel.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System; -using System.ComponentModel; -using System.Reactive.Linq; -using System.Xml.Serialization; - -namespace Bonsai.ML.Torch.NeuralNets -{ - /// - /// Saves the model to a file. - /// - [Combinator] - [ResetCombinator] - [Description("Saves the model to a file.")] - [WorkflowElementCategory(ElementCategory.Sink)] - public class SaveModel - { - /// - /// The model to save. - /// - [Description("The model to save.")] - [XmlIgnore] - public ITorchModule Model { get; set; } - - /// - /// The path to save the model. - /// - [Description("The path to save the model.")] - [Editor("Bonsai.Design.OpenFileNameEditor, Bonsai.Design", DesignTypes.UITypeEditor)] - public string ModelPath { get; set; } - - /// - /// Saves the model to the specified file path. - /// - /// - /// - /// - public IObservable Process(IObservable source) - { - return source.Do(input => { - Model.Module.save(ModelPath); - }); - } - } -} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/SaveModuleStateBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/SaveModuleStateBuilder.cs new file mode 100644 index 00000000..ce7dfb02 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/SaveModuleStateBuilder.cs @@ -0,0 +1,79 @@ +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Collections.Generic; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Reflection; +using static TorchSharp.torch; +using Bonsai.Expressions; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that saves a module's state to a file. +/// +[Combinator] +[Description("Saves a module's state to a file.")] +[WorkflowElementCategory(ElementCategory.Sink)] +public class SaveModuleStateBuilder : SingleArgumentExpressionBuilder +{ + /// + /// The path where the module's state will be stored. + /// + [Description("The path where the module's state will be stored.")] + [Editor("Bonsai.Design.SaveFileNameEditor, Bonsai.Design", DesignTypes.UITypeEditor)] + public string ModulePath { get; set; } + + /// + public override Expression Build(IEnumerable args) + { + var inputArg = args.First().Type.GetGenericArguments()[0]; + bool isInputModule = false; + for (var t = inputArg; t != null && t != typeof(object); t = t.BaseType) + { + if (t.GetType() == typeof(nn.Module) || t.IsGenericType && t.GetGenericTypeDefinition().Name.StartsWith("Module`")) + { + isInputModule = true; + break; + } + } + + if (!isInputModule) + throw new InvalidOperationException("The SaveModuleState operator requires a Module type as input."); + + var selectMethod = typeof(Observable) + .GetMethods() + .First(m => + m.Name == nameof(Observable.Select) && + m.IsGenericMethodDefinition && + m.GetParameters().Length == 2); + + var moduleParameter = Expression.Parameter(inputArg, "m"); + + var moduleSaveMethods = typeof(nn.Module) + .GetMethods(BindingFlags.Instance | BindingFlags.Public) + .Where(m => + { + var parameters = m.GetParameters(); + return m.Name == "save" && parameters.Length == 2 && parameters[0].ParameterType == typeof(string) && parameters[1].ParameterType == typeof(IList); + }); + + var moduleSaveMethod = moduleSaveMethods.First(); + + List moduleSaveArgs = + [ + Expression.Constant(ModulePath, typeof(string)), + Expression.Constant(null, typeof(IList)) + ]; + + var saveCall = Expression.Call(moduleParameter, moduleSaveMethod, moduleSaveArgs); + + var convertSaveCall = Expression.Convert(saveCall, inputArg); + var lambda = Expression.Lambda(convertSaveCall, moduleParameter); + + var genericSelectMethod = selectMethod.MakeGenericMethod(inputArg, inputArg); + + return Expression.Call(genericSelectMethod, args.First(), lambda); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/SetTrainingMode.cs b/src/Bonsai.ML.Torch/NeuralNets/SetTrainingMode.cs new file mode 100644 index 00000000..0acce866 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/SetTrainingMode.cs @@ -0,0 +1,202 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; +using static TorchSharp.torch.jit; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that sets the training mode of the input module. +/// +[Combinator] +[ResetCombinator] +[Description("Sets the training mode of the input module.")] +[WorkflowElementCategory(ElementCategory.Sink)] +public class SetTrainingMode +{ + /// + /// The training mode to set for the module. + /// + [Description("The training mode to set for the module.")] + public TrainingMode Mode { get; set; } = TrainingMode.Train; + + /// + /// Sets the training mode of the module. + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + /// + /// + public IObservable> Process(IObservable> source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + /// + /// + /// + public IObservable> Process(IObservable> source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + /// + /// + /// + /// + public IObservable> Process(IObservable> source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + /// + /// + /// + /// + /// + public IObservable> Process(IObservable> source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + /// + /// + /// + /// + /// + /// + public IObservable> Process(IObservable> source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public IObservable> Process(IObservable> source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + /// + public IObservable> Process(IObservable> source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + /// + /// + public IObservable> Process(IObservable> source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } + + /// + /// Sets the training mode of the module. + /// + /// + /// + /// + /// + /// + public IObservable> Process(IObservable> source) + { + return source.Do(input => + { + input.train(Mode == TrainingMode.Train); + }); + } +} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/Shuffle/ChannelShuffle.cs b/src/Bonsai.ML.Torch/NeuralNets/Shuffle/ChannelShuffle.cs new file mode 100644 index 00000000..362c8c3e --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Shuffle/ChannelShuffle.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Shuffle; + +/// +/// Represents an operator that creates a channel shuffle module. +/// +/// +/// See for more information. +/// +[Description("Creates a channel shuffle module.")] +public class ChannelShuffle +{ + /// + /// The number of groups to divide the channels into. + /// + [Description("The number of groups to divide the channels into.")] + public long Groups { get; set; } + + /// + /// Creates a channel shuffle module. + /// + /// + public IObservable Process() + { + return Observable.Return(ChannelShuffle(Groups)); + } + + /// + /// Creates a channel shuffle module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => ChannelShuffle(Groups)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ShuffleModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/ShuffleModuleBuilder.cs new file mode 100644 index 00000000..4dd00f7e --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ShuffleModuleBuilder.cs @@ -0,0 +1,41 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a module for shuffling input. +/// +[XmlInclude(typeof(Shuffle.ChannelShuffle))] +[DefaultProperty(nameof(ShuffleModule))] +[Combinator] +[Description("Creates a module for shuffling input.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class ShuffleModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "ShuffleModule"; + + /// + /// Initializes a new instance of the class. + /// + public ShuffleModuleBuilder() + { + Module = new Shuffle.ChannelShuffle(); + } + + /// + /// Gets or sets the specific shuffle module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific shuffle module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object ShuffleModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Sparse/Embedding.cs b/src/Bonsai.ML.Torch/NeuralNets/Sparse/Embedding.cs new file mode 100644 index 00000000..3a4e5d83 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Sparse/Embedding.cs @@ -0,0 +1,93 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Sparse; + +/// +/// Represents an operator that creates an embedding module. +/// +/// +/// See for more information. +/// +[Description("Creates an embedding module.")] +public class Embedding +{ + /// + /// The size of the dictionary of embeddings. + /// + [Description("The size of the dictionary of embeddings.")] + public long NumEmbeddings { get; set; } + + /// + /// The size of each embedding vector. + /// + [Description("The size of each embedding vector.")] + public long EmbeddingDims { get; set; } + + /// + /// If specified, the padding indices do not contribute to the gradient and are not updated during training. + /// + [Description("If specified, the padding indices do not contribute to the gradient and are not updated during training.")] + public long? PaddingIdx { get; set; } = null; + + /// + /// If specified, each embedding vector is clipped to have this as its maximum norm. + /// + [Description("If specified, each embedding vector is clipped to have this as its maximum norm.")] + public double? MaxNorm { get; set; } = null; + + /// + /// The degree of the norm to compute for the max norm. + /// + [Description("The degree of the norm to compute for the max norm.")] + public double NormType { get; set; } = 2D; + + /// + /// If true, the embedding vectors are scaled by the inverse of their frequency in the input. + /// + [Description("If true, the embedding vectors are scaled by the inverse of their frequency in the input.")] + public bool ScaleGradByFreq { get; set; } = false; + + /// + /// If true, the gradient will be sparse. + /// + [Description("If true, the gradient will be sparse.")] + public bool Sparse { get; set; } = false; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates an embedding module. + /// + /// + public IObservable Process() + { + return Observable.Return(Embedding(NumEmbeddings, EmbeddingDims, PaddingIdx, MaxNorm, NormType, ScaleGradByFreq, Sparse, Device, Type)); + } + + /// + /// Creates an embedding module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Embedding(NumEmbeddings, EmbeddingDims, PaddingIdx, MaxNorm, NormType, ScaleGradByFreq, Sparse, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Sparse/EmbeddingBag.cs b/src/Bonsai.ML.Torch/NeuralNets/Sparse/EmbeddingBag.cs new file mode 100644 index 00000000..4dfe806f --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Sparse/EmbeddingBag.cs @@ -0,0 +1,106 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Sparse; + +/// +/// Represents an operator that creates an embedding bag module. +/// +/// +/// See for more information. +/// +[Description("Creates an embedding bag module.")] +public class EmbeddingBag +{ + /// + /// The size of the dictionary of embeddings. + /// + [Description("The size of the dictionary of embeddings.")] + public long NumEmbeddings { get; set; } + + /// + /// The size of each embedding vector. + /// + [Description("The size of each embedding vector.")] + public long EmbeddingDims { get; set; } + + /// + /// If specified, each embedding vector is clipped to have this as its maximum norm. + /// + [Description("If specified, each embedding vector is clipped to have this as its maximum norm.")] + public double? MaxNorm { get; set; } = null; + + /// + /// The degree of the norm to compute for the max norm. + /// + [Description("The degree of the norm to compute for the max norm.")] + public double NormType { get; set; } = 2D; + + /// + /// If set to true, the embeddings vectors are scaled by the inverse of their frequency in the input. + /// + [Description("If set to true, the embeddings vectors are scaled by the inverse of their frequency in the input.")] + public bool ScaleGradByFreq { get; set; } = false; + + /// + /// The type of reduction to apply to the embeddings. + /// + [Description("The type of reduction to apply to the embeddings.")] + public EmbeddingBagMode Mode { get; set; } = EmbeddingBagMode.Mean; + + /// + /// If set to true, the gradient will be sparse. + /// + [Description("If set to true, the gradient will be sparse.")] + public bool Sparse { get; set; } = false; + + /// + /// If set to true, the offsets tensor includes an additional element at the end equal to the size of indices. + /// + [Description("If set to true, the offsets tensor includes an additional element at the end equal to the size of indices.")] + public bool IncludeLastOffset { get; set; } = false; + + /// + /// If specified, the entries do not contribute to the gradient and are not updated during training. + /// + [Description("If specified, the entries do not contribute to the gradient and are not updated during training.")] + public long PaddingIndex { get; set; } = -1; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType? Type { get; set; } = null; + + /// + /// Creates an embedding bag module. + /// + /// + public IObservable Process() + { + return Observable.Return(EmbeddingBag(NumEmbeddings, EmbeddingDims, MaxNorm, NormType, ScaleGradByFreq, Mode, Sparse, IncludeLastOffset, PaddingIndex, Device, Type)); + } + + /// + /// Creates an embedding bag module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => EmbeddingBag(NumEmbeddings, EmbeddingDims, MaxNorm, NormType, ScaleGradByFreq, Mode, Sparse, IncludeLastOffset, PaddingIndex, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Sparse/EmbeddingBagFromPretrained.cs b/src/Bonsai.ML.Torch/NeuralNets/Sparse/EmbeddingBagFromPretrained.cs new file mode 100644 index 00000000..0de43006 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Sparse/EmbeddingBagFromPretrained.cs @@ -0,0 +1,121 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using TorchSharp; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Sparse; + +/// +/// Represents an operator that creates an embedding module from pretrained weights. +/// +/// +/// See for more information. +/// +[Description("Creates an embedding module from pretrained weights.")] +[TypeConverter(typeof(TensorOperatorConverter))] +public class EmbeddingBagFromPretrained : IScalarTypeProvider +{ + /// + /// The pretrained weights for the embedding bag. + /// + [XmlIgnore] + [Description("The pretrained weights for the embedding bag.")] + [TypeConverter(typeof(TensorConverter))] + public Tensor Embeddings { get; set; } + + /// + /// The values of the embeddings tensor in XML string format. + /// + [Browsable(false)] + [XmlElement(nameof(Embeddings))] + [EditorBrowsable(EditorBrowsableState.Never)] + public string EmbeddingsXml + { + get => TensorConverter.ConvertToString(Embeddings, Type); + set => Embeddings = TensorConverter.ConvertFromString(value, Type); + } + + /// + /// Determines whether to freeze the embeddings weights. + /// + [Description("Determines whether to freeze the embeddings weights.")] + public bool Freeze { get; set; } = true; + + /// + /// If specified, each embedding vector is clipped to have this as its maximum norm. + /// + [Description("If specified, each embedding vector is clipped to have this as its maximum norm.")] + public double? MaxNorm { get; set; } = null; + + /// + /// The degree of the norm to compute for the max norm. + /// + [Description("The degree of the norm to compute for the max norm.")] + public double NormType { get; set; } = 2D; + + /// + /// If set to true, the embedding vectors are scaled by the inverse of their frequency in the input. + /// + [Description("If set to true, the embedding vectors are scaled by the inverse of their frequency in the input.")] + public bool ScaleGradByFreq { get; set; } = false; + + /// + /// The type of reduction to apply to the embeddings. + /// + [Description("The type of reduction to apply to the embeddings.")] + public EmbeddingBagMode Mode { get; set; } = EmbeddingBagMode.Mean; + + /// + /// If set to true, the gradient will be sparse. + /// + [Description("If set to true, the gradient will be sparse.")] + public bool Sparse { get; set; } = false; + + /// + /// If set to true, the offsets tensor includes an additional element at the end equal to the size of indices. + /// + [Description("If set to true, the offsets tensor includes an additional element at the end equal to the size of indices.")] + public bool IncludeLastOffset { get; set; } = false; + + /// + /// If specified, the entries do not contribute to the gradient and are not updated during training. + /// + [Description("If specified, the entries do not contribute to the gradient and are not updated during training.")] + public long PaddingIndex { get; set; } = -1; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor")] + public ScalarType Type { get; set; } = ScalarType.Float32; + + /// + /// Creates an embedding bag module from pretrained weights. + /// + /// + public IObservable Process() + { + return Observable.Return(EmbeddingBag_from_pretrained(Embeddings, Freeze, MaxNorm, NormType, ScaleGradByFreq, Mode, Sparse, IncludeLastOffset, PaddingIndex, Device, Type)); + } + + /// + /// Creates an embedding bag module from pretrained weights. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => EmbeddingBag_from_pretrained(Embeddings, Freeze, MaxNorm, NormType, ScaleGradByFreq, Mode, Sparse, IncludeLastOffset, PaddingIndex, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Sparse/EmbeddingFromPretrained.cs b/src/Bonsai.ML.Torch/NeuralNets/Sparse/EmbeddingFromPretrained.cs new file mode 100644 index 00000000..b081d99e --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Sparse/EmbeddingFromPretrained.cs @@ -0,0 +1,108 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Sparse; + +/// +/// Represents an operator that creates an embedding module from pretrained weights. +/// +/// +/// See for more information. +/// +[Description("Creates an embedding module from pretrained weights.")] +[TypeConverter(typeof(TensorOperatorConverter))] +public class EmbeddingFromPretrained : IScalarTypeProvider +{ + /// + /// The pretrained weights for the embedding module. + /// + [XmlIgnore] + [Description("The pretrained weights for the embedding module.")] + [TypeConverter(typeof(TensorConverter))] + public Tensor Embeddings { get; set; } + + /// + /// The values of the embeddings tensor in XML string format. + /// + [Browsable(false)] + [XmlElement(nameof(Embeddings))] + [EditorBrowsable(EditorBrowsableState.Never)] + public string EmbeddingsXml + { + get => TensorConverter.ConvertToString(Embeddings, Type); + set => Embeddings = TensorConverter.ConvertFromString(value, Type); + } + + /// + /// Determines whether to freeze the embeddings weights. + /// + [Description("Determines whether to freeze the embeddings weights.")] + public bool Freeze { get; set; } = true; + + /// + /// If specified, the entries do not contribute to the gradient and are not updated during training. + /// + [Description("If specified, the entries do not contribute to the gradient and are not updated during training.")] + public long? PaddingIdx { get; set; } = null; + + /// + /// If specified, each embedding vector is clipped to have this as its maximum norm. + /// + [Description("If specified, each embedding vector is clipped to have this as its maximum norm.")] + public double? MaxNorm { get; set; } = null; + + /// + /// The degree of the norm to compute for the max norm. + /// + [Description("The degree of the norm to compute for the max norm.")] + public double NormType { get; set; } = 2D; + + /// + /// If set to true, the embeddings vectors are scaled by the inverse of their frequency in the input. + /// + [Description("If set to true, the embeddings vectors are scaled by the inverse of their frequency in the input.")] + public bool ScaleGradByFreq { get; set; } = false; + + /// + /// If set to true, the gradient will be sparse. + /// + [Description("If set to true, the gradient will be sparse.")] + public bool Sparse { get; set; } = false; + + /// + /// The desired device of the returned tensor. + /// + [XmlIgnore] + [Description("The desired device of the returned tensor.")] + public Device Device { get; set; } = null; + + /// + /// The desired data type of the returned tensor. + /// + [Description("The desired data type of the returned tensor.")] + public ScalarType Type { get; set; } = ScalarType.Float32; + + /// + /// Creates an embedding module from pretrained weights. + /// + /// + public IObservable Process() + { + return Observable.Return(Embedding_from_pretrained(Embeddings, Freeze, PaddingIdx, MaxNorm, NormType, ScaleGradByFreq, Sparse, Device, Type)); + } + + /// + /// Creates an embedding module from pretrained weights. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Embedding_from_pretrained(Embeddings, Freeze, PaddingIdx, MaxNorm, NormType, ScaleGradByFreq, Sparse, Device, Type)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/SparseModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/SparseModuleBuilder.cs new file mode 100644 index 00000000..9f8179f6 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/SparseModuleBuilder.cs @@ -0,0 +1,44 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a module for sparsification. +/// +[XmlInclude(typeof(Sparse.Embedding))] +[XmlInclude(typeof(Sparse.EmbeddingBag))] +[XmlInclude(typeof(Sparse.EmbeddingBagFromPretrained))] +[XmlInclude(typeof(Sparse.EmbeddingFromPretrained))] +[DefaultProperty(nameof(SparseModule))] +[Combinator] +[Description("Creates a module for sparsification.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class SparseModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "SparseModule"; + + /// + /// Initializes a new instance of the class. + /// + public SparseModuleBuilder() + { + Module = new Sparse.Embedding(); + } + + /// + /// Gets or sets the specific sparse module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific sparse module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object SparseModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/TorchModuleAdapter.cs b/src/Bonsai.ML.Torch/NeuralNets/TorchModuleAdapter.cs deleted file mode 100644 index 3ec35071..00000000 --- a/src/Bonsai.ML.Torch/NeuralNets/TorchModuleAdapter.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System; -using System.Reflection; -using static TorchSharp.torch; - -namespace Bonsai.ML.Torch.NeuralNets -{ - /// - /// Represents a torch module adapter that wraps a torch module or script module. - /// - public class TorchModuleAdapter : ITorchModule - { - private readonly nn.Module _module = null; - - private readonly jit.ScriptModule _scriptModule = null; - - private readonly Func _forwardFunc; - - /// - /// The module. - /// - public nn.Module Module { get; } - - /// - /// Initializes a new instance of the class. - /// - /// - public TorchModuleAdapter(nn.Module module) - { - _module = module; - _forwardFunc = _module.forward; - Module = _module; - } - - /// - /// Initializes a new instance of the class. - /// - /// - public TorchModuleAdapter(jit.ScriptModule scriptModule) - { - _scriptModule = scriptModule; - _forwardFunc = _scriptModule.forward; - Module = _scriptModule; - } - - /// - public Tensor Forward(Tensor input) - { - return _forwardFunc(input); - } - } -} \ No newline at end of file diff --git a/src/Bonsai.ML.Torch/NeuralNets/TrainingMode.cs b/src/Bonsai.ML.Torch/NeuralNets/TrainingMode.cs new file mode 100644 index 00000000..9f9ebc9c --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/TrainingMode.cs @@ -0,0 +1,17 @@ +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Specifies the training mode for a torch module. +/// +public enum TrainingMode +{ + /// + /// Sets the module to evaluation mode. + /// + Evaluation = 0, + + /// + /// Sets the module to training mode. + /// + Train = 1, +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Transformer/Transformer.cs b/src/Bonsai.ML.Torch/NeuralNets/Transformer/Transformer.cs new file mode 100644 index 00000000..1823fc7a --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Transformer/Transformer.cs @@ -0,0 +1,79 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Transformer; + +/// +/// Represents an operator that creates a Transformer module. +/// +/// +/// See for more information. +/// +[Description("Creates a Transformer module.")] +public class Transformer +{ + /// + /// The number of expected features in the encoder/decoder inputs. + /// + [Description("The number of expected features in the encoder/decoder inputs.")] + public long DimModel { get; set; } = 512; + + /// + /// The number of heads in the multi-head attention models. + /// + [Description("The number of heads in the multi-head attention models.")] + public long NumHeads { get; set; } = 8; + + /// + /// The number of sub-encoder layers in the encoder. + /// + [Description("The number of sub-encoder layers in the encoder.")] + public long NumEncoderLayers { get; set; } = 6; + + /// + /// The number of sub-decoder layers in the decoder. + /// + [Description("The number of sub-decoder layers in the decoder.")] + public long NumDecoderLayers { get; set; } = 6; + + /// + /// The dimension of the feedforward network model. + /// + [Description("The dimension of the feedforward network model.")] + public long DimFeedforward { get; set; } = 2048; + + /// + /// The probability of dropout. + /// + [Description("The probability of dropout.")] + public double Dropout { get; set; } = 0.1D; + + /// + /// The activation function of the intermediate layer. + /// + [Description("The activation function of the intermediate layer.")] + public Activations Activation { get; set; } = Activations.ReLU; + + /// + /// Creates a Transformer module. + /// + /// + public IObservable Process() + { + return Observable.Return(nn.Transformer(DimModel, NumHeads, NumEncoderLayers, NumDecoderLayers, DimFeedforward, Dropout, Activation)); + } + + /// + /// Creates a Transformer module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => nn.Transformer(DimModel, NumHeads, NumEncoderLayers, NumDecoderLayers, DimFeedforward, Dropout, Activation)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerDecoder.cs b/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerDecoder.cs new file mode 100644 index 00000000..c04e19db --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerDecoder.cs @@ -0,0 +1,33 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Transformer; + +/// +/// Represents an operator that creates a transformer decoder module. +/// +/// +/// See for more information. +/// +[Description("Creates a transformer decoder module.")] +public class TransformerDecoder +{ + /// + /// The number of sub-decoder layers in the decoder. + /// + [Description("The number of sub-decoder layers in the decoder.")] + public long NumLayers { get; set; } + + /// + /// Creates a TransformerDecoder module from the input TransformerDecoderLayer. + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(input => TransformerDecoder(input, NumLayers)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerDecoderLayer.cs b/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerDecoderLayer.cs new file mode 100644 index 00000000..c4be092d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerDecoderLayer.cs @@ -0,0 +1,67 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Transformer; + +/// +/// Represents an operator that creates a transformer decoder layer. +/// +/// +/// See for more information. +/// +[Description("Creates a transformer decoder layer.")] +public class TransformerDecoderLayer +{ + /// + /// The number of expected features in the input. + /// + [Description("The number of expected features in the input.")] + public long DimModel { get; set; } = 512; + + /// + /// The number of heads in the multi-head attention models. + /// + [Description("The number of heads in the multi-head attention models.")] + public long NumHeads { get; set; } = 8; + + /// + /// The dimension of the feedforward network model. + /// + [Description("The dimension of the feedforward network model.")] + public long DimFeedforward { get; set; } = 2048; + + /// + /// The probability of dropout. + /// + [Description("The probability of dropout.")] + public double Dropout { get; set; } = 0.1D; + + /// + /// The activation function of the intermediate layer. + /// + [Description("The activation function of the intermediate layer.")] + public Activations Activation { get; set; } = Activations.ReLU; + + /// + /// Creates a TransformerDecoderLayer module. + /// + /// + public IObservable Process() + { + return Observable.Return(TransformerDecoderLayer(DimModel, NumHeads, DimFeedforward, Dropout, Activation)); + } + + /// + /// Creates a TransformerDecoderLayer module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => TransformerDecoderLayer(DimModel, NumHeads, DimFeedforward, Dropout, Activation)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerEncoder.cs b/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerEncoder.cs new file mode 100644 index 00000000..0573b0b7 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerEncoder.cs @@ -0,0 +1,33 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Xml.Serialization; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Transformer; + +/// +/// Represents an operator that creates a transformer encoder module. +/// +/// +/// See for more information. +/// +[Description("Creates a transformer encoder module.")] +public class TransformerEncoder +{ + /// + /// The number of sub-encoder layers in the encoder. + /// + [Description("The number of sub-encoder layers in the encoder.")] + public long NumLayers { get; set; } + + /// + /// Creates a TransformerEncoder module from the input TransformerEncoderLayer. + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(input => TransformerEncoder(input, NumLayers)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerEncoderLayer.cs b/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerEncoderLayer.cs new file mode 100644 index 00000000..f5e432ef --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Transformer/TransformerEncoderLayer.cs @@ -0,0 +1,67 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Transformer; + +/// +/// Represents an operator that creates a transformer encoder layer. +/// +/// +/// See for more information. +/// +[Description("Creates a transformer encoder layer.")] +public class TransformerEncoderLayer +{ + /// + /// The number of expected features in the input. + /// + [Description("The number of expected features in the input.")] + public long DimModel { get; set; } = 512; + + /// + /// The number of heads in the multi-head attention models. + /// + [Description("The number of heads in the multi-head attention models.")] + public long NumHeads { get; set; } = 8; + + /// + /// The dimension of the feedforward network model. + /// + [Description("The dimension of the feedforward network model.")] + public long DimFeedforward { get; set; } = 2048; + + /// + /// The probability of dropout. + /// + [Description("The probability of dropout.")] + public double Dropout { get; set; } = 0.1D; + + /// + /// The activation function of the intermediate layer. + /// + [Description("The activation function of the intermediate layer.")] + public Activations Activation { get; set; } = Activations.ReLU; + + /// + /// Creates a TransformerEncoderLayer module. + /// + /// + public IObservable Process() + { + return Observable.Return(TransformerEncoderLayer(DimModel, NumHeads, DimFeedforward, Dropout, Activation)); + } + + /// + /// Creates a TransformerEncoderLayer module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => TransformerEncoderLayer(DimModel, NumHeads, DimFeedforward, Dropout, Activation)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/TransformerModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/TransformerModuleBuilder.cs new file mode 100644 index 00000000..d99c2af2 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/TransformerModuleBuilder.cs @@ -0,0 +1,45 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a transformer module. +/// +[XmlInclude(typeof(Transformer.Transformer))] +[XmlInclude(typeof(Transformer.TransformerDecoder))] +[XmlInclude(typeof(Transformer.TransformerDecoderLayer))] +[XmlInclude(typeof(Transformer.TransformerEncoder))] +[XmlInclude(typeof(Transformer.TransformerEncoderLayer))] +[DefaultProperty(nameof(TransformerModule))] +[Combinator] +[Description("Creates a transformer module.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class TransformerModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "TransformerModule"; + + /// + /// Initializes a new instance of the class. + /// + public TransformerModuleBuilder() + { + Module = new Transformer.Transformer(); + } + + /// + /// Gets or sets the specific transformer module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific transformer module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object TransformerModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Vision/PixelShuffle.cs b/src/Bonsai.ML.Torch/NeuralNets/Vision/PixelShuffle.cs new file mode 100644 index 00000000..7239cc26 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Vision/PixelShuffle.cs @@ -0,0 +1,42 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Vision; + +/// +/// Represents an operator that creates a pixel shuffle module. +/// +/// +/// See for more information. +/// +[Description("Creates a pixel shuffle module.")] +public class PixelShuffle +{ + /// + /// The factor to increase spatial resolution by. + /// + [Description("The factor to increase spatial resolution by.")] + public long UpscaleFactor { get; set; } + + /// + /// Creates a pixel shuffle module. + /// + public IObservable Process() + { + return Observable.Return(PixelShuffle(UpscaleFactor)); + } + + /// + /// Creates a pixel shuffle module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => PixelShuffle(UpscaleFactor)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Vision/PixelUnshuffle.cs b/src/Bonsai.ML.Torch/NeuralNets/Vision/PixelUnshuffle.cs new file mode 100644 index 00000000..3139616d --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Vision/PixelUnshuffle.cs @@ -0,0 +1,43 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Vision; + +/// +/// Represents an operator that creates a pixel unshuffle module. +/// +/// +/// See for more information. +/// +[Description("Creates a pixel unshuffle module.")] +public class PixelUnshuffle +{ + /// + /// The factor to decrease spatial resolution by. + /// + [Description("The factor to decrease spatial resolution by.")] + public long DownscaleFactor { get; set; } + + /// + /// Creates a pixel unshuffle module. + /// + /// + public IObservable Process() + { + return Observable.Return(PixelUnshuffle(DownscaleFactor)); + } + + /// + /// Creates a pixel unshuffle module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => PixelUnshuffle(DownscaleFactor)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/Vision/Upsample.cs b/src/Bonsai.ML.Torch/NeuralNets/Vision/Upsample.cs new file mode 100644 index 00000000..b1e0bda2 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/Vision/Upsample.cs @@ -0,0 +1,77 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; +using static TorchSharp.torch.nn; + +namespace Bonsai.ML.Torch.NeuralNets.Vision; + +/// +/// Creates an Upsample module. +/// +/// +/// See for more information. +/// +[Description("Creates an Upsample module.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class Upsample +{ + /// + /// The output spatial sizes. + /// + [Description("The output spatial sizes.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public long[] Size { get; set; } = null; + + /// + /// The multiplier for the spatial size. + /// + [Description("The multiplier for the spatial size.")] + [TypeConverter(typeof(UnidimensionalArrayConverter))] + public double[] ScaleFactor { get; set; } = null; + + /// + /// The upsampling algorithm. + /// + [Description("The upsampling algorithm.")] + public UpsampleMode Mode { get; set; } = UpsampleMode.Nearest; + + /// + /// If True, the corner pixels of the input and output tensors are aligned. + /// + /// + /// This only has effect when mode is 'linear', 'bilinear', 'bicubic' or 'trilinear'. + /// + [Description("If True, the corner pixels of the input and output tensors are aligned.")] + public bool? AlignCorners { get; set; } = null; + + /// + /// Recomputes the scale factor for use in the interpolation calculation. + /// + /// + /// If true, then the scale factor must be passed in and is used to compute the output size. + /// If false, then size or scale factor will be used for interpolation. + /// + [Description("Recomputes the scale factor for use in the interpolation calculation.")] + public bool? RecomputeScaleFactor { get; set; } = null; + + /// + /// Creates an Upsample module. + /// + /// + public IObservable Process() + { + return Observable.Return(Upsample(Size, ScaleFactor, Mode, AlignCorners, RecomputeScaleFactor)); + } + + /// + /// Creates an Upsample module. + /// + /// + /// + /// + public IObservable Process(IObservable source) + { + return source.Select(_ => Upsample(Size, ScaleFactor, Mode, AlignCorners, RecomputeScaleFactor)); + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/VisionModuleBuilder.cs b/src/Bonsai.ML.Torch/NeuralNets/VisionModuleBuilder.cs new file mode 100644 index 00000000..fb584f3b --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/VisionModuleBuilder.cs @@ -0,0 +1,43 @@ +using System.ComponentModel; +using System.Xml.Serialization; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that creates a module for image processing. +/// +[XmlInclude(typeof(Vision.PixelShuffle))] +[XmlInclude(typeof(Vision.PixelUnshuffle))] +[XmlInclude(typeof(Vision.Upsample))] +[DefaultProperty(nameof(VisionModule))] +[Combinator] +[Description("Creates a module for image processing.")] +[WorkflowElementCategory(ElementCategory.Source)] +public class VisionModuleBuilder : ModuleCombinatorBuilder, INamedElement +{ + internal override string BuilderName => "VisionModule"; + + /// + /// Initializes a new instance of the class. + /// + public VisionModuleBuilder() + { + Module = new Vision.PixelShuffle(); + } + + /// + /// Gets or sets the specific vision module to create. + /// + [DesignOnly(true)] + [DisplayName("Module")] + [Externalizable(false)] + [RefreshProperties(RefreshProperties.All)] + [Category(nameof(CategoryAttribute.Design))] + [Description("The specific vision module to create.")] + [TypeConverter(typeof(ModuleTypeConverter))] + public object VisionModule + { + get => Module; + set => Module = value; + } +} diff --git a/src/Bonsai.ML.Torch/NeuralNets/ZeroGradient.cs b/src/Bonsai.ML.Torch/NeuralNets/ZeroGradient.cs new file mode 100644 index 00000000..2eb76410 --- /dev/null +++ b/src/Bonsai.ML.Torch/NeuralNets/ZeroGradient.cs @@ -0,0 +1,25 @@ +using System; +using System.ComponentModel; +using System.Reactive.Linq; +using static TorchSharp.torch; + +namespace Bonsai.ML.Torch.NeuralNets; + +/// +/// Represents an operator that sets the gradient of the input optimizer to zero. +/// +[Combinator] +[Description("Sets the gradient of the input optimizer to zero.")] +[WorkflowElementCategory(ElementCategory.Sink)] +public class ZeroGradient +{ + /// + /// Sets the gradient of the input optimizer to zero. + /// + /// + /// + public IObservable Process(IObservable source) where T : optim.Optimizer + { + return source.Do(input => input.zero_grad()); + } +}