predict arguments; autopitch example

Author: victor-shepardson

examples/notepredictor/autopitch.scd

@@ -0,0 +1,133 @@
+(
+~gui = false;
+MIDIIn.connectAll;
+b = NetAddr.new("127.0.0.1", 9999);
+Server.default.options.inDevice_("Built-in Microph");
+Server.default.options.outDevice_("Built-in Output");
+// Server.default.options.inDevice_("mic-buds");
+// Server.default.options.outDevice_("mic-buds");
+s.boot;
+~gui.if{
+    k = MIDIKeyboard.new(bounds: Rect(0, 0, 500, 100), octaves:11, startnote:0)
+};
+)
+
+(
+SynthDef(\pluck, {
+    var vel = \vel.kr;
+    var signal = Saw.ar(\freq.kr(20), 3e-2) * EnvGate.new(1);
+    var fr = 2.pow(Decay.ar(Impulse.ar(0), 3)*6*vel+8);
+    signal = BLowPass.ar(signal, fr)*vel;
+    Out.ar([0,1], signal);
+}).add
+)
+
+
+// measure round-trip latency
+(
+OSCdef(\return, {
+    arg msg, time, addr, recvPort;
+    (Process.elapsedTime - t).postln;
+}, '/prediction', nil);
+t = Process.elapsedTime;
+b.sendMsg("/predictor/predict",
+    \pitch, 60+12.rand, \time, 0, \vel, 0, \fix_time, 0, \fix_vel, 0);
+)
+
+// set the delay for more precise timing
+~delay = 0.01;
+
+
+// NetAddr.localAddr    // retrieve the current IP and port
+// thisProcess.openPorts; // list all open ports
+
+// model chooses pitches
+(
+~gate = 1;
+
+~reset = {
+    ~last_pitch = nil;
+    ~last_dt = nil;
+    ~last_vel = nil;
+    t = Process.elapsedTime;
+    b.sendMsg("/predictor/reset");
+    y!?{y.free};
+    y = nil;
+    b.sendMsg("/predictor/predict", \pitch, 128, \time, 0, \vel, 0);
+
+};
+
+~reset.();
+
+// footswitch
+MIDIdef.program(\switch, {
+    arg num, chan, src;
+    num.switch
+    {1}{~gate = 0}
+    {2}{~gate = 1}
+    {3}{
+        ~gate = 0;
+        SystemClock.clear;
+        b.sendMsg("/predictor/reset");
+        y.release;
+        SystemClock.clear;
+        };
+    ~gate.postln;
+});
+
+
+// MIDI from controller
+MIDIdef.noteOn(\input, {
+    arg vel, pitch, chan, src;
+    var t2 = Process.elapsedTime;
+    var dt = t2-(t?t2); //time since last note
+
+    // release the previous note
+    y.release(0.1);
+
+    // attack the current note with the old pitch
+    y = Synth(\pluck, [\freq, ~last_pitch.midicps, \vel, vel/127]);
+
+    // get a new prediction in light of last note,
+    // fixing dt and vel to performed values so just pitch is predicted
+    b.sendMsg("/predictor/predict",
+        \pitch, ~last_pitch, \time, ~last_dt, \vel, ~last_vel,
+        \fix_time, dt, \fix_vel, vel);
+
+    ~last_dt = dt;
+    ~last_vel = vel;
+
+    // mark time of current note
+    t = t2;
+});
+
+// OSC return from python
+OSCdef(\return, {
+    arg msg, time, addr, recvPort;
+    var pitch = msg[1]; // MIDI number of predicted note
+    var dt = msg[2]; // time to predicted note
+    var vel = msg[3]; // velocity 0-127
+
+    // store the pitch and immediately set (unless there is no synth,
+    // indicating this is the first note)
+    ~last_pitch = pitch;
+    ~last_dt.isNil.if{~last_dt = dt};
+    ~last_vel.isNil.if{~last_vel = vel};
+    y!?{y.set(\freq, ~last_pitch.midicps)};
+
+    [pitch, dt, vel].postln;
+
+}, "/prediction", nil);
+)
+
+~reset.()
+// send a note manually if you don't have a midi controller
+MIDIdef.all[\input].func.(64)
+
+(
+MIDIdef.all[\input].func.(64);
+MIDIdef.all[\input].func.(64);
+)
+
+// load another model
+// b.sendMsg("/predictor/load", "/path/to/checkpoint");

notepredictor/notepredictor/model.py

@@ -193,10 +193,16 @@ def cell_state_names(self):
     def cell_state(self):
         return tuple(getattr(self, n) for n in self.cell_state_names())
 
-    @property
-    def samplers(self):
+    def get_samplers(self, allow_start=False, allow_end=False):
+        def sample_pitch(x):
+            if not allow_start:
+                x[...,self.start_token] = -np.inf
+            if not allow_end:
+                x[...,self.end_token] = -np.inf
+            return D.Categorical(logits=x).sample()
+
         return (
-            lambda x: D.Categorical(logits=x).sample(), 
+            sample_pitch, 
             lambda x: self.time_dist.sample(x),
             lambda x: self.vel_dist.sample(x),
         )
@@ -285,15 +291,18 @@ def forward(self, pitches, times, velocities, validation=False):
         return r
 
     # TODO: force
-    def predict(self, pitch, time, vel, force=(None, None, None)):
+    def predict(self, 
+            pitch, time, vel, 
+            fix_pitch=None, fix_time=None, fix_vel=None, 
+            allow_end=False, allow_start=False):
         """
         supply the most recent note and return a prediction for the next note.
 
         Args:
             pitch: int. MIDI number of current note.
             time: float. elapsed time since previous note.
             vel: float. (possibly dequantized) MIDI velocity from 0-127 inclusive.
-            force: Tuple[Optional[Number]].
+            fix_*: same as above, but to fix a value for the predicted note
 
         Returns: dict of
             'pitch': int. predicted MIDI number of next note.
@@ -322,23 +331,24 @@ def predict(self, pitch, time, vel, force=(None, None, None)):
 
             modalities = list(zip(
                 self.projections,
-                self.samplers,
+                self.get_samplers(allow_start=allow_start, allow_end=allow_end),
                 self.embeddings,
                 ))
 
             context = []
             predicted = []
             params = []
 
-            force = [
+            fix = [
                 None if item is None else torch.tensor([[item]], dtype=dtype)
                 for item, dtype in zip(
-                    force, [torch.long, torch.float, torch.float])]
+                    [fix_pitch, fix_time, fix_vel],
+                    [torch.long, torch.float, torch.float])]
 
             # permute h_tgt, embs, modalities
             # if any modalities are determined, embed them;
             det_idx, undet_idx = [], []
-            for i,(item, embed) in enumerate(zip(force, self.embeddings)):
+            for i,(item, embed) in enumerate(zip(fix, self.embeddings)):
                 if item is None:
                     undet_idx.append(i)
                 else:
@@ -354,11 +364,17 @@ def predict(self, pitch, time, vel, force=(None, None, None)):
             # for each undetermined modality, 
             # sample a new value conditioned on alteady determined ones
 
+            # TODO: allow constraints; 
+            # attempt to sort the strongest constraints first
+            # constraints can be:
+            # discrete set, in which case evaluate probs and then sample categorical
+            # range, in which case truncate
+
             while len(undet_idx):
                 i = undet_idx.pop(0) # index of modality to determine
                 j = len(det_idx) # number already determined
                 project, sample, embed = modalities[i]
-                # determine the next modality
+                # determine value for the next modality
                 hidden = self.xformer(context, h_ctx, perm_h_tgt[:j+1])[j]
                 params.append(project(hidden))
                 pred = sample(params[-1])