MUS 15 Project 2: Audio Transcription

Test

Gino Prasad, Saba Heydari Seradj, Ashish Dalvi

05/27/2023

For our MUS 15 project, our group created an audio transcriber from scratch. All of the code was made from scratch by us using python.

We have uploaded this code to https://github.com/GinoP123/MusicTranscription

import torch
import torchaudio
from IPython.display import Audio
import numpy as np
import matplotlib.pyplot as plt
from scipy.fft import fft, ifft
import scipy.ndimage
import pandas as pd
import tqdm

print(torch.__version__)
print(torchaudio.__version__)

2.0.0.dev20221214
0.14.0.dev20221214

Loading Example Audio

audio_path = 'wav_files/audio1623001661.wav'

bruyeres_wav, sample_rate = torchaudio.load(audio_path)
bruyeres_wav = bruyeres_wav[0][19*sample_rate:-7*sample_rate]

Audio(bruyeres_wav, rate=sample_rate)

Assigning Pitches to Frequencies

prelim_notes_octave = 4
prelim_notes = [("C", 261.63), ("C#", 277.18), ("D", 293.66), ("D#", 311.13),
         ("E", 329.63), ("F", 349.23), ("F#", 369.99), ("G", 392.00), ("G#", 415.30), ("A", 440.00), ("A#", 466.16), ("B", 493.88), ]
prelim_notes

[('C', 261.63),
 ('C#', 277.18),
 ('D', 293.66),
 ('D#', 311.13),
 ('E', 329.63),
 ('F', 349.23),
 ('F#', 369.99),
 ('G', 392.0),
 ('G#', 415.3),
 ('A', 440.0),
 ('A#', 466.16),
 ('B', 493.88)]

notes, names = zip(*[(freq * (2 ** (octave - prelim_notes_octave)), f"{name}{octave}") for octave in range(8) for name, freq in prelim_notes])
notes = pd.DataFrame(notes, index=names, columns=['frequency'])
notes

	frequency
C0	16.351875
C#0	17.323750
D0	18.353750
D#0	19.445625
E0	20.601875
...	...
G7	3136.000000
G#7	3322.400000
A7	3520.000000
A#7	3729.280000
B7	3951.040000

96 rows × 1 columns

Setting the hyperparameters

window_size = 2 ** 13
hop_size = window_size // 4

# # Gaussian Kernel Parameters
# gkernel_length = 21
# gkernel_std = 1

# gaussian_scale_percentile = 99
# radius = 3

min_amplitude_threshold = 50
overtone_comb_limit = 3 # highest (c * frequency) computed for comb
overtone_dissipation_rate = 0.02
overtone_kernel = np.array([min_amplitude_threshold * (overtone_dissipation_rate ** i) for i in range(overtone_comb_limit)])
overtone_kernel

array([5.e+01, 1.e+00, 2.e-02])

reasonable_notes = ('C0', 'C6')

reasonable_notes = ('C3', 'C7')
mask = notes['frequency'] >= notes.loc[reasonable_notes[0]]['frequency']
mask *= notes['frequency'] <= notes.loc[reasonable_notes[1]]['frequency']
reasonable_notes = notes[mask]
reasonable_notes.head()

	frequency
C3	130.815
C#3	138.590
D3	146.830
D#3	155.565
E3	164.815

amplitude = 1
def get_frequency(note=None, frequency=None, length=window_size/sample_rate):
    assert note is not None or frequency is not None
    if frequency is None:
        frequency = notes.loc[note]['frequency']
    base = np.arange(0, length*sample_rate).astype(np.float64)
    c = (frequency * 2 * np.pi) / sample_rate
    wavelet_ = amplitude * np.sin(c * base)
    return wavelet_

Audio(get_frequency(note='C5', length=5), rate=sample_rate)

Predicting Frequencies

def get_amplitude(window, note=None, frequency=None, return_phase=False):
    assert note is not None or frequency is not None
    if frequency is None:
        frequency = notes.loc[note].frequency
    c = (frequency * 2 * np.pi) / sample_rate
    sin_coeff, cos_coeff = [np.dot(func(c * np.arange(len(window))), window) for func in (np.sin, np.cos)]
    term = cos_coeff + (1j*sin_coeff)
    if return_phase:
        return np.absolute(term), np.angle(term)
    return np.absolute(term)

def get_combs(window):
    cache, combs = {}, {}
    for note, fundamental_frequency in zip(notes.index, notes.frequency):
        combs[note] = []
        for overtone in range(1, overtone_comb_limit+1):
            frequency = fundamental_frequency * overtone
            if frequency not in cache:
                cache[frequency] = get_amplitude(window, frequency=frequency)
            combs[note].append(cache[frequency])
        combs[note] = np.array(combs[note])
    return combs

def predict_frequencies_window(window):
    predicted = []
    for note, comb in get_combs(window).items():
        if (comb > overtone_kernel).all():
            predicted.append(note)
    return predicted

def predict_frequencies(wav):
    N = len(wav)
    predicted = []
    for start in tqdm.tqdm(range(0, N, hop_size)):
        if N - start < window_size:
            break
        window = wav[start:start+window_size].numpy()
        predicted.append(predict_frequencies_window(window))
    return predicted

Example Window Clip

clip = bruyeres_wav[int(sample_rate*(0.5)):int(sample_rate*(0.5)+window_size)].numpy()

Audio(clip, rate=sample_rate)

predict_frequencies_window(clip)

['D#5']

get_amplitude(clip, 'D#5')

139.77996524110867

Audio(get_frequency(note='D#5', length=5), rate=sample_rate)

Hann Window

def hann_window():
    return (1 - np.cos(2 * np.pi * np.arange(window_size) / window_size)) / (window_size / hop_size)
hann_window = hann_window()

plt.plot(hann_window)

[<matplotlib.lines.Line2D at 0x1743c5430>]

extended_window = np.concatenate([hann_window] * 16)
composite = extended_window.copy()
composite[:-hop_size] += extended_window[hop_size:]
composite[:-2*hop_size] += extended_window[2*hop_size:]
composite[:-3*hop_size] += extended_window[3*hop_size:]
plt.plot(composite)

[<matplotlib.lines.Line2D at 0x295f1cb50>]

Reconstructing Predicted Frequency

def reconstruct(predicted, freq=False):
    output_length = (len(predicted) - 1) * hop_size + window_size
    composite = np.zeros(output_length)
    for i, window in enumerate(predicted):
        base = np.zeros(window_size)
        for note in window:
            if not freq:
                frequency = notes.loc[note].frequency
            else:
                frequency = note
            for overtone in range(1,overtone_comb_limit+1):
                base += get_frequency(frequency=frequency*overtone*(overtone_dissipation_rate **(overtone-1)))
        composite[i*hop_size:(i*hop_size)+window_size] += base * hann_window
    return composite

Audio(reconstruct([['A5'] for _ in range(500)]), rate=sample_rate)

Audio(get_frequency(note='A5', length=20), rate=sample_rate)

Reconstructing Entire Song

# predicted_frequencies = predict_frequencies(bruyeres_wav)

Audio(reconstruct(predicted_frequencies), rate=sample_rate)

mask = np.load('/Users/ginoprasad/Downloads/midi_mask.npy')

from midi2audio import FluidSynth

FluidSynth().play_midi('/Users/ginoprasad/Downloads/MIDI-Unprocessed_041_PIANO041_MID--AUDIO-split_07-06-17_Piano-e_1-01_wav--1.midi')

fluidsynth: error: fluid_is_soundfont(): fopen() failed: 'File does not exist.'
Parameter '/Users/ginoprasad/.fluidsynth/default_sound_font.sf2' not a SoundFont or MIDI file or error occurred identifying it.
fluidsynth: error: fluid_sfloader_load(): Failed to open '/opt/homebrew/Cellar/fluid-synth/2.3.0/share/soundfonts/default.sf2': File does not exist.
fluidsynth: error: Unable to open file '/opt/homebrew/Cellar/fluid-synth/2.3.0/share/soundfonts/default.sf2'
fluidsynth: error: Failed to load SoundFont "/opt/homebrew/Cellar/fluid-synth/2.3.0/share/soundfonts/default.sf2"
fluidsynth: warning: No preset found on channel 0 [bank=0 prog=0]

KeyboardInterrupt

frequencies = np.array([400 * (2 ** ((n-69)/12)) for n in range(128)])
def reconstruct_mask(mask):
    output_length = (len(mask) - 1) * hop_size + window_size
    composite = np.zeros(output_length)
    for i, window in enumerate(mask):
        base = np.zeros(hop_size)
        for amp, frequency in zip(window, frequencies):
            if amp != 0:
                for overtone in range(1,overtone_comb_limit+1):
                    base += amp * get_frequency(frequency=frequency*overtone*(overtone_dissipation_rate **(overtone-1)), length=hop_size/sample_rate)
        composite[i*hop_size:(i*hop_size)+hop_size] += base
    return composite

Audio(reconstruct_mask(mask), rate=sample_rate)

hop_size / sample_rate

0.064

from matplotlib.pyplot import imshow

mask.shape

(3021, 128)