# -*- coding: utf-8 -*-
"""MIDI_Search.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1q_PuqEBGEZKMeb975WEPKH-Vrv30Tc_N

# MIDI Search (Ver 1.3)

***

## MIDI Search/Plagiarizm checker

***

#### Project Los Angeles

#### Tegridy Code 2021

***

# Setup Environment, clone needed repos, and install all required dependencies
"""

#@title Install all dependencies (run only once per session)
!git clone https://github.com/asigalov61/tegridy-tools
!apt install fluidsynth #Pip does not work for some reason. Only apt works
!pip install midi2audio

if not os.path.exists('/content/Dataset'):
    os.makedirs('/content/Dataset')
os.chdir('/content/')

#@title Import all needed modules

print('Loading needed modules. Please wait...')
import os
import copy
import pickle

os.chdir('/content/tegridy-tools/tegridy-tools')
import TMIDI
import MIDI

import tqdm.auto

from midi2audio import FluidSynth
from IPython.display import display, Javascript, HTML, Audio

from google.colab import output, drive

chords_list_f = []
melody_list_f = []

os.chdir('/content/')
print('Loading complete. Enjoy! :)')

"""# Download and process desired MIDI datasets

## PLEASE NOTE: You need to have at least 16GB free RAM to load large datasets
"""

# Commented out IPython magic to ensure Python compatibility.
#@title Download complete (large) pre-processed MIDI datasets
# %cd /content/

# POP909 dataset (Complete)
# https://github.com/music-x-lab/POP909-Dataset
!wget --no-check-certificate -O POP.zip "https://onedrive.live.com/download?cid=8A0D502FC99C608F&resid=8A0D502FC99C608F%2118362&authkey=AD3_7eDWpgUhcaw"
!unzip POP.zip

# Clean MIDI subset (Complete)
# https://colinraffel.com/projects/lmd/
# https://github.com/craffel/midi-dataset
!wget --no-check-certificate -O clean_midi.zip "https://onedrive.live.com/download?cid=8A0D502FC99C608F&resid=8A0D502FC99C608F%2118368&authkey=AAFEzC5YBABk7rA"
!unzip clean_midi.zip

# Google MAESTRO Piano dataset (Complete)
# https://magenta.tensorflow.org/datasets/maestro
!wget --no-check-certificate -O MAESTRO.zip "https://onedrive.live.com/download?cid=8A0D502FC99C608F&resid=8A0D502FC99C608F%2118370&authkey=ABxzaWHKIeN1e3A"
!unzip MAESTRO.zip

# Reddit 30000 MIDIs subset (aka Melody Kit 1.0 by Melody Man)
# All channels except the drums
!wget --no-check-certificate -O 30000.zip "https://onedrive.live.com/download?cid=8A0D502FC99C608F&resid=8A0D502FC99C608F%2118372&authkey=AFwTJaVZyiVuetY"
!unzip 30000.zip

# %cd /content/

#@title Load/Merge and parse the dataset

full_path_to_the_dataset = "/content/30000.pkl" #@param {type:"string"}
merge_several_datasets = False #@param {type:"boolean"}

print('Loading the dataset...Please wait...')
with open(full_path_to_the_dataset, 'rb') as filehandle:
  MusicDataset = pickle.load(filehandle)

if merge_several_datasets:
  print('Merging datasets...Please wait...')
  chords_list_f.extend(MusicDataset[0])
  melody_list_f.extend(MusicDataset[1])

else:
  chords_list = MusicDataset[0]
  melody_list_f = MusicDataset[1]

chords_list_f = [ele for inner_list in chords_list for ele in inner_list]

print('Parsing the dataset...Please wait...')
chords_notes_list = []
melody_notes_list = []

for chord in chords_list_f:
  chords_notes_list.append(chord[4])

for note in melody_list_f:
  melody_notes_list.append(note[4])

print('Done!')

print('Number of chords recorded:', len(MusicDataset[0]))
print('The longest chord:', len(max(MusicDataset[0], key=len)), 'notes') 
print(max(MusicDataset[0], key=len))
print('Number of recorded melody events:', len(melody_list_f))
print('First melody event:', melody_list_f[0], 'Last Melody event:', melody_list_f[-1])
print('Total number of MIDI events recorded:', len(MusicDataset[0])+len(melody_list_f))
print('Dataset is loaded! Enjoy :)')

"""# If you are not sure where to start or what settings to select, please use original defaults"""

#@title Specify input MIDI file to search for
full_path_to_input_MIDI_file = "/content/tegridy-tools/tegridy-tools/seed.mid" #@param {type:"string"}
print('Loading the MIDI file...')
c_l, m_l = TMIDI.Tegridy_MIDI_Processor(full_path_to_input_MIDI_file, 
                                              -1, 
                                              1,
                                              )

chordz = []
melody = []

for notez in m_l:
  chordz.append([notez])
  melody.append(notez)
  for chord in c_l:
    if notez[1] == chord[0][1]:
      chordz.append(chord[1:])

chords = [ele for inner_list in chordz for ele in inner_list]      
      
print('MIDI file loaded.')
print('Total of', len(chords), 'notes.') 
print('Enjoy :)')

#@title Initial Search
number_of_notes_to_match = 5 #@param {type:"slider", min:2, max:10, step:1}

output_song = []
pattern = []

print('Initial pattern search...')
print('Loading input notes...')

print('Input number of notes is', len(chords))
output_song.extend(chords)

for note in output_song[len(output_song)-number_of_notes_to_match:]:
  pattern.append(note[4])

#pattern = [69, 65, 66, 62, 63]

print('Starting search...')
print('Looking for pitches pattern', pattern)
pattern_start_index = [i for i in range(0,len(chords_notes_list)) if list(chords_notes_list[i:i+len(pattern)])==pattern]

if pattern_start_index == []:
  print('Nothing found')
else:
  print('Found matching notes at index', pattern_start_index[0])

#@title Re-Search

output_song = []
pattern = []

print('Pattern re-search...')

try:
  pidx = pattern_start_index[0]+number_of_notes_to_match
except:
  pass


print('Loading input notes...')

print('Input number of notes is', len(chords))
output_song.extend(chords)

for note in output_song[len(output_song)-number_of_notes_to_match:]:
  pattern.append(note[4])

#pattern = [69, 65, 66, 62, 63]

print('Starting re-search...')
print('Looking for pitches pattern', pattern)
pattern_start_index = [i for i in range(pidx,len(chords_notes_list)) if list(chords_notes_list[i:i+len(pattern)])==pattern]

if pattern_start_index == []:
  print('Nothing found')
else:
  print('Found matching notes at index', pattern_start_index[0])

"""# Generate, download, and listen to the output"""

#@title Convert to MIDI
#@markdown Standard MIDI timings are 400/120(80)

number_of_ticks_per_quarter = 424 #@param {type:"slider", min:8, max:1000, step:8}
number_of_notes_to_play = 300 #@param {type:"slider", min:10, max:500, step:10}
simulate_velocity = True #@param {type:"boolean"}

output_song = []

for n in chords:
  if simulate_velocity:
    n[5] = n[4]
  output_song.append(n)

output_s = chords_list_f[pattern_start_index[0]+number_of_notes_to_match:pattern_start_index[0]+number_of_notes_to_play]
delta = output_song[-1][1]
out = copy.deepcopy(output_s)

ptime = 0
time = 0

for n in range(len(out)-1):
  no = copy.deepcopy(out[n])
  no[1] = delta
  if simulate_velocity:
    no[5] = no[4]

  output_song.append(no)
  ptime = out[n][1]
  time = out[n+1][1]
  delta += time - ptime

output_signature = 'MIDI Search'

output_header = [number_of_ticks_per_quarter, [['track_name', 0, bytes(output_signature, 'utf-8')]]] 

list_of_MIDI_patches = [0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 0, 0, 0, 0, 0, 0]                                                  

patch_list = [['patch_change', 0, 0, list_of_MIDI_patches[0]], 
                ['patch_change', 0, 1, list_of_MIDI_patches[1]],
                ['patch_change', 0, 2, list_of_MIDI_patches[2]],
                ['patch_change', 0, 3, list_of_MIDI_patches[3]],
                ['patch_change', 0, 4, list_of_MIDI_patches[4]],
                ['patch_change', 0, 5, list_of_MIDI_patches[5]],
                ['patch_change', 0, 6, list_of_MIDI_patches[6]],
                ['patch_change', 0, 7, list_of_MIDI_patches[7]],
                ['patch_change', 0, 8, list_of_MIDI_patches[8]],
                ['patch_change', 0, 9, list_of_MIDI_patches[9]],
                ['patch_change', 0, 10, list_of_MIDI_patches[10]],
                ['patch_change', 0, 11, list_of_MIDI_patches[11]],
                ['patch_change', 0, 12, list_of_MIDI_patches[12]],
                ['patch_change', 0, 13, list_of_MIDI_patches[13]],
                ['patch_change', 0, 14, list_of_MIDI_patches[14]],
                ['patch_change', 0, 15, list_of_MIDI_patches[15]],
                ['track_name', 0, bytes('Composition Track', 'utf-8')]]


output = output_header + [patch_list + output_song]

midi_data = MIDI.score2midi(output)
detailed_MIDI_stats = MIDI.score2stats(output)

with open('MIDI_Search' + '.mid', 'wb') as midi_file:
    midi_file.write(midi_data)
    midi_file.close()
print('Done!')

from google.colab import files
files.download('MIDI_Search' + '.mid')
print('Detailed MIDI stats:')
detailed_MIDI_stats

#@title Listen to the last generated composition
#@markdown NOTE: May be very slow with the long compositions
print('Synthesizing the last output MIDI. Please stand-by... ')
FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str('MIDI_Search' + '.mid'), str('MIDI_Search' + '.wav'))
Audio(str('MIDI_Search' + '.wav'), rate=16000)

"""# Congrats! :) You did it :)

## If you like you can create your own dataset here
"""

# Commented out IPython magic to ensure Python compatibility.
#@title Process MIDIs to special MIDI dataset with Tegridy MIDI Processor
#@markdown NOTES:

#@markdown 3) MIDI Channel = -1 means all MIDI channels except the drums (Channel 9) will be processed. MIDI Channel = 16 means all channels will be processed. Otherwise, only one channel will be processed.

full_path_to_output_dataset_to = "/content/MIDI_Search_Dataset.pkl" #@param {type:"string"}
desired_MIDI_channel_to_process = -1 #@param {type:"slider", min:-1, max:15, step:1}

melody_notes_in_chords = True
debug = False 

print('TMIDI Processor')
print('Starting up...')

chords_list = []

###########

average_note_pitch = 0
min_note = 127
max_note = 0

files_count = 0

ev = 0

chords_list_f = []
melody_list_f = []

chords_list = []
chords_count = 0

melody_chords = []
melody_count = 0

song_names = []

###########

def list_average(num):
  sum_num = 0
  for t in num:
      sum_num = sum_num + t           

  avg = sum_num / len(num)
  return avg

###########

print('Loading MIDI files...')
print('This may take a while on a large dataset in particular.')

# %cd /content/
dataset_addr = "/content/Dataset/"
os.chdir(dataset_addr)
filez = os.listdir(dataset_addr)

print('Processing MIDI files. Please wait...')
for f in tqdm.auto.tqdm(filez):
  files_count += 1
  chords_list, melody_list = TMIDI.Tegridy_MIDI_Processor(f, 
                                                     desired_MIDI_channel_to_process, 
                                                     1,
                                                     )
  temp_chords_list = []
  melody_list_final = []

  for notez in melody_list:
    if melody_notes_in_chords:
      temp_chords_list.append([notez])
    melody_list_final.append(notez)
    for chord in chords_list:
      if notez[1] == chord[0][1]:
        temp_chords_list.append(chord[1:])

  chords_list_f.extend(temp_chords_list)
  melody_list_f.extend(melody_list_final)
  chords_count += len(chords_list)
  melody_count += len(melody_list)

average_note_pitch = int((min_note + max_note) / 2)

print('Task complete :)')
print('==================================================')
print('Number of processed dataset MIDI files:', files_count)
print('Average note pitch =', average_note_pitch)
print('Number of MIDI chords recorded:', chords_count)
print('The longest chord:', len(max(chords_list_f, key=len)), 'notes') 
print(max(chords_list_f, key=len))
print('Number of recorded melody events:', len(melody_list_f))
print('First melody event:', melody_list_f[0], 'Last Melody event:', melody_list_f[-1])
print('Total number of MIDI events recorded:', len(chords_list_f))

MusicDataset = [chords_list_f, melody_list_f]

with open(full_path_to_output_dataset_to, 'wb') as filehandle:
    # store the data as binary data stream
    pickle.dump(MusicDataset, filehandle)

print('Dataset was saved at:', full_path_to_output_dataset_to)
print('Task complete. Enjoy! :)')