initial upload attempt

initial upload attempt

system/tts_engines/rvc/train/data_utils.py

@@ -0,0 +1,452 @@
+import os
+import numpy as np
+import torch
+import torch.utils.data
+
+from mel_processing import spectrogram_torch
+from utils import load_filepaths_and_text, load_wav_to_torch
+
+
+class TextAudioLoaderMultiNSFsid(torch.utils.data.Dataset):
+    def __init__(self, hparams):
+        self.audiopaths_and_text = load_filepaths_and_text(hparams.training_files)
+        self.max_wav_value = hparams.max_wav_value
+        self.sampling_rate = hparams.sampling_rate
+        self.filter_length = hparams.filter_length
+        self.hop_length = hparams.hop_length
+        self.win_length = hparams.win_length
+        self.sampling_rate = hparams.sampling_rate
+        self.min_text_len = getattr(hparams, "min_text_len", 1)
+        self.max_text_len = getattr(hparams, "max_text_len", 5000)
+        self._filter()
+
+    def _filter(self):
+        audiopaths_and_text_new = []
+        lengths = []
+        for audiopath, text, pitch, pitchf, dv in self.audiopaths_and_text:
+            if self.min_text_len <= len(text) and len(text) <= self.max_text_len:
+                audiopaths_and_text_new.append([audiopath, text, pitch, pitchf, dv])
+                lengths.append(os.path.getsize(audiopath) // (3 * self.hop_length))
+        self.audiopaths_and_text = audiopaths_and_text_new
+        self.lengths = lengths
+
+    def get_sid(self, sid):
+        sid = torch.LongTensor([int(sid)])
+        return sid
+
+    def get_audio_text_pair(self, audiopath_and_text):
+        file = audiopath_and_text[0]
+        phone = audiopath_and_text[1]
+        pitch = audiopath_and_text[2]
+        pitchf = audiopath_and_text[3]
+        dv = audiopath_and_text[4]
+
+        phone, pitch, pitchf = self.get_labels(phone, pitch, pitchf)
+        spec, wav = self.get_audio(file)
+        dv = self.get_sid(dv)
+
+        len_phone = phone.size()[0]
+        len_spec = spec.size()[-1]
+        if len_phone != len_spec:
+            len_min = min(len_phone, len_spec)
+            len_wav = len_min * self.hop_length
+
+            spec = spec[:, :len_min]
+            wav = wav[:, :len_wav]
+
+            phone = phone[:len_min, :]
+            pitch = pitch[:len_min]
+            pitchf = pitchf[:len_min]
+
+        return (spec, wav, phone, pitch, pitchf, dv)
+
+    def get_labels(self, phone, pitch, pitchf):
+        phone = np.load(phone)
+        phone = np.repeat(phone, 2, axis=0)
+        pitch = np.load(pitch)
+        pitchf = np.load(pitchf)
+        n_num = min(phone.shape[0], 900)
+        phone = phone[:n_num, :]
+        pitch = pitch[:n_num]
+        pitchf = pitchf[:n_num]
+        phone = torch.FloatTensor(phone)
+        pitch = torch.LongTensor(pitch)
+        pitchf = torch.FloatTensor(pitchf)
+        return phone, pitch, pitchf
+
+    def get_audio(self, filename):
+        audio, sampling_rate = load_wav_to_torch(filename)
+        if sampling_rate != self.sampling_rate:
+            raise ValueError(
+                "{} SR doesn't match target {} SR".format(
+                    sampling_rate, self.sampling_rate
+                )
+            )
+        audio_norm = audio
+        audio_norm = audio_norm.unsqueeze(0)
+        spec_filename = filename.replace(".wav", ".spec.pt")
+        if os.path.exists(spec_filename):
+            try:
+                spec = torch.load(spec_filename)
+            except Exception as error:
+                print(f"{spec_filename}: {error}")
+                spec = spectrogram_torch(
+                    audio_norm,
+                    self.filter_length,
+                    self.hop_length,
+                    self.win_length,
+                    center=False,
+                )
+                spec = torch.squeeze(spec, 0)
+                torch.save(spec, spec_filename, _use_new_zipfile_serialization=False)
+        else:
+            spec = spectrogram_torch(
+                audio_norm,
+                self.filter_length,
+                self.hop_length,
+                self.win_length,
+                center=False,
+            )
+            spec = torch.squeeze(spec, 0)
+            torch.save(spec, spec_filename, _use_new_zipfile_serialization=False)
+        return spec, audio_norm
+
+    def __getitem__(self, index):
+        return self.get_audio_text_pair(self.audiopaths_and_text[index])
+
+    def __len__(self):
+        return len(self.audiopaths_and_text)
+
+
+class TextAudioCollateMultiNSFsid:
+    def __init__(self, return_ids=False):
+        self.return_ids = return_ids
+
+    def __call__(self, batch):
+        _, ids_sorted_decreasing = torch.sort(
+            torch.LongTensor([x[0].size(1) for x in batch]), dim=0, descending=True
+        )
+
+        max_spec_len = max([x[0].size(1) for x in batch])
+        max_wave_len = max([x[1].size(1) for x in batch])
+        spec_lengths = torch.LongTensor(len(batch))
+        wave_lengths = torch.LongTensor(len(batch))
+        spec_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max_spec_len)
+        wave_padded = torch.FloatTensor(len(batch), 1, max_wave_len)
+        spec_padded.zero_()
+        wave_padded.zero_()
+
+        max_phone_len = max([x[2].size(0) for x in batch])
+        phone_lengths = torch.LongTensor(len(batch))
+        phone_padded = torch.FloatTensor(
+            len(batch), max_phone_len, batch[0][2].shape[1]
+        )
+        pitch_padded = torch.LongTensor(len(batch), max_phone_len)
+        pitchf_padded = torch.FloatTensor(len(batch), max_phone_len)
+        phone_padded.zero_()
+        pitch_padded.zero_()
+        pitchf_padded.zero_()
+        sid = torch.LongTensor(len(batch))
+
+        for i in range(len(ids_sorted_decreasing)):
+            row = batch[ids_sorted_decreasing[i]]
+
+            spec = row[0]
+            spec_padded[i, :, : spec.size(1)] = spec
+            spec_lengths[i] = spec.size(1)
+
+            wave = row[1]
+            wave_padded[i, :, : wave.size(1)] = wave
+            wave_lengths[i] = wave.size(1)
+
+            phone = row[2]
+            phone_padded[i, : phone.size(0), :] = phone
+            phone_lengths[i] = phone.size(0)
+
+            pitch = row[3]
+            pitch_padded[i, : pitch.size(0)] = pitch
+            pitchf = row[4]
+            pitchf_padded[i, : pitchf.size(0)] = pitchf
+
+            sid[i] = row[5]
+
+        return (
+            phone_padded,
+            phone_lengths,
+            pitch_padded,
+            pitchf_padded,
+            spec_padded,
+            spec_lengths,
+            wave_padded,
+            wave_lengths,
+            sid,
+        )
+
+
+class TextAudioLoader(torch.utils.data.Dataset):
+    def __init__(self, hparams):
+        self.audiopaths_and_text = load_filepaths_and_text(hparams.training_files)
+        self.max_wav_value = hparams.max_wav_value
+        self.sampling_rate = hparams.sampling_rate
+        self.filter_length = hparams.filter_length
+        self.hop_length = hparams.hop_length
+        self.win_length = hparams.win_length
+        self.sampling_rate = hparams.sampling_rate
+        self.min_text_len = getattr(hparams, "min_text_len", 1)
+        self.max_text_len = getattr(hparams, "max_text_len", 5000)
+        self._filter()
+
+    def _filter(self):
+        audiopaths_and_text_new = []
+        lengths = []
+        for entry in self.audiopaths_and_text:
+            if len(entry) >= 3:
+                audiopath, text, dv = entry[:3]
+                if self.min_text_len <= len(text) and len(text) <= self.max_text_len:
+                    audiopaths_and_text_new.append([audiopath, text, dv])
+                    lengths.append(os.path.getsize(audiopath) // (3 * self.hop_length))
+
+        self.audiopaths_and_text = audiopaths_and_text_new
+        self.lengths = lengths
+
+    def get_sid(self, sid):
+        sid = os.path.basename(os.path.dirname(sid))
+
+        try:
+            sid = torch.LongTensor([int("".join(filter(str.isdigit, sid)))])
+        except ValueError as error:
+            print(f"Error converting speaker ID '{sid}' to integer. Exception: {error}")
+            sid = torch.LongTensor([0])
+
+        return sid
+
+    def get_audio_text_pair(self, audiopath_and_text):
+        file = audiopath_and_text[0]
+        phone = audiopath_and_text[1]
+        dv = audiopath_and_text[2]
+
+        phone = self.get_labels(phone)
+        spec, wav = self.get_audio(file)
+        dv = self.get_sid(dv)
+
+        len_phone = phone.size()[0]
+        len_spec = spec.size()[-1]
+        if len_phone != len_spec:
+            len_min = min(len_phone, len_spec)
+            len_wav = len_min * self.hop_length
+            spec = spec[:, :len_min]
+            wav = wav[:, :len_wav]
+            phone = phone[:len_min, :]
+        return (spec, wav, phone, dv)
+
+    def get_labels(self, phone):
+        phone = np.load(phone)
+        phone = np.repeat(phone, 2, axis=0)
+        n_num = min(phone.shape[0], 900)
+        phone = phone[:n_num, :]
+        phone = torch.FloatTensor(phone)
+        return phone
+
+    def get_audio(self, filename):
+        audio, sampling_rate = load_wav_to_torch(filename)
+        if sampling_rate != self.sampling_rate:
+            raise ValueError(
+                "{} SR doesn't match target {} SR".format(
+                    sampling_rate, self.sampling_rate
+                )
+            )
+        audio_norm = audio
+        audio_norm = audio_norm.unsqueeze(0)
+        spec_filename = filename.replace(".wav", ".spec.pt")
+        if os.path.exists(spec_filename):
+            try:
+                spec = torch.load(spec_filename)
+            except Exception as error:
+                print(f"{spec_filename}: {error}")
+                spec = spectrogram_torch(
+                    audio_norm,
+                    self.filter_length,
+                    self.hop_length,
+                    self.win_length,
+                    center=False,
+                )
+                spec = torch.squeeze(spec, 0)
+                torch.save(spec, spec_filename, _use_new_zipfile_serialization=False)
+        else:
+            spec = spectrogram_torch(
+                audio_norm,
+                self.filter_length,
+                self.hop_length,
+                self.win_length,
+                center=False,
+            )
+            spec = torch.squeeze(spec, 0)
+            torch.save(spec, spec_filename, _use_new_zipfile_serialization=False)
+        return spec, audio_norm
+
+    def __getitem__(self, index):
+        return self.get_audio_text_pair(self.audiopaths_and_text[index])
+
+    def __len__(self):
+        return len(self.audiopaths_and_text)
+
+
+class TextAudioCollate:
+    def __init__(self, return_ids=False):
+        self.return_ids = return_ids
+
+    def __call__(self, batch):
+        _, ids_sorted_decreasing = torch.sort(
+            torch.LongTensor([x[0].size(1) for x in batch]), dim=0, descending=True
+        )
+
+        max_spec_len = max([x[0].size(1) for x in batch])
+        max_wave_len = max([x[1].size(1) for x in batch])
+        spec_lengths = torch.LongTensor(len(batch))
+        wave_lengths = torch.LongTensor(len(batch))
+        spec_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max_spec_len)
+        wave_padded = torch.FloatTensor(len(batch), 1, max_wave_len)
+        spec_padded.zero_()
+        wave_padded.zero_()
+
+        max_phone_len = max([x[2].size(0) for x in batch])
+        phone_lengths = torch.LongTensor(len(batch))
+        phone_padded = torch.FloatTensor(
+            len(batch), max_phone_len, batch[0][2].shape[1]
+        )
+        phone_padded.zero_()
+        sid = torch.LongTensor(len(batch))
+
+        for i in range(len(ids_sorted_decreasing)):
+            row = batch[ids_sorted_decreasing[i]]
+
+            spec = row[0]
+            spec_padded[i, :, : spec.size(1)] = spec
+            spec_lengths[i] = spec.size(1)
+
+            wave = row[1]
+            wave_padded[i, :, : wave.size(1)] = wave
+            wave_lengths[i] = wave.size(1)
+
+            phone = row[2]
+            phone_padded[i, : phone.size(0), :] = phone
+            phone_lengths[i] = phone.size(0)
+
+            sid[i] = row[3]
+
+        return (
+            phone_padded,
+            phone_lengths,
+            spec_padded,
+            spec_lengths,
+            wave_padded,
+            wave_lengths,
+            sid,
+        )
+
+
+class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
+    def __init__(
+        self,
+        dataset,
+        batch_size,
+        boundaries,
+        num_replicas=None,
+        rank=None,
+        shuffle=True,
+    ):
+        super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
+        self.lengths = dataset.lengths
+        self.batch_size = batch_size
+        self.boundaries = boundaries
+
+        self.buckets, self.num_samples_per_bucket = self._create_buckets()
+        self.total_size = sum(self.num_samples_per_bucket)
+        self.num_samples = self.total_size // self.num_replicas
+
+    def _create_buckets(self):
+        buckets = [[] for _ in range(len(self.boundaries) - 1)]
+        for i in range(len(self.lengths)):
+            length = self.lengths[i]
+            idx_bucket = self._bisect(length)
+            if idx_bucket != -1:
+                buckets[idx_bucket].append(i)
+
+        for i in range(len(buckets) - 1, -1, -1):  #
+            if len(buckets[i]) == 0:
+                buckets.pop(i)
+                self.boundaries.pop(i + 1)
+
+        num_samples_per_bucket = []
+        for i in range(len(buckets)):
+            len_bucket = len(buckets[i])
+            total_batch_size = self.num_replicas * self.batch_size
+            rem = (
+                total_batch_size - (len_bucket % total_batch_size)
+            ) % total_batch_size
+            num_samples_per_bucket.append(len_bucket + rem)
+        return buckets, num_samples_per_bucket
+
+    def __iter__(self):
+        g = torch.Generator()
+        g.manual_seed(self.epoch)
+
+        indices = []
+        if self.shuffle:
+            for bucket in self.buckets:
+                indices.append(torch.randperm(len(bucket), generator=g).tolist())
+        else:
+            for bucket in self.buckets:
+                indices.append(list(range(len(bucket))))
+
+        batches = []
+        for i in range(len(self.buckets)):
+            bucket = self.buckets[i]
+            len_bucket = len(bucket)
+            ids_bucket = indices[i]
+            num_samples_bucket = self.num_samples_per_bucket[i]
+
+            rem = num_samples_bucket - len_bucket
+            ids_bucket = (
+                ids_bucket
+                + ids_bucket * (rem // len_bucket)
+                + ids_bucket[: (rem % len_bucket)]
+            )
+
+            ids_bucket = ids_bucket[self.rank :: self.num_replicas]
+
+            # batching
+            for j in range(len(ids_bucket) // self.batch_size):
+                batch = [
+                    bucket[idx]
+                    for idx in ids_bucket[
+                        j * self.batch_size : (j + 1) * self.batch_size
+                    ]
+                ]
+                batches.append(batch)
+
+        if self.shuffle:
+            batch_ids = torch.randperm(len(batches), generator=g).tolist()
+            batches = [batches[i] for i in batch_ids]
+        self.batches = batches
+
+        assert len(self.batches) * self.batch_size == self.num_samples
+        return iter(self.batches)
+
+    def _bisect(self, x, lo=0, hi=None):
+        if hi is None:
+            hi = len(self.boundaries) - 1
+
+        if hi > lo:
+            mid = (hi + lo) // 2
+            if self.boundaries[mid] < x and x <= self.boundaries[mid + 1]:
+                return mid
+            elif x <= self.boundaries[mid]:
+                return self._bisect(x, lo, mid)
+            else:
+                return self._bisect(x, mid + 1, hi)
+        else:
+            return -1
+
+    def __len__(self):
+        return self.num_samples // self.batch_size

system/tts_engines/rvc/train/extract/extract_f0_print.py

@@ -0,0 +1,237 @@
+import os
+import sys
+import numpy as np
+import pyworld
+import torchcrepe
+import torch
+import parselmouth
+import tqdm
+from multiprocessing import Process, cpu_count
+
+current_directory = os.getcwd()
+sys.path.append(current_directory)
+
+
+from rvc.lib.utils import load_audio
+
+
+exp_dir = sys.argv[1]
+f0_method = sys.argv[2]
+num_processes = cpu_count()
+
+try:
+    hop_length = int(sys.argv[3])
+except ValueError:
+    hop_length = 128
+
+DoFormant = False
+Quefrency = 1.0
+Timbre = 1.0
+
+
+class FeatureInput:
+    def __init__(self, sample_rate=16000, hop_size=160):
+        self.fs = sample_rate
+        self.hop = hop_size
+
+        self.f0_method_dict = self.get_f0_method_dict()
+
+        self.f0_bin = 256
+        self.f0_max = 1100.0
+        self.f0_min = 50.0
+        self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700)
+        self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700)
+
+    def mncrepe(self, method, x, p_len, hop_length):
+        f0 = None
+        torch_device_index = 0
+        torch_device = (
+            torch.device(f"cuda:{torch_device_index % torch.cuda.device_count()}")
+            if torch.cuda.is_available()
+            else (
+                torch.device("mps")
+                if torch.backends.mps.is_available()
+                else torch.device("cpu")
+            )
+        )
+
+        audio = torch.from_numpy(x.astype(np.float32)).to(torch_device, copy=True)
+        audio /= torch.quantile(torch.abs(audio), 0.999)
+        audio = torch.unsqueeze(audio, dim=0)
+        if audio.ndim == 2 and audio.shape[0] > 1:
+            audio = torch.mean(audio, dim=0, keepdim=True).detach()
+        audio = audio.detach()
+
+        if method == "crepe":
+            pitch = torchcrepe.predict(
+                audio,
+                self.fs,
+                hop_length,
+                self.f0_min,
+                self.f0_max,
+                "full",
+                batch_size=hop_length * 2,
+                device=torch_device,
+                pad=True,
+            )
+            p_len = p_len or x.shape[0] // hop_length
+            source = np.array(pitch.squeeze(0).cpu().float().numpy())
+            source[source < 0.001] = np.nan
+            target = np.interp(
+                np.arange(0, len(source) * p_len, len(source)) / p_len,
+                np.arange(0, len(source)),
+                source,
+            )
+            f0 = np.nan_to_num(target)
+
+        return f0
+
+    def get_pm(self, x, p_len):
+        f0 = (
+            parselmouth.Sound(x, self.fs)
+            .to_pitch_ac(
+                time_step=160 / 16000,
+                voicing_threshold=0.6,
+                pitch_floor=self.f0_min,
+                pitch_ceiling=self.f0_max,
+            )
+            .selected_array["frequency"]
+        )
+
+        return np.pad(
+            f0,
+            [
+                [
+                    max(0, (p_len - len(f0) + 1) // 2),
+                    max(0, p_len - len(f0) - (p_len - len(f0) + 1) // 2),
+                ]
+            ],
+            mode="constant",
+        )
+
+    def get_harvest(self, x):
+        f0_spectral = pyworld.harvest(
+            x.astype(np.double),
+            fs=self.fs,
+            f0_ceil=self.f0_max,
+            f0_floor=self.f0_min,
+            frame_period=1000 * self.hop / self.fs,
+        )
+        return pyworld.stonemask(x.astype(np.double), *f0_spectral, self.fs)
+
+    def get_dio(self, x):
+        f0_spectral = pyworld.dio(
+            x.astype(np.double),
+            fs=self.fs,
+            f0_ceil=self.f0_max,
+            f0_floor=self.f0_min,
+            frame_period=1000 * self.hop / self.fs,
+        )
+        return pyworld.stonemask(x.astype(np.double), *f0_spectral, self.fs)
+
+    def get_rmvpe(self, x):
+        if not hasattr(self, "model_rmvpe"):
+            from rvc.lib.rmvpe import RMVPE
+
+            self.model_rmvpe = RMVPE(current_directory / "models" / "rvc_base" / "rmvpe.pt", is_half=False, device="cpu")
+        return self.model_rmvpe.infer_from_audio(x, thred=0.03)
+
+    def get_f0_method_dict(self):
+        return {
+            "pm": self.get_pm,
+            "harvest": self.get_harvest,
+            "dio": self.get_dio,
+            "rmvpe": self.get_rmvpe,
+        }
+
+    def compute_f0(self, path, f0_method, hop_length):
+        x = load_audio(path, self.fs)
+        p_len = x.shape[0] // self.hop
+
+        if f0_method in self.f0_method_dict:
+            f0 = (
+                self.f0_method_dict[f0_method](x, p_len)
+                if f0_method == "pm"
+                else self.f0_method_dict[f0_method](x)
+            )
+        elif f0_method == "crepe":
+            f0 = self.mncrepe(f0_method, x, p_len, hop_length)
+        return f0
+
+    def coarse_f0(self, f0):
+        f0_mel = 1127 * np.log(1 + f0 / 700)
+        f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - self.f0_mel_min) * (
+            self.f0_bin - 2
+        ) / (self.f0_mel_max - self.f0_mel_min) + 1
+
+        # use 0 or 1
+        f0_mel[f0_mel <= 1] = 1
+        f0_mel[f0_mel > self.f0_bin - 1] = self.f0_bin - 1
+        f0_coarse = np.rint(f0_mel).astype(int)
+        assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (
+            f0_coarse.max(),
+            f0_coarse.min(),
+        )
+        return f0_coarse
+
+    def process_paths(self, paths, f0_method, hop_length, thread_n):
+        if len(paths) == 0:
+            print("There are no paths to process.")
+            return
+        with tqdm.tqdm(total=len(paths), leave=True, position=thread_n) as pbar:
+            description = f"Thread {thread_n} | Hop-Length {hop_length}"
+            pbar.set_description(description)
+
+            for idx, (inp_path, opt_path1, opt_path2) in enumerate(paths):
+                try:
+                    if os.path.exists(opt_path1 + ".npy") and os.path.exists(
+                        opt_path2 + ".npy"
+                    ):
+                        pbar.update(1)
+                        continue
+
+                    feature_pit = self.compute_f0(inp_path, f0_method, hop_length)
+                    np.save(
+                        opt_path2,
+                        feature_pit,
+                        allow_pickle=False,
+                    )  # nsf
+                    coarse_pit = self.coarse_f0(feature_pit)
+                    np.save(
+                        opt_path1,
+                        coarse_pit,
+                        allow_pickle=False,
+                    )  # ori
+                    pbar.update(1)
+                except Exception as error:
+                    print(f"f0fail-{idx}-{inp_path}-{error}")
+
+
+if __name__ == "__main__":
+    feature_input = FeatureInput()
+    paths = []
+    input_root = f"{exp_dir}/1_16k_wavs"
+    output_root1 = f"{exp_dir}/2a_f0"
+    output_root2 = f"{exp_dir}/2b-f0nsf"
+
+    os.makedirs(output_root1, exist_ok=True)
+    os.makedirs(output_root2, exist_ok=True)
+    for name in sorted(list(os.listdir(input_root))):
+        input_path = f"{input_root}/{name}"
+        if "spec" in input_path:
+            continue
+        output_path1 = f"{output_root1}/{name}"
+        output_path2 = f"{output_root2}/{name}"
+        paths.append([input_path, output_path1, output_path2])
+
+    processes = []
+    print("Using f0 method: " + f0_method)
+    for i in range(num_processes):
+        p = Process(
+            target=feature_input.process_paths,
+            args=(paths[i::num_processes], f0_method, hop_length, i),
+        )
+        processes.append(p)
+        p.start()
+    for i in range(num_processes):
+        processes[i].join()

system/tts_engines/rvc/train/extract/extract_feature_print.py

@@ -0,0 +1,94 @@
+import os
+import sys
+import tqdm
+import torch
+import torch.nn.functional as F
+import soundfile as sf
+import numpy as np
+
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+from rvc.lib.utils import load_embedding
+
+device = sys.argv[1]
+n_parts = int(sys.argv[2])
+i_part = int(sys.argv[3])
+i_gpu = sys.argv[4]
+exp_dir = sys.argv[5]
+os.environ["CUDA_VISIBLE_DEVICES"] = str(i_gpu)
+version = sys.argv[6]
+is_half = bool(sys.argv[7])
+embedder_model = sys.argv[8]
+
+
+wav_path = f"{exp_dir}/1_16k_wavs"
+out_path = f"{exp_dir}/3_feature256" if version == "v1" else f"{exp_dir}/3_feature768"
+os.makedirs(out_path, exist_ok=True)
+
+
+def read_wave(wav_path, normalize=False):
+    wav, sr = sf.read(wav_path)
+    assert sr == 16000
+    feats = torch.from_numpy(wav)
+    feats = feats.half() if is_half else feats.float()
+    feats = feats.mean(-1) if feats.dim() == 2 else feats
+    feats = feats.view(1, -1)
+    if normalize:
+        with torch.no_grad():
+            feats = F.layer_norm(feats, feats.shape)
+    return feats
+
+
+print("Starting feature extraction...")
+models, saved_cfg, task = load_embedding(embedder_model)
+model = models[0]
+model = model.to(device)
+if device not in ["mps", "cpu"]:
+    model = model.half()
+model.eval()
+
+todo = sorted(os.listdir(wav_path))[i_part::n_parts]
+n = max(1, len(todo) // 10)
+
+if len(todo) == 0:
+    print(
+        "An error occurred in the feature extraction, make sure you have provided the audios correctly."
+    )
+else:
+    print(f"{len(todo)}")
+    with tqdm.tqdm(total=len(todo)) as pbar:
+        for idx, file in enumerate(todo):
+            try:
+                if file.endswith(".wav"):
+                    wav_file_path = os.path.join(wav_path, file)
+                    out_file_path = os.path.join(out_path, file.replace("wav", "npy"))
+
+                    if os.path.exists(out_file_path):
+                        continue
+
+                    feats = read_wave(wav_file_path, normalize=saved_cfg.task.normalize)
+                    padding_mask = torch.BoolTensor(feats.shape).fill_(False)
+                    inputs = {
+                        "source": feats.to(device),
+                        "padding_mask": padding_mask.to(device),
+                        "output_layer": 9 if version == "v1" else 12,
+                    }
+                    with torch.no_grad():
+                        logits = model.extract_features(**inputs)
+                        feats = (
+                            model.final_proj(logits[0])
+                            if version == "v1"
+                            else logits[0]
+                        )
+
+                    feats = feats.squeeze(0).float().cpu().numpy()
+                    if np.isnan(feats).sum() == 0:
+                        np.save(out_file_path, feats, allow_pickle=False)
+                    else:
+                        print(f"{file} - contains nan")
+                    pbar.set_description(f"Processing {file} {feats.shape}")
+            except Exception as error:
+                print(error)
+            pbar.update(1)
+
+    print("Feature extraction completed successfully!")

system/tts_engines/rvc/train/extract/preparing_files.py

@@ -0,0 +1,71 @@
+import os
+import json
+import pathlib
+from random import shuffle
+
+from rvc.configs.config import Config
+
+config = Config()
+current_directory = os.getcwd()
+
+
+def generate_config(rvc_version, sampling_rate, model_path):
+    if rvc_version == "v1" or sampling_rate == "40000":
+        config_path = f"v1/{sampling_rate}.json"
+    else:
+        config_path = f"v2/{sampling_rate}.json"
+    config_save_path = os.path.join(model_path, "config.json")
+    if not pathlib.Path(config_save_path).exists():
+        with open(config_save_path, "w", encoding="utf-8") as f:
+            json.dump(
+                config.json_config[config_path],
+                f,
+                ensure_ascii=False,
+                indent=4,
+                sort_keys=True,
+            )
+            f.write("\n")
+
+
+def generate_filelist(f0_method, model_path, rvc_version, sampling_rate):
+    gt_wavs_dir = f"{model_path}/0_gt_wavs"
+    feature_dir = (
+        f"{model_path}/3_feature256"
+        if rvc_version == "v1"
+        else f"{model_path}/3_feature768"
+    )
+    if f0_method:
+        f0_dir = f"{model_path}/2a_f0"
+        f0nsf_dir = f"{model_path}/2b-f0nsf"
+        names = (
+            set([name.split(".")[0] for name in os.listdir(gt_wavs_dir)])
+            & set([name.split(".")[0] for name in os.listdir(feature_dir)])
+            & set([name.split(".")[0] for name in os.listdir(f0_dir)])
+            & set([name.split(".")[0] for name in os.listdir(f0nsf_dir)])
+        )
+    else:
+        names = set([name.split(".")[0] for name in os.listdir(gt_wavs_dir)]) & set(
+            [name.split(".")[0] for name in os.listdir(feature_dir)]
+        )
+    options = []
+    for name in names:
+        if f0_method:
+            options.append(
+                f"{gt_wavs_dir}/{name}.wav|{feature_dir}/{name}.npy|{f0_dir}/{name}.wav.npy|{f0nsf_dir}/{name}.wav.npy|0"
+            )
+        else:
+            options.append(f"{gt_wavs_dir}/{name}.wav|{feature_dir}/{name}.npy|0")
+    fea_dim = 256 if rvc_version == "v1" else 768
+    if f0_method:
+        for _ in range(2):
+            options.append(
+                f"{current_directory}/logs/mute/0_gt_wavs/mute{sampling_rate}.wav|{current_directory}/logs/mute/3_feature{fea_dim}/mute.npy|{current_directory}/logs/mute/2a_f0/mute.wav.npy|{current_directory}/logs/mute/2b-f0nsf/mute.wav.npy|0"
+            )
+    else:
+        for _ in range(2):
+            options.append(
+                f"{current_directory}/logs/mute/0_gt_wavs/mute{sampling_rate}.wav|{current_directory}/logs/mute/3_feature{fea_dim}/mute.npy|0"
+            )
+    shuffle(options)
+    with open(f"{model_path}/filelist.txt", "w") as f:
+        f.write("\n".join(options))

system/tts_engines/rvc/train/losses.py

@@ -0,0 +1,58 @@
+import torch
+
+
+def feature_loss(fmap_r, fmap_g):
+    loss = 0
+    for dr, dg in zip(fmap_r, fmap_g):
+        for rl, gl in zip(dr, dg):
+            rl = rl.float().detach()
+            gl = gl.float()
+            loss += torch.mean(torch.abs(rl - gl))
+
+    return loss * 2
+
+
+def discriminator_loss(disc_real_outputs, disc_generated_outputs):
+    loss = 0
+    r_losses = []
+    g_losses = []
+    for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
+        dr = dr.float()
+        dg = dg.float()
+        r_loss = torch.mean((1 - dr) ** 2)
+        g_loss = torch.mean(dg**2)
+        loss += r_loss + g_loss
+        r_losses.append(r_loss.item())
+        g_losses.append(g_loss.item())
+
+    return loss, r_losses, g_losses
+
+
+def generator_loss(disc_outputs):
+    loss = 0
+    gen_losses = []
+    for dg in disc_outputs:
+        dg = dg.float()
+        l = torch.mean((1 - dg) ** 2)
+        gen_losses.append(l)
+        loss += l
+
+    return loss, gen_losses
+
+
+def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
+    """
+    z_p, logs_q: [b, h, t_t]
+    m_p, logs_p: [b, h, t_t]
+    """
+    z_p = z_p.float()
+    logs_q = logs_q.float()
+    m_p = m_p.float()
+    logs_p = logs_p.float()
+    z_mask = z_mask.float()
+
+    kl = logs_p - logs_q - 0.5
+    kl += 0.5 * ((z_p - m_p) ** 2) * torch.exp(-2.0 * logs_p)
+    kl = torch.sum(kl * z_mask)
+    l = kl / torch.sum(z_mask)
+    return l

system/tts_engines/rvc/train/mel_processing.py

@@ -0,0 +1,84 @@
+import torch
+import torch.utils.data
+from librosa.filters import mel as librosa_mel_fn
+
+
+def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
+    return torch.log(torch.clamp(x, min=clip_val) * C)
+
+
+def dynamic_range_decompression_torch(x, C=1):
+    return torch.exp(x) / C
+
+
+def spectral_normalize_torch(magnitudes):
+    return dynamic_range_compression_torch(magnitudes)
+
+
+def spectral_de_normalize_torch(magnitudes):
+    return dynamic_range_decompression_torch(magnitudes)
+
+
+mel_basis = {}
+hann_window = {}
+
+
+def spectrogram_torch(y, n_fft, hop_size, win_size, center=False):
+    global hann_window
+    dtype_device = str(y.dtype) + "_" + str(y.device)
+    wnsize_dtype_device = str(win_size) + "_" + dtype_device
+    if wnsize_dtype_device not in hann_window:
+        hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
+            dtype=y.dtype, device=y.device
+        )
+
+    y = torch.nn.functional.pad(
+        y.unsqueeze(1),
+        (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
+        mode="reflect",
+    )
+    y = y.squeeze(1)
+
+    spec = torch.stft(
+        y,
+        n_fft,
+        hop_length=hop_size,
+        win_length=win_size,
+        window=hann_window[wnsize_dtype_device],
+        center=center,
+        pad_mode="reflect",
+        normalized=False,
+        onesided=True,
+        return_complex=True,
+    )
+
+    spec = torch.sqrt(spec.real.pow(2) + spec.imag.pow(2) + 1e-6)
+
+    return spec
+
+
+def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
+    global mel_basis
+    dtype_device = str(spec.dtype) + "_" + str(spec.device)
+    fmax_dtype_device = str(fmax) + "_" + dtype_device
+    if fmax_dtype_device not in mel_basis:
+        mel = librosa_mel_fn(
+            sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax
+        )
+        mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
+            dtype=spec.dtype, device=spec.device
+        )
+
+    melspec = torch.matmul(mel_basis[fmax_dtype_device], spec)
+    melspec = spectral_normalize_torch(melspec)
+    return melspec
+
+
+def mel_spectrogram_torch(
+    y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False
+):
+    spec = spectrogram_torch(y, n_fft, hop_size, win_size, center)
+
+    melspec = spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax)
+
+    return melspec

system/tts_engines/rvc/train/preprocess/preprocess.py

@@ -0,0 +1,130 @@
+from multiprocessing import cpu_count
+import os
+import sys
+
+from scipy import signal
+from scipy.io import wavfile
+import librosa
+import numpy as np
+
+now_directory = os.getcwd()
+sys.path.append(now_directory)
+
+from rvc.lib.utils import load_audio
+from rvc.train.slicer import Slicer
+
+experiment_directory = sys.argv[1]
+input_root = sys.argv[2]
+sampling_rate = int(sys.argv[3])
+percentage = float(sys.argv[4])
+num_processes = cpu_count()
+
+import multiprocessing
+
+
+class PreProcess:
+    def __init__(self, sr, exp_dir, per=3.0):
+        self.slicer = Slicer(
+            sr=sr,
+            threshold=-42,
+            min_length=1500,
+            min_interval=400,
+            hop_size=15,
+            max_sil_kept=500,
+        )
+        self.sr = sr
+        self.b_high, self.a_high = signal.butter(N=5, Wn=48, btype="high", fs=self.sr)
+        self.per = per
+        self.overlap = 0.3
+        self.tail = self.per + self.overlap
+        self.max_amplitude = 0.9
+        self.alpha = 0.75
+        self.exp_dir = exp_dir
+        self.gt_wavs_dir = f"{exp_dir}/0_gt_wavs"
+        self.wavs16k_dir = f"{exp_dir}/1_16k_wavs"
+        os.makedirs(self.exp_dir, exist_ok=True)
+        os.makedirs(self.gt_wavs_dir, exist_ok=True)
+        os.makedirs(self.wavs16k_dir, exist_ok=True)
+
+    def normalize_and_write(self, tmp_audio, idx0, idx1):
+        tmp_max = np.abs(tmp_audio).max()
+        if tmp_max > 2.5:
+            print(f"{idx0}-{idx1}-{tmp_max}-filtered")
+            return
+        tmp_audio = (tmp_audio / tmp_max * (self.max_amplitude * self.alpha)) + (
+            1 - self.alpha
+        ) * tmp_audio
+        wavfile.write(
+            f"{self.gt_wavs_dir}/{idx0}_{idx1}.wav",
+            self.sr,
+            tmp_audio.astype(np.float32),
+        )
+        tmp_audio = librosa.resample(
+            tmp_audio, orig_sr=self.sr, target_sr=16000
+        )  # , res_type="soxr_vhq"
+        wavfile.write(
+            f"{self.wavs16k_dir}/{idx0}_{idx1}.wav",
+            16000,
+            tmp_audio.astype(np.float32),
+        )
+
+    def process_audio(self, path, idx0):
+        try:
+            audio = load_audio(path, self.sr)
+            audio = signal.lfilter(self.b_high, self.a_high, audio)
+
+            idx1 = 0
+            for audio_segment in self.slicer.slice(audio):
+                i = 0
+                while 1:
+                    start = int(self.sr * (self.per - self.overlap) * i)
+                    i += 1
+                    if len(audio_segment[start:]) > self.tail * self.sr:
+                        tmp_audio = audio_segment[
+                            start : start + int(self.per * self.sr)
+                        ]
+                        self.normalize_and_write(tmp_audio, idx0, idx1)
+                        idx1 += 1
+                    else:
+                        tmp_audio = audio_segment[start:]
+                        idx1 += 1
+                        break
+                self.normalize_and_write(tmp_audio, idx0, idx1)
+        except Exception as error:
+            print(f"{path}: {error}")
+
+    def process_audio_multiprocessing(self, infos):
+        for path, idx0 in infos:
+            self.process_audio(path, idx0)
+
+    def process_audio_multiprocessing_input_directory(self, input_root, num_processes):
+        try:
+            infos = [
+                (f"{input_root}/{name}", idx)
+                for idx, name in enumerate(sorted(list(os.listdir(input_root))))
+            ]
+            processes = []
+            for i in range(num_processes):
+                p = multiprocessing.Process(
+                    target=self.process_audio_multiprocessing,
+                    args=(infos[i::num_processes],),
+                )
+                processes.append(p)
+                p.start()
+            for i in range(num_processes):
+                processes[i].join()
+        except Exception as error:
+            print(error)
+
+
+def preprocess_training_set(input_root, sr, num_processes, exp_dir, per):
+    pp = PreProcess(sr, exp_dir, per)
+    print("Starting preprocessing...")
+    pp.process_audio_multiprocessing_input_directory(input_root, num_processes)
+    print("Preprocessing completed!")
+
+
+if __name__ == "__main__":
+    preprocess_training_set(
+        input_root, sampling_rate, num_processes, experiment_directory, percentage
+    )

system/tts_engines/rvc/train/process/change_info.py

@@ -0,0 +1,14 @@
+import os
+import torch
+
+
+def change_info(path, info, name):
+    try:
+        ckpt = torch.load(path, map_location="cpu")
+        ckpt["info"] = info
+        if name == "":
+            name = os.path.basename(path)
+        torch.save(ckpt, f"logs/{name}/{name}")
+        return "Success."
+    except Exception as error:
+        print(error)

system/tts_engines/rvc/train/process/extract_index.py

@@ -0,0 +1,88 @@
+import os
+import sys
+import faiss
+import numpy as np
+from sklearn.cluster import MiniBatchKMeans
+from multiprocessing import cpu_count
+
+exp_dir = sys.argv[1]
+version = sys.argv[2]
+
+try:
+    if version == "v1":
+        feature_dir = os.path.join(exp_dir, "3_feature256")
+    elif version == "v2":
+        feature_dir = os.path.join(exp_dir, "3_feature768")
+
+    npys = []
+    listdir_res = sorted(os.listdir(feature_dir))
+
+    for name in listdir_res:
+        file_path = os.path.join(feature_dir, name)
+        phone = np.load(file_path)
+        npys.append(phone)
+
+    big_npy = np.concatenate(npys, axis=0)
+
+    big_npy_idx = np.arange(big_npy.shape[0])
+    np.random.shuffle(big_npy_idx)
+    big_npy = big_npy[big_npy_idx]
+
+    if big_npy.shape[0] > 2e5:
+        big_npy = (
+            MiniBatchKMeans(
+                n_clusters=10000,
+                verbose=True,
+                batch_size=256 * cpu_count(),
+                compute_labels=False,
+                init="random",
+            )
+            .fit(big_npy)
+            .cluster_centers_
+        )
+
+    np.save(os.path.join(exp_dir, "total_fea.npy"), big_npy)
+
+    n_ivf = min(int(16 * np.sqrt(big_npy.shape[0])), big_npy.shape[0] // 39)
+
+    # index_trained
+    index_trained = faiss.index_factory(
+        256 if version == "v1" else 768, f"IVF{n_ivf},Flat"
+    )
+    index_ivf_trained = faiss.extract_index_ivf(index_trained)
+    index_ivf_trained.nprobe = 1
+    index_trained.train(big_npy)
+
+    index_filename_trained = (
+        f"trained_IVF{n_ivf}_Flat_nprobe_{index_ivf_trained.nprobe}_{version}.index"
+    )
+    index_filepath_trained = os.path.join(exp_dir, index_filename_trained)
+
+    faiss.write_index(index_trained, index_filepath_trained)
+
+    # index_added
+    index_added = faiss.index_factory(
+        256 if version == "v1" else 768, f"IVF{n_ivf},Flat"
+    )
+    index_ivf_added = faiss.extract_index_ivf(index_added)
+    index_ivf_added.nprobe = 1
+    index_added.train(big_npy)
+
+    index_filename_added = (
+        f"added_IVF{n_ivf}_Flat_nprobe_{index_ivf_added.nprobe}_{version}.index"
+    )
+    index_filepath_added = os.path.join(exp_dir, index_filename_added)
+
+    batch_size_add = 8192
+    for i in range(0, big_npy.shape[0], batch_size_add):
+        index_added.add(big_npy[i : i + batch_size_add])
+
+    faiss.write_index(index_added, index_filepath_added)
+    print(f"Saved index file '{index_filepath_added}'")
+
+except Exception as error:
+    print(f"Failed to train index: {error}")
+    if "one array to concatenate" in str(error):
+        print(
+            "If you are running this code in a virtual environment, make sure you have enough GPU available to generate the Index file."
+        )

system/tts_engines/rvc/train/process/extract_model.py

@@ -0,0 +1,82 @@
+import os
+import torch
+import hashlib
+import datetime
+from collections import OrderedDict
+
+
+def replace_keys_in_dict(d, old_key_part, new_key_part):
+    if isinstance(d, OrderedDict):
+        updated_dict = OrderedDict()
+    else:
+        updated_dict = {}
+    for key, value in d.items():
+        new_key = key.replace(old_key_part, new_key_part)
+        if isinstance(value, dict):
+            value = replace_keys_in_dict(value, old_key_part, new_key_part)
+        updated_dict[new_key] = value
+    return updated_dict
+
+
+def extract_model(ckpt, sr, if_f0, name, model_dir, epoch, step, version, hps):
+    try:
+        print(f"Saved model '{model_dir}' (epoch {epoch} and step {step})")
+        pth_file = f"{name}_{epoch}e_{step}s.pth"
+        pth_file_old_version_path = os.path.join(
+            model_dir, f"{pth_file}_old_version.pth"
+        )
+        opt = OrderedDict(
+            weight={
+                key: value.half() for key, value in ckpt.items() if "enc_q" not in key
+            }
+        )
+        opt["config"] = [
+            hps.data.filter_length // 2 + 1,
+            32,
+            hps.model.inter_channels,
+            hps.model.hidden_channels,
+            hps.model.filter_channels,
+            hps.model.n_heads,
+            hps.model.n_layers,
+            hps.model.kernel_size,
+            hps.model.p_dropout,
+            hps.model.resblock,
+            hps.model.resblock_kernel_sizes,
+            hps.model.resblock_dilation_sizes,
+            hps.model.upsample_rates,
+            hps.model.upsample_initial_channel,
+            hps.model.upsample_kernel_sizes,
+            hps.model.spk_embed_dim,
+            hps.model.gin_channels,
+            hps.data.sampling_rate,
+        ]
+
+        opt["epoch"] = epoch
+        opt["step"] = step
+        opt["sr"] = sr
+        opt["f0"] = if_f0
+        opt["version"] = version
+        opt["creation_date"] = datetime.datetime.now().isoformat()
+
+        hash_input = f"{str(ckpt)} {epoch} {step} {datetime.datetime.now().isoformat()}"
+        model_hash = hashlib.sha256(hash_input.encode()).hexdigest()
+        opt["model_hash"] = model_hash
+
+        torch.save(opt, model_dir)
+
+        model = torch.load(model_dir, map_location=torch.device("cpu"))
+        torch.save(
+            replace_keys_in_dict(
+                replace_keys_in_dict(
+                    model, ".parametrizations.weight.original1", ".weight_v"
+                ),
+                ".parametrizations.weight.original0",
+                ".weight_g",
+            ),
+            pth_file_old_version_path,
+        )
+        os.remove(model_dir)
+        os.rename(pth_file_old_version_path, model_dir)
+
+    except Exception as error:
+        print(error)

system/tts_engines/rvc/train/process/extract_small_model.py

@@ -0,0 +1,175 @@
+import os
+import torch
+import hashlib
+import datetime
+from collections import OrderedDict
+
+
+def replace_keys_in_dict(d, old_key_part, new_key_part):
+    # Use OrderedDict if the original is an OrderedDict
+    if isinstance(d, OrderedDict):
+        updated_dict = OrderedDict()
+    else:
+        updated_dict = {}
+    for key, value in d.items():
+        # Replace the key part if found
+        new_key = key.replace(old_key_part, new_key_part)
+        # If the value is a dictionary, apply the function recursively
+        if isinstance(value, dict):
+            value = replace_keys_in_dict(value, old_key_part, new_key_part)
+        updated_dict[new_key] = value
+    return updated_dict
+
+
+def extract_small_model(path, name, sr, if_f0, version, epoch, step):
+    try:
+        ckpt = torch.load(path, map_location="cpu")
+        pth_file = f"{name}.pth"
+        pth_file_old_version_path = os.path.join("logs", f"{pth_file}_old_version.pth")
+        opt = OrderedDict(
+            weight={
+                key: value.half() for key, value in ckpt.items() if "enc_q" not in key
+            }
+        )
+        if "model" in ckpt:
+            ckpt = ckpt["model"]
+        opt = OrderedDict()
+        opt["weight"] = {}
+        for key in ckpt.keys():
+            if "enc_q" in key:
+                continue
+            opt["weight"][key] = ckpt[key].half()
+        if sr == "40000":
+            opt["config"] = [
+                1025,
+                32,
+                192,
+                192,
+                768,
+                2,
+                6,
+                3,
+                0,
+                "1",
+                [3, 7, 11],
+                [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+                [10, 10, 2, 2],
+                512,
+                [16, 16, 4, 4],
+                109,
+                256,
+                40000,
+            ]
+        elif sr == "48000":
+            if version == "v1":
+                opt["config"] = [
+                    1025,
+                    32,
+                    192,
+                    192,
+                    768,
+                    2,
+                    6,
+                    3,
+                    0,
+                    "1",
+                    [3, 7, 11],
+                    [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+                    [10, 6, 2, 2, 2],
+                    512,
+                    [16, 16, 4, 4, 4],
+                    109,
+                    256,
+                    48000,
+                ]
+            else:
+                opt["config"] = [
+                    1025,
+                    32,
+                    192,
+                    192,
+                    768,
+                    2,
+                    6,
+                    3,
+                    0,
+                    "1",
+                    [3, 7, 11],
+                    [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+                    [12, 10, 2, 2],
+                    512,
+                    [24, 20, 4, 4],
+                    109,
+                    256,
+                    48000,
+                ]
+        elif sr == "32000":
+            if version == "v1":
+                opt["config"] = [
+                    513,
+                    32,
+                    192,
+                    192,
+                    768,
+                    2,
+                    6,
+                    3,
+                    0,
+                    "1",
+                    [3, 7, 11],
+                    [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+                    [10, 4, 2, 2, 2],
+                    512,
+                    [16, 16, 4, 4, 4],
+                    109,
+                    256,
+                    32000,
+                ]
+            else:
+                opt["config"] = [
+                    513,
+                    32,
+                    192,
+                    192,
+                    768,
+                    2,
+                    6,
+                    3,
+                    0,
+                    "1",
+                    [3, 7, 11],
+                    [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+                    [10, 8, 2, 2],
+                    512,
+                    [20, 16, 4, 4],
+                    109,
+                    256,
+                    32000,
+                ]
+
+        opt["epoch"] = epoch
+        opt["step"] = step
+        opt["sr"] = sr
+        opt["f0"] = int(if_f0)
+        opt["version"] = version
+        opt["creation_date"] = datetime.datetime.now().isoformat()
+
+        hash_input = f"{str(ckpt)} {epoch} {step} {datetime.datetime.now().isoformat()}"
+        model_hash = hashlib.sha256(hash_input.encode()).hexdigest()
+        opt["model_hash"] = model_hash
+
+        model = torch.load(pth_file_old_version_path, map_location=torch.device("cpu"))
+        torch.save(
+            replace_keys_in_dict(
+                replace_keys_in_dict(
+                    model, ".parametrizations.weight.original1", ".weight_v"
+                ),
+                ".parametrizations.weight.original0",
+                ".weight_g",
+            ),
+            pth_file_old_version_path,
+        )
+        os.remove(pth_file_old_version_path)
+        os.rename(pth_file_old_version_path, pth_file)
+    except Exception as error:
+        print(error)

system/tts_engines/rvc/train/process/model_blender.py

@@ -0,0 +1,63 @@
+import os
+import torch
+from collections import OrderedDict
+
+
+def extract(ckpt):
+    a = ckpt["model"]
+    opt = OrderedDict()
+    opt["weight"] = {}
+    for key in a.keys():
+        if "enc_q" in key:
+            continue
+        opt["weight"][key] = a[key]
+    return opt
+
+
+def model_blender(name, path1, path2, ratio):
+    try:
+        message = f"Model {path1} and {path2} are merged with alpha {ratio}."
+        ckpt1 = torch.load(path1, map_location="cpu")
+        ckpt2 = torch.load(path2, map_location="cpu")
+        cfg = ckpt1["config"]
+        cfg_f0 = ckpt1["f0"]
+        cfg_version = ckpt1["version"]
+
+        if "model" in ckpt1:
+            ckpt1 = extract(ckpt1)
+        else:
+            ckpt1 = ckpt1["weight"]
+        if "model" in ckpt2:
+            ckpt2 = extract(ckpt2)
+        else:
+            ckpt2 = ckpt2["weight"]
+
+        if sorted(list(ckpt1.keys())) != sorted(list(ckpt2.keys())):
+            return "Fail to merge the models. The model architectures are not the same."
+
+        opt = OrderedDict()
+        opt["weight"] = {}
+        for key in ckpt1.keys():
+            if key == "emb_g.weight" and ckpt1[key].shape != ckpt2[key].shape:
+                min_shape0 = min(ckpt1[key].shape[0], ckpt2[key].shape[0])
+                opt["weight"][key] = (
+                    ratio * (ckpt1[key][:min_shape0].float())
+                    + (1 - ratio) * (ckpt2[key][:min_shape0].float())
+                ).half()
+            else:
+                opt["weight"][key] = (
+                    ratio * (ckpt1[key].float()) + (1 - ratio) * (ckpt2[key].float())
+                ).half()
+
+        opt["config"] = cfg
+        opt["sr"] = message
+        opt["f0"] = cfg_f0
+        opt["version"] = cfg_version
+        opt["info"] = message
+
+        torch.save(opt, os.path.join("logs", "%s.pth" % name))
+        print(message)
+        return message, os.path.join("logs", "%s.pth" % name)
+    except Exception as error:
+        print(error)
+        return error

system/tts_engines/rvc/train/process/model_information.py

@@ -0,0 +1,40 @@
+import torch
+from datetime import datetime
+
+
+def prettify_date(date_str):
+    if date_str is None:
+        return "None"
+    try:
+        date_time_obj = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%f")
+        return date_time_obj.strftime("%Y-%m-%d %H:%M:%S")
+    except ValueError:
+        return "Invalid date format"
+
+
+def model_information(path):
+    model_data = torch.load(path, map_location="cpu")
+
+    print(f"Loaded model from {path}")
+
+    epochs = model_data.get("epoch", "None")
+    steps = model_data.get("step", "None")
+    sr = model_data.get("sr", "None")
+    f0 = model_data.get("f0", "None")
+    version = model_data.get("version", "None")
+    creation_date = model_data.get("creation_date", "None")
+    model_hash = model_data.get("model_hash", None)
+
+    pitch_guidance = "True" if f0 == 1 else "False"
+
+    creation_date_str = prettify_date(creation_date) if creation_date else "None"
+
+    return (
+        f"Epochs: {epochs}\n"
+        f"Steps: {steps}\n"
+        f"RVC Version: {version}\n"
+        f"Sampling Rate: {sr}\n"
+        f"Pitch Guidance: {pitch_guidance}\n"
+        f"Creation Date: {creation_date_str}\n"
+        f"Hash (ID): {model_hash}"
+    )

system/tts_engines/rvc/train/slicer.py

@@ -0,0 +1,171 @@
+import numpy as np
+
+
+class Slicer:
+    def __init__(
+        self,
+        sr: int,
+        threshold: float = -40.0,
+        min_length: int = 5000,
+        min_interval: int = 300,
+        hop_size: int = 20,
+        max_sil_kept: int = 5000,
+    ):
+        if not min_length >= min_interval >= hop_size:
+            raise ValueError("min_length >= min_interval >= hop_size is required")
+        if not max_sil_kept >= hop_size:
+            raise ValueError("max_sil_kept >= hop_size is required")
+
+        min_interval = sr * min_interval / 1000
+        self.threshold = 10 ** (threshold / 20.0)
+        self.hop_size = round(sr * hop_size / 1000)
+        self.win_size = min(round(min_interval), 4 * self.hop_size)
+        self.min_length = round(sr * min_length / 1000 / self.hop_size)
+        self.min_interval = round(min_interval / self.hop_size)
+        self.max_sil_kept = round(sr * max_sil_kept / 1000 / self.hop_size)
+
+    def _apply_slice(self, waveform, begin, end):
+        start_idx = begin * self.hop_size
+        if len(waveform.shape) > 1:
+            end_idx = min(waveform.shape[1], end * self.hop_size)
+            return waveform[:, start_idx:end_idx]
+        else:
+            end_idx = min(waveform.shape[0], end * self.hop_size)
+            return waveform[start_idx:end_idx]
+
+    def slice(self, waveform):
+        samples = waveform.mean(axis=0) if len(waveform.shape) > 1 else waveform
+        if samples.shape[0] <= self.min_length:
+            return [waveform]
+
+        rms_list = get_rms(
+            y=samples, frame_length=self.win_size, hop_length=self.hop_size
+        ).squeeze(0)
+        sil_tags = []
+        silence_start, clip_start = None, 0
+
+        for i, rms in enumerate(rms_list):
+            if rms < self.threshold:
+                if silence_start is None:
+                    silence_start = i
+                continue
+
+            if silence_start is None:
+                continue
+
+            is_leading_silence = silence_start == 0 and i > self.max_sil_kept
+            need_slice_middle = (
+                i - silence_start >= self.min_interval
+                and i - clip_start >= self.min_length
+            )
+
+            if not is_leading_silence and not need_slice_middle:
+                silence_start = None
+                continue
+
+            if i - silence_start <= self.max_sil_kept:
+                pos = rms_list[silence_start : i + 1].argmin() + silence_start
+                if silence_start == 0:
+                    sil_tags.append((0, pos))
+                else:
+                    sil_tags.append((pos, pos))
+                clip_start = pos
+            elif i - silence_start <= self.max_sil_kept * 2:
+                pos = rms_list[
+                    i - self.max_sil_kept : silence_start + self.max_sil_kept + 1
+                ].argmin()
+                pos += i - self.max_sil_kept
+                pos_l = (
+                    rms_list[
+                        silence_start : silence_start + self.max_sil_kept + 1
+                    ].argmin()
+                    + silence_start
+                )
+                pos_r = (
+                    rms_list[i - self.max_sil_kept : i + 1].argmin()
+                    + i
+                    - self.max_sil_kept
+                )
+                if silence_start == 0:
+                    sil_tags.append((0, pos_r))
+                    clip_start = pos_r
+                else:
+                    sil_tags.append((min(pos_l, pos), max(pos_r, pos)))
+                    clip_start = max(pos_r, pos)
+            else:
+                pos_l = (
+                    rms_list[
+                        silence_start : silence_start + self.max_sil_kept + 1
+                    ].argmin()
+                    + silence_start
+                )
+                pos_r = (
+                    rms_list[i - self.max_sil_kept : i + 1].argmin()
+                    + i
+                    - self.max_sil_kept
+                )
+                if silence_start == 0:
+                    sil_tags.append((0, pos_r))
+                else:
+                    sil_tags.append((pos_l, pos_r))
+                clip_start = pos_r
+            silence_start = None
+
+        total_frames = rms_list.shape[0]
+
+        if (
+            silence_start is not None
+            and total_frames - silence_start >= self.min_interval
+        ):
+            silence_end = min(total_frames, silence_start + self.max_sil_kept)
+            pos = rms_list[silence_start : silence_end + 1].argmin() + silence_start
+            sil_tags.append((pos, total_frames + 1))
+
+        if not sil_tags:
+            return [waveform]
+        else:
+            chunks = []
+            if sil_tags[0][0] > 0:
+                chunks.append(self._apply_slice(waveform, 0, sil_tags[0][0]))
+
+            for i in range(len(sil_tags) - 1):
+                chunks.append(
+                    self._apply_slice(waveform, sil_tags[i][1], sil_tags[i + 1][0])
+                )
+
+            if sil_tags[-1][1] < total_frames:
+                chunks.append(
+                    self._apply_slice(waveform, sil_tags[-1][1], total_frames)
+                )
+
+            return chunks
+
+
+def get_rms(
+    y,
+    frame_length=2048,
+    hop_length=512,
+    pad_mode="constant",
+):
+    padding = (int(frame_length // 2), int(frame_length // 2))
+    y = np.pad(y, padding, mode=pad_mode)
+
+    axis = -1
+    out_strides = y.strides + tuple([y.strides[axis]])
+    x_shape_trimmed = list(y.shape)
+    x_shape_trimmed[axis] -= frame_length - 1
+    out_shape = tuple(x_shape_trimmed) + tuple([frame_length])
+    xw = np.lib.stride_tricks.as_strided(y, shape=out_shape, strides=out_strides)
+
+    if axis < 0:
+        target_axis = axis - 1
+    else:
+        target_axis = axis + 1
+
+    xw = np.moveaxis(xw, -1, target_axis)
+    slices = [slice(None)] * xw.ndim
+    slices[axis] = slice(0, None, hop_length)
+    x = xw[tuple(slices)]
+
+    power = np.mean(np.abs(x) ** 2, axis=-2, keepdims=True)
+    return np.sqrt(power)

system/tts_engines/rvc/train/train.py

@@ -0,0 +1,720 @@
+import torch
+import sys
+import os
+import datetime
+import json
+import re
+
+from utils import (
+    get_hparams,
+    plot_spectrogram_to_numpy,
+    summarize,
+    load_checkpoint,
+    save_checkpoint,
+    latest_checkpoint_path,
+)
+from random import randint, shuffle
+from time import sleep
+from time import time as ttime
+
+from torch.cuda.amp import GradScaler, autocast
+
+from torch.nn import functional as F
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.utils.data import DataLoader
+from torch.utils.tensorboard import SummaryWriter
+import torch.distributed as dist
+import torch.multiprocessing as mp
+
+now_dir = os.getcwd()
+sys.path.append(os.path.join(now_dir))
+
+
+from data_utils import (
+    DistributedBucketSampler,
+    TextAudioCollate,
+    TextAudioCollateMultiNSFsid,
+    TextAudioLoader,
+    TextAudioLoaderMultiNSFsid,
+)
+
+from losses import (
+    discriminator_loss,
+    feature_loss,
+    generator_loss,
+    kl_loss,
+)
+from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
+
+from rvc.train.process.extract_model import extract_model
+
+from rvc.lib.infer_pack import commons
+
+hps = get_hparams()
+if hps.version == "v1":
+    from rvc.lib.infer_pack.models import MultiPeriodDiscriminator
+    from rvc.lib.infer_pack.models import SynthesizerTrnMs256NSFsid as RVC_Model_f0
+    from rvc.lib.infer_pack.models import (
+        SynthesizerTrnMs256NSFsid_nono as RVC_Model_nof0,
+    )
+elif hps.version == "v2":
+    from rvc.lib.infer_pack.models import (
+        SynthesizerTrnMs768NSFsid as RVC_Model_f0,
+        SynthesizerTrnMs768NSFsid_nono as RVC_Model_nof0,
+        MultiPeriodDiscriminatorV2 as MultiPeriodDiscriminator,
+    )
+
+os.environ["CUDA_VISIBLE_DEVICES"] = hps.gpus.replace("-", ",")
+n_gpus = len(hps.gpus.split("-"))
+
+
+torch.backends.cudnn.deterministic = False
+torch.backends.cudnn.benchmark = False
+
+global_step = 0
+lowest_value = {"step": 0, "value": float("inf"), "epoch": 0}
+last_loss_gen_all = 0
+epochs_since_last_lowest = 0
+
+
+class EpochRecorder:
+    def __init__(self):
+        self.last_time = ttime()
+
+    def record(self):
+        now_time = ttime()
+        elapsed_time = now_time - self.last_time
+        self.last_time = now_time
+        elapsed_time = round(elapsed_time, 1)
+        elapsed_time_str = str(datetime.timedelta(seconds=int(elapsed_time)))
+        current_time = datetime.datetime.now().strftime("%H:%M:%S")
+        return f"time={current_time} | training_speed={elapsed_time_str}"
+
+
+def main():
+    def start():
+        children = []
+        pid_file_path = os.path.join(now_dir, "rvc", "train", "train_pid.txt")
+        with open(pid_file_path, "w") as pid_file:
+            for i in range(n_gpus):
+                subproc = mp.Process(
+                    target=run,
+                    args=(i, n_gpus, hps),
+                )
+                children.append(subproc)
+                subproc.start()
+                pid_file.write(str(subproc.pid) + "\n")
+
+        for i in range(n_gpus):
+            children[i].join()
+
+    n_gpus = torch.cuda.device_count()
+
+    if torch.cuda.is_available() == False and torch.backends.mps.is_available() == True:
+        n_gpus = 1
+    if n_gpus < 1:
+        print("GPU not detected, reverting to CPU (not recommended)")
+        n_gpus = 1
+
+    if hps.sync_graph == 1:
+        print(
+            "Sync graph is now activated! With sync graph enabled, the model undergoes a single epoch of training. Once the graphs are synchronized, training proceeds for the previously specified number of epochs."
+        )
+        hps.custom_total_epoch = 1
+        hps.custom_save_every_weights = "1"
+        start()
+
+        logs_path = os.path.join(now_dir, "logs")
+        model_config_file = os.path.join(now_dir, "logs", hps.name, "config.json")
+        rvc_config_file = os.path.join(
+            now_dir, "rvc", "configs", hps.version, str(hps.sample_rate) + ".json"
+        )
+        if not os.path.exists(rvc_config_file):
+            rvc_config_file = os.path.join(
+                now_dir, "rvc", "configs", "v1", str(hps.sample_rate) + ".json"
+            )
+
+        pattern = rf"{os.path.basename(hps.name)}_1e_(\d+)s\.pth"
+
+        for filename in os.listdir(logs_path):
+            match = re.match(pattern, filename)
+            if match:
+                steps = int(match.group(1))
+
+        def edit_config(config_file):
+            with open(config_file, "r", encoding="utf8") as json_file:
+                config_data = json.load(json_file)
+
+            config_data["train"]["log_interval"] = steps
+
+            with open(config_file, "w", encoding="utf8") as json_file:
+                json.dump(
+                    config_data,
+                    json_file,
+                    indent=2,
+                    separators=(",", ": "),
+                    ensure_ascii=False,
+                )
+
+        edit_config(model_config_file)
+        edit_config(rvc_config_file)
+
+        for root, dirs, files in os.walk(
+            os.path.join(now_dir, "logs", hps.name), topdown=False
+        ):
+            for name in files:
+                file_path = os.path.join(root, name)
+                file_name, file_extension = os.path.splitext(name)
+                if file_extension == ".0":
+                    os.remove(file_path)
+                elif ("D" in name or "G" in name) and file_extension == ".pth":
+                    os.remove(file_path)
+                elif (
+                    "added" in name or "trained" in name
+                ) and file_extension == ".index":
+                    os.remove(file_path)
+            for name in dirs:
+                if name == "eval":
+                    folder_path = os.path.join(root, name)
+                    for item in os.listdir(folder_path):
+                        item_path = os.path.join(folder_path, item)
+                        if os.path.isfile(item_path):
+                            os.remove(item_path)
+                    os.rmdir(folder_path)
+
+        print("Successfully synchronized graphs!")
+        hps.custom_total_epoch = hps.total_epoch
+        hps.custom_save_every_weights = hps.save_every_weights
+        start()
+    else:
+        hps.custom_total_epoch = hps.total_epoch
+        hps.custom_save_every_weights = hps.save_every_weights
+        start()
+
+
+def run(
+    rank,
+    n_gpus,
+    hps,
+):
+    global global_step
+    if rank == 0:
+        writer = SummaryWriter(log_dir=hps.model_dir)
+        writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval"))
+
+    os.environ["MASTER_ADDR"] = "localhost"
+    os.environ["MASTER_PORT"] = str(randint(20000, 55555))
+    dist.init_process_group(
+        backend="gloo", init_method="env://", world_size=n_gpus, rank=rank
+    )
+    torch.manual_seed(hps.train.seed)
+    if torch.cuda.is_available():
+        torch.cuda.set_device(rank)
+
+    if hps.if_f0 == 1:
+        train_dataset = TextAudioLoaderMultiNSFsid(hps.data)
+    else:
+        train_dataset = TextAudioLoader(hps.data)
+
+    train_sampler = DistributedBucketSampler(
+        train_dataset,
+        hps.train.batch_size * n_gpus,
+        [100, 200, 300, 400, 500, 600, 700, 800, 900],
+        num_replicas=n_gpus,
+        rank=rank,
+        shuffle=True,
+    )
+
+    if hps.if_f0 == 1:
+        collate_fn = TextAudioCollateMultiNSFsid()
+    else:
+        collate_fn = TextAudioCollate()
+    train_loader = DataLoader(
+        train_dataset,
+        num_workers=4,
+        shuffle=False,
+        pin_memory=True,
+        collate_fn=collate_fn,
+        batch_sampler=train_sampler,
+        persistent_workers=True,
+        prefetch_factor=8,
+    )
+    if hps.if_f0 == 1:
+        net_g = RVC_Model_f0(
+            hps.data.filter_length // 2 + 1,
+            hps.train.segment_size // hps.data.hop_length,
+            **hps.model,
+            is_half=hps.train.fp16_run,
+            sr=hps.sample_rate,
+        )
+    else:
+        net_g = RVC_Model_nof0(
+            hps.data.filter_length // 2 + 1,
+            hps.train.segment_size // hps.data.hop_length,
+            **hps.model,
+            is_half=hps.train.fp16_run,
+        )
+    if torch.cuda.is_available():
+        net_g = net_g.cuda(rank)
+    net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm)
+    if torch.cuda.is_available():
+        net_d = net_d.cuda(rank)
+    optim_g = torch.optim.AdamW(
+        net_g.parameters(),
+        hps.train.learning_rate,
+        betas=hps.train.betas,
+        eps=hps.train.eps,
+    )
+    optim_d = torch.optim.AdamW(
+        net_d.parameters(),
+        hps.train.learning_rate,
+        betas=hps.train.betas,
+        eps=hps.train.eps,
+    )
+    if torch.cuda.is_available():
+        net_g = DDP(net_g, device_ids=[rank])
+        net_d = DDP(net_d, device_ids=[rank])
+    else:
+        net_g = DDP(net_g)
+        net_d = DDP(net_d)
+
+    try:
+        print("Starting training...")
+        _, _, _, epoch_str = load_checkpoint(
+            latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_d
+        )
+        _, _, _, epoch_str = load_checkpoint(
+            latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, optim_g
+        )
+        global_step = (epoch_str - 1) * len(train_loader)
+
+    except:
+        epoch_str = 1
+        global_step = 0
+        if hps.pretrainG != "":
+            if rank == 0:
+                print(f"Loaded pretrained_G {hps.pretrainG}")
+            if hasattr(net_g, "module"):
+                print(
+                    net_g.module.load_state_dict(
+                        torch.load(hps.pretrainG, map_location="cpu")["model"]
+                    )
+                )
+            else:
+                print(
+                    net_g.load_state_dict(
+                        torch.load(hps.pretrainG, map_location="cpu")["model"]
+                    )
+                )
+        if hps.pretrainD != "":
+            if rank == 0:
+                print(f"Loaded pretrained_D {hps.pretrainD}")
+            if hasattr(net_d, "module"):
+                print(
+                    net_d.module.load_state_dict(
+                        torch.load(hps.pretrainD, map_location="cpu")["model"]
+                    )
+                )
+            else:
+                print(
+                    net_d.load_state_dict(
+                        torch.load(hps.pretrainD, map_location="cpu")["model"]
+                    )
+                )
+
+    scheduler_g = torch.optim.lr_scheduler.ExponentialLR(
+        optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
+    )
+    scheduler_d = torch.optim.lr_scheduler.ExponentialLR(
+        optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
+    )
+
+    scaler = GradScaler(enabled=hps.train.fp16_run)
+
+    cache = []
+    for epoch in range(epoch_str, hps.train.epochs + 1):
+        if rank == 0:
+            train_and_evaluate(
+                rank,
+                epoch,
+                hps,
+                [net_g, net_d],
+                [optim_g, optim_d],
+                scaler,
+                [train_loader, None],
+                [writer, writer_eval],
+                cache,
+            )
+        else:
+            train_and_evaluate(
+                rank,
+                epoch,
+                hps,
+                [net_g, net_d],
+                [optim_g, optim_d],
+                scaler,
+                [train_loader, None],
+                None,
+                cache,
+            )
+
+        scheduler_g.step()
+        scheduler_d.step()
+
+
+def train_and_evaluate(rank, epoch, hps, nets, optims, scaler, loaders, writers, cache):
+    global global_step, last_loss_gen_all, lowest_value, epochs_since_last_lowest
+
+    if epoch == 1:
+        lowest_value = {"step": 0, "value": float("inf"), "epoch": 0}
+        last_loss_gen_all = 0.0
+        epochs_since_last_lowest = 0
+
+    net_g, net_d = nets
+    optim_g, optim_d = optims
+    train_loader = loaders[0] if loaders is not None else None
+    if writers is not None:
+        writer = writers[0]
+
+    train_loader.batch_sampler.set_epoch(epoch)
+
+    net_g.train()
+    net_d.train()
+
+    if hps.if_cache_data_in_gpu == True:
+        data_iterator = cache
+        if cache == []:
+            for batch_idx, info in enumerate(train_loader):
+                if hps.if_f0 == 1:
+                    (
+                        phone,
+                        phone_lengths,
+                        pitch,
+                        pitchf,
+                        spec,
+                        spec_lengths,
+                        wave,
+                        wave_lengths,
+                        sid,
+                    ) = info
+                else:
+                    (
+                        phone,
+                        phone_lengths,
+                        spec,
+                        spec_lengths,
+                        wave,
+                        wave_lengths,
+                        sid,
+                    ) = info
+                if torch.cuda.is_available():
+                    phone = phone.cuda(rank, non_blocking=True)
+                    phone_lengths = phone_lengths.cuda(rank, non_blocking=True)
+                    if hps.if_f0 == 1:
+                        pitch = pitch.cuda(rank, non_blocking=True)
+                        pitchf = pitchf.cuda(rank, non_blocking=True)
+                    sid = sid.cuda(rank, non_blocking=True)
+                    spec = spec.cuda(rank, non_blocking=True)
+                    spec_lengths = spec_lengths.cuda(rank, non_blocking=True)
+                    wave = wave.cuda(rank, non_blocking=True)
+                    wave_lengths = wave_lengths.cuda(rank, non_blocking=True)
+                if hps.if_f0 == 1:
+                    cache.append(
+                        (
+                            batch_idx,
+                            (
+                                phone,
+                                phone_lengths,
+                                pitch,
+                                pitchf,
+                                spec,
+                                spec_lengths,
+                                wave,
+                                wave_lengths,
+                                sid,
+                            ),
+                        )
+                    )
+                else:
+                    cache.append(
+                        (
+                            batch_idx,
+                            (
+                                phone,
+                                phone_lengths,
+                                spec,
+                                spec_lengths,
+                                wave,
+                                wave_lengths,
+                                sid,
+                            ),
+                        )
+                    )
+        else:
+            shuffle(cache)
+    else:
+        data_iterator = enumerate(train_loader)
+
+    epoch_recorder = EpochRecorder()
+    for batch_idx, info in data_iterator:
+        if hps.if_f0 == 1:
+            (
+                phone,
+                phone_lengths,
+                pitch,
+                pitchf,
+                spec,
+                spec_lengths,
+                wave,
+                wave_lengths,
+                sid,
+            ) = info
+        else:
+            phone, phone_lengths, spec, spec_lengths, wave, wave_lengths, sid = info
+        if (hps.if_cache_data_in_gpu == False) and torch.cuda.is_available():
+            phone = phone.cuda(rank, non_blocking=True)
+            phone_lengths = phone_lengths.cuda(rank, non_blocking=True)
+            if hps.if_f0 == 1:
+                pitch = pitch.cuda(rank, non_blocking=True)
+                pitchf = pitchf.cuda(rank, non_blocking=True)
+            sid = sid.cuda(rank, non_blocking=True)
+            spec = spec.cuda(rank, non_blocking=True)
+            spec_lengths = spec_lengths.cuda(rank, non_blocking=True)
+            wave = wave.cuda(rank, non_blocking=True)
+
+        with autocast(enabled=hps.train.fp16_run):
+            if hps.if_f0 == 1:
+                (
+                    y_hat,
+                    ids_slice,
+                    x_mask,
+                    z_mask,
+                    (z, z_p, m_p, logs_p, m_q, logs_q),
+                ) = net_g(phone, phone_lengths, pitch, pitchf, spec, spec_lengths, sid)
+            else:
+                (
+                    y_hat,
+                    ids_slice,
+                    x_mask,
+                    z_mask,
+                    (z, z_p, m_p, logs_p, m_q, logs_q),
+                ) = net_g(phone, phone_lengths, spec, spec_lengths, sid)
+            mel = spec_to_mel_torch(
+                spec,
+                hps.data.filter_length,
+                hps.data.n_mel_channels,
+                hps.data.sampling_rate,
+                hps.data.mel_fmin,
+                hps.data.mel_fmax,
+            )
+            y_mel = commons.slice_segments(
+                mel, ids_slice, hps.train.segment_size // hps.data.hop_length
+            )
+            with autocast(enabled=False):
+                y_hat_mel = mel_spectrogram_torch(
+                    y_hat.float().squeeze(1),
+                    hps.data.filter_length,
+                    hps.data.n_mel_channels,
+                    hps.data.sampling_rate,
+                    hps.data.hop_length,
+                    hps.data.win_length,
+                    hps.data.mel_fmin,
+                    hps.data.mel_fmax,
+                )
+            if hps.train.fp16_run == True:
+                y_hat_mel = y_hat_mel.half()
+            wave = commons.slice_segments(
+                wave, ids_slice * hps.data.hop_length, hps.train.segment_size
+            )
+
+            y_d_hat_r, y_d_hat_g, _, _ = net_d(wave, y_hat.detach())
+            with autocast(enabled=False):
+                loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(
+                    y_d_hat_r, y_d_hat_g
+                )
+        optim_d.zero_grad()
+        scaler.scale(loss_disc).backward()
+        scaler.unscale_(optim_d)
+        grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
+        scaler.step(optim_d)
+
+        with autocast(enabled=hps.train.fp16_run):
+            y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(wave, y_hat)
+            with autocast(enabled=False):
+                loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
+                loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl
+                loss_fm = feature_loss(fmap_r, fmap_g)
+                loss_gen, losses_gen = generator_loss(y_d_hat_g)
+                loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl
+
+                if loss_gen_all < lowest_value["value"]:
+                    lowest_value["value"] = loss_gen_all
+                    lowest_value["step"] = global_step
+                    lowest_value["epoch"] = epoch
+                    # print(f'Lowest generator loss updated: {lowest_value["value"]} at epoch {epoch}, step {global_step}')
+                    if epoch > lowest_value["epoch"]:
+                        print(
+                            "Alert: The lower generating loss has been exceeded by a lower loss in a subsequent epoch."
+                        )
+
+        optim_g.zero_grad()
+        scaler.scale(loss_gen_all).backward()
+        scaler.unscale_(optim_g)
+        grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
+        scaler.step(optim_g)
+        scaler.update()
+
+        if rank == 0:
+            if global_step % hps.train.log_interval == 0:
+                lr = optim_g.param_groups[0]["lr"]
+                # print("Epoch: {} [{:.0f}%]".format(epoch, 100.0 * batch_idx / len(train_loader)))
+
+                if loss_mel > 75:
+                    loss_mel = 75
+                if loss_kl > 9:
+                    loss_kl = 9
+
+                scalar_dict = {
+                    "loss/g/total": loss_gen_all,
+                    "loss/d/total": loss_disc,
+                    "learning_rate": lr,
+                    "grad_norm_d": grad_norm_d,
+                    "grad_norm_g": grad_norm_g,
+                }
+                scalar_dict.update(
+                    {
+                        "loss/g/fm": loss_fm,
+                        "loss/g/mel": loss_mel,
+                        "loss/g/kl": loss_kl,
+                    }
+                )
+
+                scalar_dict.update(
+                    {"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}
+                )
+                scalar_dict.update(
+                    {"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}
+                )
+                scalar_dict.update(
+                    {"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}
+                )
+                image_dict = {
+                    "slice/mel_org": plot_spectrogram_to_numpy(
+                        y_mel[0].data.cpu().numpy()
+                    ),
+                    "slice/mel_gen": plot_spectrogram_to_numpy(
+                        y_hat_mel[0].data.cpu().numpy()
+                    ),
+                    "all/mel": plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()),
+                }
+                summarize(
+                    writer=writer,
+                    global_step=global_step,
+                    images=image_dict,
+                    scalars=scalar_dict,
+                )
+
+                # optim_g.step()
+                # optim_d.step()
+
+        global_step += 1
+
+    if epoch % hps.save_every_epoch == 0 and rank == 0:
+        checkpoint_suffix = "{}.pth".format(
+            global_step if hps.if_latest == 0 else 2333333
+        )
+        save_checkpoint(
+            net_g,
+            optim_g,
+            hps.train.learning_rate,
+            epoch,
+            os.path.join(hps.model_dir, "G_" + checkpoint_suffix),
+        )
+        save_checkpoint(
+            net_d,
+            optim_d,
+            hps.train.learning_rate,
+            epoch,
+            os.path.join(hps.model_dir, "D_" + checkpoint_suffix),
+        )
+
+        if rank == 0 and hps.custom_save_every_weights == "1":
+            if hasattr(net_g, "module"):
+                ckpt = net_g.module.state_dict()
+            else:
+                ckpt = net_g.state_dict()
+            extract_model(
+                ckpt,
+                hps.sample_rate,
+                hps.if_f0,
+                hps.name,
+                os.path.join(
+                    hps.model_dir, "{}_{}e_{}s.pth".format(hps.name, epoch, global_step)
+                ),
+                epoch,
+                global_step,
+                hps.version,
+                hps,
+            )
+
+    if hps.overtraining_detector == 1:
+        if lowest_value["value"] < last_loss_gen_all:
+            epochs_since_last_lowest += 1
+        else:
+            epochs_since_last_lowest = 0
+
+        if epochs_since_last_lowest >= hps.overtraining_threshold:
+            print(
+                "Stopping training due to possible overtraining. Lowest generator loss: {} at epoch {}, step {}".format(
+                    lowest_value["value"], lowest_value["epoch"], lowest_value["step"]
+                )
+            )
+            os._exit(2333333)
+
+    if rank == 0:
+        if epoch > 1:
+            print(
+                f"{hps.name} | epoch={epoch} | step={global_step} | {epoch_recorder.record()} | lowest_value={lowest_value['value']} (epoch {lowest_value['epoch']} and step {lowest_value['step']})"
+            )
+        else:
+            print(
+                f"{hps.name} | epoch={epoch} | step={global_step} | {epoch_recorder.record()}"
+            )
+        last_loss_gen_all = loss_gen_all
+
+    if epoch >= hps.custom_total_epoch and rank == 0:
+        print(
+            f"Training has been successfully completed with {epoch} epoch, {global_step} steps and {round(loss_gen_all.item(), 3)} loss gen."
+        )
+        print(
+            f"Lowest generator loss: {lowest_value['value']} at epoch {lowest_value['epoch']}, step {lowest_value['step']}"
+        )
+
+        pid_file_path = os.path.join(now_dir, "rvc", "train", "train_pid.txt")
+        os.remove(pid_file_path)
+
+        if hasattr(net_g, "module"):
+            ckpt = net_g.module.state_dict()
+        else:
+            ckpt = net_g.state_dict()
+
+        extract_model(
+            ckpt,
+            hps.sample_rate,
+            hps.if_f0,
+            hps.name,
+            os.path.join(
+                hps.model_dir, "{}_{}e_{}s.pth".format(hps.name, epoch, global_step)
+            ),
+            epoch,
+            global_step,
+            hps.version,
+            hps,
+        )
+        sleep(1)
+        os._exit(2333333)
+
+
+if __name__ == "__main__":
+    torch.multiprocessing.set_start_method("spawn")
+    main()

system/tts_engines/rvc/train/utils.py

@@ -0,0 +1,311 @@
+import os
+import glob
+import json
+import torch
+import argparse
+import numpy as np
+from scipy.io.wavfile import read
+from collections import OrderedDict
+
+
+def replace_keys_in_dict(d, old_key_part, new_key_part):
+    if isinstance(d, OrderedDict):
+        updated_dict = OrderedDict()
+    else:
+        updated_dict = {}
+    for key, value in d.items():
+        if isinstance(key, str):
+            new_key = key.replace(old_key_part, new_key_part)
+        else:
+            new_key = key
+        if isinstance(value, dict):
+            value = replace_keys_in_dict(value, old_key_part, new_key_part)
+        updated_dict[new_key] = value
+    return updated_dict
+
+
+def load_checkpoint(checkpoint_path, model, optimizer=None, load_opt=1):
+    assert os.path.isfile(checkpoint_path)
+    checkpoint_old_dict = torch.load(checkpoint_path, map_location="cpu")
+    checkpoint_new_version_path = os.path.join(
+        os.path.dirname(checkpoint_path),
+        f"{os.path.splitext(os.path.basename(checkpoint_path))[0]}_new_version.pth",
+    )
+
+    torch.save(
+        replace_keys_in_dict(
+            replace_keys_in_dict(
+                checkpoint_old_dict, ".weight_v", ".parametrizations.weight.original1"
+            ),
+            ".weight_g",
+            ".parametrizations.weight.original0",
+        ),
+        checkpoint_new_version_path,
+    )
+
+    os.remove(checkpoint_path)
+    os.rename(checkpoint_new_version_path, checkpoint_path)
+
+    checkpoint_dict = torch.load(checkpoint_path, map_location="cpu")
+    saved_state_dict = checkpoint_dict["model"]
+    if hasattr(model, "module"):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    new_state_dict = {}
+    for k, v in state_dict.items():
+        try:
+            new_state_dict[k] = saved_state_dict[k]
+            if saved_state_dict[k].shape != state_dict[k].shape:
+                print(
+                    "shape-%s-mismatch|need-%s|get-%s",
+                    k,
+                    state_dict[k].shape,
+                    saved_state_dict[k].shape,
+                )
+                raise KeyError
+        except:
+            print("%s is not in the checkpoint", k)
+            new_state_dict[k] = v
+    if hasattr(model, "module"):
+        model.module.load_state_dict(new_state_dict, strict=False)
+    else:
+        model.load_state_dict(new_state_dict, strict=False)
+
+    iteration = checkpoint_dict["iteration"]
+    learning_rate = checkpoint_dict["learning_rate"]
+    if optimizer is not None and load_opt == 1:
+        optimizer.load_state_dict(checkpoint_dict["optimizer"])
+    print(f"Loaded checkpoint '{checkpoint_path}' (epoch {iteration})")
+    return model, optimizer, learning_rate, iteration
+
+
+def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path):
+    print(f"Saved model '{checkpoint_path}' (epoch {iteration})")
+    checkpoint_old_version_path = os.path.join(
+        os.path.dirname(checkpoint_path),
+        f"{os.path.splitext(os.path.basename(checkpoint_path))[0]}_old_version.pth",
+    )
+    if hasattr(model, "module"):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    torch.save(
+        {
+            "model": state_dict,
+            "iteration": iteration,
+            "optimizer": optimizer.state_dict(),
+            "learning_rate": learning_rate,
+        },
+        checkpoint_path,
+    )
+    checkpoint = torch.load(checkpoint_path, map_location=torch.device("cpu"))
+    torch.save(
+        replace_keys_in_dict(
+            replace_keys_in_dict(
+                checkpoint, ".parametrizations.weight.original1", ".weight_v"
+            ),
+            ".parametrizations.weight.original0",
+            ".weight_g",
+        ),
+        checkpoint_old_version_path,
+    )
+    os.remove(checkpoint_path)
+    os.rename(checkpoint_old_version_path, checkpoint_path)
+
+
+def summarize(
+    writer,
+    global_step,
+    scalars={},
+    histograms={},
+    images={},
+    audios={},
+    audio_sampling_rate=22050,
+):
+    for k, v in scalars.items():
+        writer.add_scalar(k, v, global_step)
+    for k, v in histograms.items():
+        writer.add_histogram(k, v, global_step)
+    for k, v in images.items():
+        writer.add_image(k, v, global_step, dataformats="HWC")
+    for k, v in audios.items():
+        writer.add_audio(k, v, global_step, audio_sampling_rate)
+
+
+def latest_checkpoint_path(dir_path, regex="G_*.pth"):
+    f_list = glob.glob(os.path.join(dir_path, regex))
+    f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
+    x = f_list[-1]
+    return x
+
+
+def plot_spectrogram_to_numpy(spectrogram):
+    import matplotlib.pylab as plt
+    import numpy as np
+
+    fig, ax = plt.subplots(figsize=(10, 2))
+    im = ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation="none")
+    plt.colorbar(im, ax=ax)
+    plt.xlabel("Frames")
+    plt.ylabel("Channels")
+    plt.tight_layout()
+
+    fig.canvas.draw()
+    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
+    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
+    plt.close()
+    return data
+
+
+def load_wav_to_torch(full_path):
+    sampling_rate, data = read(full_path)
+    return torch.FloatTensor(data.astype(np.float32)), sampling_rate
+
+
+def load_filepaths_and_text(filename, split="|"):
+    with open(filename, encoding="utf-8") as f:
+        filepaths_and_text = [line.strip().split(split) for line in f]
+    return filepaths_and_text
+
+
+def get_hparams():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-se",
+        "--save_every_epoch",
+        type=int,
+        required=True,
+        help="checkpoint save frequency (epoch)",
+    )
+    parser.add_argument(
+        "-te", "--total_epoch", type=int, required=True, help="total_epoch"
+    )
+    parser.add_argument(
+        "-pg", "--pretrainG", type=str, default="", help="Pretrained Discriminator path"
+    )
+    parser.add_argument(
+        "-pd", "--pretrainD", type=str, default="", help="Pretrained Generator path"
+    )
+    parser.add_argument("-g", "--gpus", type=str, default="0", help="split by -")
+    parser.add_argument(
+        "-bs", "--batch_size", type=int, required=True, help="batch size"
+    )
+    parser.add_argument(
+        "-e", "--experiment_dir", type=str, required=True, help="experiment dir"
+    )
+    parser.add_argument(
+        "-sr", "--sample_rate", type=str, required=True, help="sample rate, 32k/40k/48k"
+    )
+    parser.add_argument(
+        "-sw",
+        "--save_every_weights",
+        type=str,
+        default="0",
+        help="save the extracted model in weights directory when saving checkpoints",
+    )
+    parser.add_argument(
+        "-v", "--version", type=str, required=True, help="model version"
+    )
+    parser.add_argument(
+        "-f0",
+        "--if_f0",
+        type=int,
+        required=True,
+        help="use f0 as one of the inputs of the model, 1 or 0",
+    )
+    parser.add_argument(
+        "-l",
+        "--if_latest",
+        type=int,
+        required=True,
+        help="if only save the latest G/D pth file, 1 or 0",
+    )
+    parser.add_argument(
+        "-c",
+        "--if_cache_data_in_gpu",
+        type=int,
+        required=True,
+        help="if caching the dataset in GPU memory, 1 or 0",
+    )
+
+    parser.add_argument(
+        "-od",
+        "--overtraining_detector",
+        type=int,
+        required=True,
+        help="Detect overtraining or not, 1 or 0",
+    )
+    parser.add_argument(
+        "-ot",
+        "--overtraining_threshold",
+        type=int,
+        default=50,
+        help="overtraining_threshold",
+    )
+    parser.add_argument(
+        "-sg",
+        "--sync-graph",
+        type=int,
+        required=True,
+        help="Sync graph or not, 1 or 0",
+    )
+
+    args = parser.parse_args()
+    name = args.experiment_dir
+    experiment_dir = os.path.join("./logs", args.experiment_dir)
+    config_save_path = os.path.join(experiment_dir, "config.json")
+    with open(config_save_path, "r") as f:
+        config = json.load(f)
+    hparams = HParams(**config)
+    hparams.model_dir = hparams.experiment_dir = experiment_dir
+    hparams.save_every_epoch = args.save_every_epoch
+    hparams.name = name
+    hparams.total_epoch = args.total_epoch
+    hparams.pretrainG = args.pretrainG
+    hparams.pretrainD = args.pretrainD
+    hparams.version = args.version
+    hparams.gpus = args.gpus
+    hparams.train.batch_size = args.batch_size
+    hparams.sample_rate = args.sample_rate
+    hparams.if_f0 = args.if_f0
+    hparams.if_latest = args.if_latest
+    hparams.save_every_weights = args.save_every_weights
+    hparams.if_cache_data_in_gpu = args.if_cache_data_in_gpu
+    hparams.data.training_files = f"{experiment_dir}/filelist.txt"
+    hparams.overtraining_detector = args.overtraining_detector
+    hparams.overtraining_threshold = args.overtraining_threshold
+    hparams.sync_graph = args.sync_graph
+    return hparams
+
+
+class HParams:
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            if type(v) == dict:
+                v = HParams(**v)
+            self[k] = v
+
+    def keys(self):
+        return self.__dict__.keys()
+
+    def items(self):
+        return self.__dict__.items()
+
+    def values(self):
+        return self.__dict__.values()
+
+    def __len__(self):
+        return len(self.__dict__)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def __setitem__(self, key, value):
+        return setattr(self, key, value)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+    def __repr__(self):
+        return self.__dict__.__repr__()