chore: import upstream snapshot with attribution

scripts/dataset_processing/spoken_wikipedia/preprocess.py

@@ -0,0 +1,204 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+This script can be used to preprocess Spoken Wikipedia corpus before running ctc-segmentation.
+The input folder consists of subfolders with following stricture
+  ├── <Name of Wikipedia article>
+  │   ├── aligned.swc
+  │   ├── audiometa.txt
+  │   ├── audio.ogg
+  │   ├── info.json
+  │   ├── wiki.html
+  │   ├── wiki.txt
+  │   └── wiki.xml
+
+
+## The destination folder will contain look enumerated .ogg and .txt files like this:
+  ├── audio
+  |   ├── 1.ogg
+  |   ├── 2.ogg
+  |   ...
+  └── text
+      ├── 1.txt     
+      ├── 2.txt     
+      ...
+"""
+
+import argparse
+import os
+import re
+import shutil
+import subprocess
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--input_folder", required=True, type=str, help="Input folder in which each subfolder contains an article"
+)
+parser.add_argument(
+    "--destination_folder", required=True, type=str, help="Destination folder with audio and text subfolder"
+)
+args = parser.parse_args()
+
+
+def replace_diacritics(text):
+    text = re.sub(r"[éèëēêęěė]", "e", text)
+    text = re.sub(r"[ãâāáäăâàąåạả]", "a", text)
+    text = re.sub(r"[úūüùưûů]", "u", text)
+    text = re.sub(r"[ôōóöõòő]", "o", text)
+    text = re.sub(r"[ćçč]", "c", text)
+    text = re.sub(r"[ïīíîıì]", "i", text)
+    text = re.sub(r"[ñńňņ]", "n", text)
+    text = re.sub(r"[țť]", "t", text)
+    text = re.sub(r"[łľ]", "l", text)
+    text = re.sub(r"[żžź]", "z", text)
+    text = re.sub(r"[ğ]", "g", text)
+    text = re.sub(r"[ř]", "r", text)
+    text = re.sub(r"[ý]", "y", text)
+    text = re.sub(r"[æ]", "ae", text)
+    text = re.sub(r"[œ]", "oe", text)
+    text = re.sub(r"[șşšś]", "s", text)
+    return text
+
+
+def get_audio(name, n):
+    """
+    Copies .ogg file. If there are several .ogg files, concatenates them.
+
+    Args:
+        name - name of folder within Spoken Wikipedia
+        n - integer that will serve as output file name, e.g. if n=1, file 1.ogg will be created
+    """
+    audio_path = os.path.join(args.input_folder, name, "audio.ogg")
+    if not os.path.exists(audio_path):
+        ##  Some folders have multiple .ogg files, so we need to first combine them into one file. Example:
+        ##  |── Universe
+        ##  │   ├── aligned.swc
+        ##  │   ├── audio1.ogg
+        ##  │   ├── audio2.ogg
+        ##  │   ├── audio3.ogg
+        ##  │   ├── audio4.ogg
+        ##  │   ├── audiometa.txt
+        ##  │   ├── info.json
+        ##  │   ├── wiki.html
+        ##  │   ├── wiki.txt
+        ##  │   └── wiki.xml
+
+        multiple_ogg_files = []
+        for i in range(1, 5):
+            path = os.path.join(args.input_folder, name, "audio" + str(i) + ".ogg")
+            if os.path.exists(path):
+                multiple_ogg_files.append(path)
+            else:
+                break
+        if len(multiple_ogg_files) == 0:
+            return
+        elif len(multiple_ogg_files) == 1:
+            shutil.copy(multiple_ogg_files[0], audio_path)
+        else:
+            tmp_file_name = "ffmeg_inputs.txt"
+            print("tmp_file_name=", tmp_file_name)
+            with open(tmp_file_name, "w", encoding="utf-8") as tmp_file:
+                for path in multiple_ogg_files:
+                    tmp_file.write("file '" + path + "'\n")
+            ffmpeg_cmd = ["ffmpeg", "-f", "concat", "-i", tmp_file_name, "-c", "copy", audio_path]
+            print("ffmpeg command:", ffmpeg_cmd)
+            subprocess.run(ffmpeg_cmd, check=True)
+
+    output_audio_path = args.destination_folder + "/audio/" + str(n) + ".ogg"
+    shutil.copy(audio_path, output_audio_path)
+
+
+def get_text(name, n):
+    """
+    Cleans wiki.txt.
+
+    Args:
+        name - name of folder within Spoken Wikipedia
+        n - integer that will serve as output file name, e.g. if n=1, file 1.txt will be created
+    """
+
+    # Then we need to clean the text
+    out_text = open(args.destination_folder + "/text/" + str(n) + ".txt", "w", encoding="utf-8")
+    with open(args.input_folder + "/" + name + "/wiki.txt", "r", encoding="utf-8") as f:
+        for line in f:
+            do_break = False
+            line2 = line.strip()
+            ref_parts = line2.split("<ref")
+            for idx, s in enumerate(ref_parts):
+                if idx != 0:
+                    s = "<ref" + s
+                if s.startswith("[[Image") and s.endswith("]]"):
+                    continue
+                if s.startswith("[[File") and s.endswith("]]"):
+                    continue
+                if s.startswith(":"):
+                    continue
+                if s.startswith("{|") or s.startswith("|}") or s.startswith("|") or s.startswith("!"):
+                    continue
+                if s.startswith("{{") and (s.endswith("}}") or "}}" not in s):
+                    continue
+                if s.startswith("{{pp-move"):
+                    continue
+                s = re.sub(r"\[\[Image\:[^\]]+\]\]", r"", s)
+                s = re.sub(r"\[\[File\:[^\]]+\]\]", r"", s)
+                s = re.sub(r"\[http[^\]]+\]", r"", s)
+                s = re.sub(r"<math>[^<>]+</math>", r"", s)
+                s = re.sub(r"<!\-\-.+\-\->", r"", s)  # <!--DashBot-->    can be inside <ref>
+                s = re.sub(r"<ref>.+</ref>", r"", s)
+                s = re.sub(r"<ref .+</ref>", r"", s)
+                s = re.sub(r"<ref[^<>]+/>", r"", s)
+                s = re.sub(r"<[^ <>]+>", r"", s)  # <sub>, <sup>, </u>
+                if (
+                    re.match(r"== *Notes *==", s)
+                    or re.match(r"== *References *==", s)
+                    or re.match(r"== *External links *==", s)
+                    or re.match(r"== *See also *==", s)
+                ):
+                    do_break = True
+                    break
+                s = re.sub(r"{{convert\|(\d+)\|(\w+)\|[^}]+}}", r"\g<1> \g<2>", s)  # {{convert|7600|lb|kg}}
+                s = re.sub(r"{{cquote\|", r"", s)
+                s = re.sub(r"{{[^{}]+}}", r"", s)
+                s = s.replace("{{", "").replace("}}", "")
+                s = re.sub(r"(lang[^()]+)", r"", s)  # (lang-bn...)
+                s = re.sub(r"==+", r"", s)
+                s = re.sub(r"''+", r" ", s)  # remove multiple quotes
+                s = re.sub(r" '", r" ", s)  # remove quote at the beginning
+                s = re.sub(r"' ", r" ", s)  # remove quote at the end
+                s = re.sub(r"[…\*]", r" ", s)
+                s = re.sub(r"\\u....", r" ", s)  # remove unicode
+                s = re.sub(r"&[^ ;&]+;", r"", s)  # &nbsp; &mdash;
+
+                s = replace_diacritics(s)
+
+                s = re.sub(r"\[\[[^\]]+\|([^\]]+)\]\]", r"\g<1>", s)  # if several variants, take the last one
+                s = re.sub(r"\[\[([^\]]+)\]\]", r"\g<1>", s)
+
+                out_text.write(s + "\n")
+            if do_break:
+                break
+    out_text.close()
+
+
+if __name__ == "__main__":
+    n = 0
+    for name in os.listdir(args.input_folder):
+        n += 1
+        if not os.path.exists(args.input_folder + "/" + name + "/wiki.txt"):
+            print("wiki.txt does not exist in " + name)
+            continue
+        get_audio(name, n)
+        get_text(name, n)

scripts/dataset_processing/spoken_wikipedia/run.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Download the Spoken Wikipedia corpus for English
+## Note, that there are some other languages available
+## @InProceedings{KHN16.518,
+##  author = {Arne K{\"o}hn and Florian Stegen and Timo Baumann},
+##  title = {Mining the Spoken Wikipedia for Speech Data and Beyond},
+##  booktitle = {Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC 2016)},
+##  year = {2016},
+##  month = {may},
+##  date = {23-28},
+##  location = {Portorož, Slovenia},
+##  editor = {Nicoletta Calzolari (Conference Chair) and Khalid Choukri and Thierry Declerck and Marko Grobelnik and Bente Maegaard and Joseph Mariani and Asuncion Moreno and Jan Odijk and Stelios Piperidis},
+##  publisher = {European Language Resources Association (ELRA)},
+##  address = {Paris, France},
+##  isbn = {978-2-9517408-9-1},
+##  islrn = {684-927-624-257-3/},
+##  language = {english}
+## }
+
+wget https://corpora.uni-hamburg.de/hzsk/de/islandora/object/file:swc-2.0_en-with-audio/datastream/TAR/en-with-audio.tar .
+tar -xvf en-with-audio.tar
+
+##  We get a folder English with 1339 subfolders, each subfolder corresponds to a Wikipedia article. Example:
+##  ├── Universal_suffrage
+##  │   ├── aligned.swc
+##  │   ├── audiometa.txt
+##  │   ├── audio.ogg
+##  │   ├── info.json
+##  │   ├── wiki.html
+##  │   ├── wiki.txt
+##  │   └── wiki.xml
+
+##  We will use two files: audio.ogg and wiki.txt
+
+## Some folders have multiple .ogg files, this will be handled during preprocess.py. Example:
+##  |── Universe
+##  │   ├── aligned.swc
+##  │   ├── audio1.ogg
+##  │   ├── audio2.ogg
+##  │   ├── audio3.ogg
+##  │   ├── audio4.ogg
+##  │   ├── audiometa.txt
+##  │   ├── info.json
+##  │   ├── wiki.html
+##  │   ├── wiki.txt
+##  │   └── wiki.xml
+
+## Some rare folders are incomplete, these will be skipped during preprocessing.
+
+## Rename some folders with special symbols because they cause problems to ffmpeg when concatening multiple .ogg files
+mv "english/The_Hitchhiker%27s_Guide_to_the_Galaxy" "english/The_Hitchhikers_guide_to_the_Galaxy"
+mv "english/SummerSlam_(2003)" "english/SummerSlam_2003"
+mv "english/Over_the_Edge_(1999)" "english/Over_the_Edge_1999"
+mv "english/Lost_(TV_series)" "english/Lost_TV_series"
+mv "english/S._A._Andr%c3%a9e%27s_Arctic_Balloon_Expedition_of_1897" "english/S_A_Andres_Arctic_Balloon_Expedition_of_1897"
+
+## path to NeMo repository, e.g. /home/user/NeMo
+NEMO_PATH=
+
+INPUT_DIR="english"
+OUTPUT_DIR=${INPUT_DIR}_result
+
+rm -rf $OUTPUT_DIR
+rm -rf ${INPUT_DIR}_prepared
+mkdir ${INPUT_DIR}_prepared
+mkdir ${INPUT_DIR}_prepared/audio
+mkdir ${INPUT_DIR}_prepared/text
+python ${NEMO_PATH}/scripts/dataset_processing/spoken_wikipedia/preprocess.py --input_folder ${INPUT_DIR} --destination_folder ${INPUT_DIR}_prepared
+
+## Now we have ${INPUT_DIR}_prepared folder with the following structure:
+##  ├── audio
+##  |   ├── 1.ogg
+##  |   ├── 2.ogg
+##  |   ...
+##  └── text
+##      ├── 1.txt     
+##      ├── 2.txt     
+##      ...
+
+MODEL_FOR_SEGMENTATION="stt_en_fastconformer_ctc_large"
+MODEL_FOR_RECOGNITION="stt_en_conformer_ctc_large"
+## We set this threshold as very permissive, later we will use other metrics for filtering
+THRESHOLD=-10
+
+${NEMO_PATH}/tools/ctc_segmentation/run_segmentation.sh \
+--SCRIPTS_DIR=${NEMO_PATH}/tools/ctc_segmentation/scripts \
+--MODEL_NAME_OR_PATH=${MODEL_FOR_SEGMENTATION} \
+--DATA_DIR=${INPUT_DIR}_prepared \
+--OUTPUT_DIR=${OUTPUT_DIR} \
+--MIN_SCORE=${THRESHOLD}
+
+# Thresholds for filtering
+CER_THRESHOLD=20
+WER_THRESHOLD=30
+CER_EDGE_THRESHOLD=30
+LEN_DIFF_RATIO_THRESHOLD=0.15
+EDGE_LEN=25
+BATCH_SIZE=1
+
+${NEMO_PATH}/tools/ctc_segmentation/run_filter.sh \
+--SCRIPTS_DIR=${NEMO_PATH}/tools/ctc_segmentation/scripts \
+--MODEL_NAME_OR_PATH=${MODEL_FOR_RECOGNITION} \
+--BATCH_SIZE=${BATCH_SIZE} \
+--MANIFEST=$OUTPUT_DIR/manifests/manifest.json \
+--INPUT_AUDIO_DIR=${INPUT_DIR}_prepared/audio/ \
+--EDGE_LEN=${EDGE_LEN} \
+--CER_THRESHOLD=${CER_THRESHOLD} \
+--WER_THRESHOLD=${WER_THRESHOLD} \
+--CER_EDGE_THRESHOLD=${CER_EDGE_THRESHOLD} \
+--LEN_DIFF_RATIO_THRESHOLD=${LEN_DIFF_RATIO_THRESHOLD}
+
+python ${NEMO_PATH}/examples/asr/speech_to_text_eval.py \
+  dataset_manifest=${OUTPUT_DIR}/manifests/manifest_transcribed_metrics_filtered.json \
+  use_cer=True \
+  only_score_manifest=True
+
+python ${NEMO_PATH}/examples/asr/speech_to_text_eval.py \
+  dataset_manifest=${OUTPUT_DIR}/manifests/manifest_transcribed_metrics_filtered.json \
+  use_cer=False \
+  only_score_manifest=True