datatrove/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
datatrove/data.py,sha256=V4RctBl3AJoml2cvr-SwqgEjiDl_IItu4n8HOHDlSAo,1145
datatrove/io.py,sha256=Pkz7o5Y6ewc_758-z2CLI_v8NFmItTaxiiJbVFT1HCo,14903
datatrove/assets/banned_subwords.txt,sha256=R3BEXNjSsWSV796IvE78OlIL3A8xN9FNhc09ibGUojI,1673
datatrove/assets/banned_words.txt,sha256=eq5RCczE2DVxZxaPo6kTMfovr9jrjMnDjMkUfCYQ-rA,1732
datatrove/assets/soft_banned_words.txt,sha256=5QQwMKzMozsONoppOZdXa-6kxYJ52mQb_0C04CKBsGM,674
datatrove/assets/url_filterblacklists.tar.gz,sha256=llndpkDhZ0JvXk3Aw4ChHsbg5dnpsbV13i1Jo0nVQqc,17678810
datatrove/executor/__init__.py,sha256=nBMsc2-ySpEAO-ArqC0ny6Rn6NhfpUMvub4Xe4PM1zA,82
datatrove/executor/base.py,sha256=aUJyhji86QqiXw0IyXw1iX9TWVZVFXny8IIqd5soqdM,6684
datatrove/executor/local.py,sha256=nrdSpyu6VsYQDfKjANNMJ9RocwCNhqqhc3Bulwj6lzY,6543
datatrove/executor/slurm.py,sha256=IlhbPExLG5X4gI2F3h1MnUSggUO81meI-IMG8Hfg0RM,14889
datatrove/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
datatrove/pipeline/base.py,sha256=TeUGOtySvcsJmj_jbHqUhIq_RCGZXWtqPtGP9Tu4iqc,4175
datatrove/pipeline/decont/__init__.py,sha256=_prPzXmokM5Zc98r-tTgAUS3z-5Se2guMt-MBzu_MFE,81
datatrove/pipeline/decont/n_grams.py,sha256=TDK-tml9gThlJ1J1_VTo6z5Ie6ViHJNw9prct60nNhU,10103
datatrove/pipeline/dedup/__init__.py,sha256=_PF3nVW0EYa1x4o6S79XsSqHBrYj0EL8QESDuNq1Gow,493
datatrove/pipeline/dedup/bloom_filter.py,sha256=mvt7ebLuLjIuYilnAa3fqHhLHBs9-QPwKoEvjtPapsg,7969
datatrove/pipeline/dedup/exact_substrings.py,sha256=7QboPm-xg4hKU6OYh5wSdZeVUCYWw6iOnrrJj97qiGo,14050
datatrove/pipeline/dedup/minhash.py,sha256=tANReQUgjZ1NrZ2yjOyfgyn2qpBarWntrPv3kQe3gzU,26332
datatrove/pipeline/dedup/sentence_dedup.py,sha256=Tep6TKJag_ipz9M_gwAN9QawDWcZIJ2ciaoF6iWi34k,21294
datatrove/pipeline/dedup/url_dedup.py,sha256=l9ghuy6YkmkDSVYQp61x_iOF34uAhYSk5ZDEdpjODBo,16640
datatrove/pipeline/extractors/__init__.py,sha256=sHzyoqkwven6X4XdKoWzvmHXq8BUbS8Kl-t1-kvrs8k,80
datatrove/pipeline/extractors/base.py,sha256=jTT-lZoJSlU830CRUbOSyIlaUwIYysxH8jeeRE3ZKR8,2267
datatrove/pipeline/extractors/modular.py,sha256=OzDkwS3vDQeTeNYcRImezh025qcelhcN8cYqE7u-z90,2049
datatrove/pipeline/extractors/trafilatura.py,sha256=1q0LVmEdx7p8q7SH9taOduz1-xLW1wilqsSDfHMG4-c,1754
datatrove/pipeline/filters/__init__.py,sha256=YmKhWjPyC7XNL9qpghktatbZy6IIJISoEHvJNmfyW8M,554
datatrove/pipeline/filters/base_filter.py,sha256=jjAkLlpSb9FXln97EyIPOFg9EzA46NxFPon2aGYJ7Jc,3337
datatrove/pipeline/filters/c4_filters.py,sha256=0hlkI8pWOV0z4beUtdg5Wf5IY-ggcmF4BBfQRCOiPy0,11622
datatrove/pipeline/filters/fasttext_filter.py,sha256=LJ2mjDBTkYXRJjKLSvd8ZbMeThSwB635A7Ip6gPnw4E,5187
datatrove/pipeline/filters/fineweb_quality_filter.py,sha256=MDyA5X34SsZsFcsac-rbrsPiTMs3Upn9zHcW8YpNtbE,2187
datatrove/pipeline/filters/gopher_quality_filter.py,sha256=IcXSxCU4oQ-XgdQe8rPpGYzG8uL_LfJrljqUhigXE3Q,5156
datatrove/pipeline/filters/gopher_repetition_filter.py,sha256=OalqzDvdyAc-Ldh4YpvjH84n-QJ_xIJPzC_xXXMtKv4,4935
datatrove/pipeline/filters/lambda_filter.py,sha256=TcEIMbpHB0pRecvoUDQ-2MnFwfpL8udGPgqZsbiIGs0,827
datatrove/pipeline/filters/language_filter.py,sha256=a2k1TpF7BVUvhoeHwbB6cXiIj2U0k_NilRruwCbBfCM,2639
datatrove/pipeline/filters/regex_filter.py,sha256=0heUgDmmP98qXuxocMNCZy63WLsl6F_DD2bAMgtOp6A,737
datatrove/pipeline/filters/sampler_filter.py,sha256=QhDn-6_L_kVALYIDMWSNmSR0A2NkS3rrS3iu1sIf2OI,741
datatrove/pipeline/filters/unigram_log_probs.py,sha256=bATv21XAxXJvrGiQ28tMKRLjMknwVvZzwJspzZGVH-c,2811
datatrove/pipeline/filters/url_filter.py,sha256=Ll0b46iBVc_VkL150G_HE2keaiKZ44S5FCJpzm1LWiE,5440
datatrove/pipeline/formatters/__init__.py,sha256=9AexxNkQhphgOuMftuPpfLVxQCnxeg8mC4OjvloRd3g,117
datatrove/pipeline/formatters/base.py,sha256=F3-AojKl3ZA2nTr_IoXIqfW_mTRcBFRfwWw2odjlgdc,665
datatrove/pipeline/formatters/ftfy.py,sha256=Ln3CJYfbu6WrR83FxYw1CccWf1McuTaoB_x-US38mJw,228
datatrove/pipeline/formatters/pii.py,sha256=V73FNrwuV-8B9802NDt5f_HjJml2cOXL3hyBAkptUmU,3535
datatrove/pipeline/formatters/symbol_lines_remover.py,sha256=htmuLW3dTwLy4XNXfV5OCsbNgk7gkMHcN299_6aF7Cg,1237
datatrove/pipeline/readers/__init__.py,sha256=bTCTLGdjs7YduMD5sY6q7_WM1iy43q5FL5h7mt64j5E,199
datatrove/pipeline/readers/base.py,sha256=L_ALS73vBX4np2kdvruWYY3DtayKfxrryN0-JqU9gQ4,9560
datatrove/pipeline/readers/csv.py,sha256=qXM_Jg9tGkAAJoXFbEekUm26qWchYeMlb73JIl6s7bc,3141
datatrove/pipeline/readers/huggingface.py,sha256=5ZsFqgIfkIg8IVIH2wF4fIFDnwfS6yFS-MXcGTptWtA,5649
datatrove/pipeline/readers/ipc.py,sha256=ntBQB9T4Ae62qS0IySLiJflBpQy9ZS1Wv9PCQXyLFHU,3877
datatrove/pipeline/readers/jsonl.py,sha256=pvBItv8u7HXzasGMnXABbH-KqQjqqfu8pGM8dPFSZuo,3609
datatrove/pipeline/readers/parquet.py,sha256=7BPb7uRJ2UrYe4caG6SANHL52saRutqC94hNJXuaQOc,3615
datatrove/pipeline/readers/warc.py,sha256=xqz1rXnqrx7z1pWRtZczhKx90XsxKZbRxMxu-5YYVW0,5197
datatrove/pipeline/stats/__init__.py,sha256=mmd0EjLUXvl52Qg33ZX1i9Wm9_kOthsbg5wpQh8O_mA,750
datatrove/pipeline/stats/base.py,sha256=XPNdXNy5osgsU9W-hcNG51NnhCYQugE3rOD8qpMO4qY,5609
datatrove/pipeline/stats/config.py,sha256=79mqjq_RWWIXVnDbkkIadqPZNCQ6yrwhicofEcO7FPc,771
datatrove/pipeline/stats/contamination_stats.py,sha256=-i1G3IS6eRTVCk50idcqBI9JMcma0ULYNnJZQgPNmCA,1797
datatrove/pipeline/stats/doc_stats.py,sha256=FJThwh9HqSc_VLdcoEUuS1c0R-K3z8XLfJDWqOOFS98,2097
datatrove/pipeline/stats/lang_stats.py,sha256=w9Vn8ArXjwlIkAOuplWB8RTS1Stm1bV0REanFTOXmzM,1296
datatrove/pipeline/stats/line_stats.py,sha256=J7yIkYlUT1LkNom1-nFs3IRpB72isN3OZ-X_9KOroBs,3658
datatrove/pipeline/stats/merger.py,sha256=ltpKwjbBUejAA189Lcflr-BXw_ypQySYGDOW5EZevog,3478
datatrove/pipeline/stats/paragraph_stats.py,sha256=aLTjPt_9BTp9WS_WgOW62ksl6qvl8JAwDHs11Gt2Jt8,2931
datatrove/pipeline/stats/perplexity_stats.py,sha256=684cn1r8BBGFiMoRJY9sevJ33bk9GD40dR3ay2VJKhc,1338
datatrove/pipeline/stats/sentence_stats.py,sha256=bmepjeMh3nVF4HRYsCAuISdUxKbv4XsEyQ0Y0Bgle20,2521
datatrove/pipeline/stats/token_stats.py,sha256=iSjdbdIAuJuyoXP6Lkatv1pC1RsjpIXLJuhoqN3UaIs,1399
datatrove/pipeline/stats/word_stats.py,sha256=8N0yli9Yc6QrtB0knTWWGASVXahpP-CB0YcAuA9XD54,3369
datatrove/pipeline/tokens/__init__.py,sha256=Av-j6fsikkwOpmo3PcawQxarEri_5JPXktTp2IrN4lo,198
datatrove/pipeline/tokens/context_shuffler.py,sha256=MOT2zQMR1jp4XPpid_NCRPqVCG31gPq03l7t5YhZ4dU,3331
datatrove/pipeline/tokens/counter.py,sha256=3K6OB47PGUlKYYtqxHlUGiDD6QexZrgQrnyWVRlH9co,3231
datatrove/pipeline/tokens/merger.py,sha256=1yv9ZrtjLq6bgWOe6LM1qCJVW8pFVVuGCgnA417iYjg,9112
datatrove/pipeline/tokens/tokenizer.py,sha256=wEkJ9v7kcwHCy5Di0AcH2EMYSGphg3vppTM9taM9SKo,19397
datatrove/pipeline/writers/__init__.py,sha256=4UPe-fekukvJ2cYpFACUlYd2Y0TTL-AS0Np6PcxJLq0,132
datatrove/pipeline/writers/disk_base.py,sha256=nDYh4K1h0jImu5bZ8xEuIZ-cZeq4ELv9TTzbazc124M,7256
datatrove/pipeline/writers/huggingface.py,sha256=pnF7BBxrTu6un2VbwzfcpqkW1B2JSYri41N86kXl7lo,5624
datatrove/pipeline/writers/jsonl.py,sha256=rIVNIOda0DRonf6c3Mkr0ovYdDFPKr0bJSTrJRL0faU,1668
datatrove/pipeline/writers/parquet.py,sha256=-zb7OO9APfEhPKTIWUlhUh1lbaL7kFvXcuGF17FRmIg,3121
datatrove/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
datatrove/tools/check_dataset.py,sha256=-1RnWemtTXZWHGNOFyZBsF3xCD-kZpeNDDJE26jYnQQ,3630
datatrove/tools/failed_logs.py,sha256=5JYaqh1CIVlsp7tu5-N5hMwzwjBIWpfzCCGWlqsEX24,2639
datatrove/tools/inspect_data.py,sha256=mf4T0spOJGehci39PWDMVpakuqIda6Pxd9_u_L9LUEA,5741
datatrove/tools/jobs_status.py,sha256=h-K5mlKNY1u7xgWNiPcwRzUw0NRdQIntjc15ZWaLstw,3437
datatrove/tools/launch_pickled_pipeline.py,sha256=HKHZ7cSEOmPA6KTldiFO20twAxY-5czGopi5PonMYX0,512
datatrove/tools/merge_stats.py,sha256=2n0VJRiV_0UTTpCtjK9HbecEjOaSS4bP5Uenv1Y3cR8,1175
datatrove/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
datatrove/utils/_import_utils.py,sha256=xDocbiYi11dXdw3-ilMvw9dVWACnZNoq_u-kBikEz8Y,2287
datatrove/utils/batching.py,sha256=g6BinvLQZrhNQr6dYbITJPfAiUt8F6aE5UOGCgRrJew,447
datatrove/utils/binaryio.py,sha256=Qasnh3KsdKy5PUco2gVc3Gio5BGfuiU39KZ73HdON1E,3049
datatrove/utils/dataset.py,sha256=ubPzo57K_H6ETZZABXDHakPPhyXio19CCo2jAbW2igM,5419
datatrove/utils/hashing.py,sha256=tMofxOQsCMj2mbkKCceMzJWra5nr-ApL3SNhuc2INS4,1522
datatrove/utils/lid.py,sha256=m3lyuiZu0r-0SI-qdEaw3ZjzNH66D19FUHlzgmspA-E,3066
datatrove/utils/logging.py,sha256=NoVn8IrdtHOOWf00cuvXVpj1GdREiSJFR7grrxAOt7I,2090
datatrove/utils/perplexity.py,sha256=AOUacQlb7kMnGQd93NtOjeC6I6d1SVgpZ87QhEyicZY,4654
datatrove/utils/stats.py,sha256=P_uAhM-3f6FG-wEBi9rsaO32xX_BU463n4Tn6E6VJ_U,14395
datatrove/utils/text.py,sha256=VJVgqcAHEst5Wr_R5DZ63L9xiV1N7BrW0Qsy_FGwsJY,4232
datatrove/utils/tokenization.py,sha256=0V_oGXr2UY94YikVvSnzWR1MRUZBlb5wAK47r1atKaI,1893
datatrove/utils/typeshelper.py,sha256=v0Uxaz5rLdzXxCNJeLZLoU1lchUGodyoL187mRGbATo,2703
datatrove/utils/word_tokenizers.py,sha256=iejw4GD4MqFjlbJLErhLY5J9bFAOJ7lS93YcPhtbUiY,13798
datatrove/utils/hashes/sha1.py,sha256=wffxM0B1aED2J_IScq8qzhk8WyrdGpZx8qfg4A-Z7yo,689
datatrove/utils/hashes/xxhash.py,sha256=kSE2atuhcReX8oOmFPCkr1OxBVK-VYrdRvsU1k36Z0k,148
datatrove-0.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
datatrove-0.3.0.dist-info/METADATA,sha256=HNqEcp0e5g1jnqkMzZwXN-5sxYdz_95YLcGxoqVGIxs,30275
datatrove-0.3.0.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
datatrove-0.3.0.dist-info/entry_points.txt,sha256=LMMMud15s4s6jucjsr57crbMppkvg6xRtN6unsgp2MM,330
datatrove-0.3.0.dist-info/top_level.txt,sha256=EA6CAg36D1YzT-oXcFMx0ImOvfCRDHZasZHwxcjFXSQ,10
datatrove-0.3.0.dist-info/RECORD,,
