add transcripts to meilisearch

This commit is contained in:
Christopher Moyer 2024-12-17 21:42:58 -05:00
parent af3fc06655
commit 86e6799edf
3 changed files with 773 additions and 28 deletions

668
Cargo.lock generated
View file

@ -115,7 +115,7 @@ dependencies = [
"nom", "nom",
"num-traits", "num-traits",
"rusticata-macros", "rusticata-macros",
"thiserror", "thiserror 1.0.69",
"time", "time",
] ]
@ -308,6 +308,12 @@ dependencies = [
"windows-targets 0.52.6", "windows-targets 0.52.6",
] ]
[[package]]
name = "base64"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.22.1" version = "0.22.1"
@ -375,6 +381,12 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
[[package]] [[package]]
name = "cbc" name = "cbc"
version = "0.1.2" version = "0.1.2"
@ -399,6 +411,12 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]] [[package]]
name = "chrono" name = "chrono"
version = "0.4.39" version = "0.4.39"
@ -453,7 +471,7 @@ checksum = "45b1f4c00870f07dc34adcac82bb6a72cc5aabca8536ba1797e01df51d2ce9a0"
dependencies = [ dependencies = [
"directories", "directories",
"serde", "serde",
"thiserror", "thiserror 1.0.69",
"toml", "toml",
] ]
@ -463,6 +481,15 @@ version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
[[package]]
name = "convert_case"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
dependencies = [
"unicode-segmentation",
]
[[package]] [[package]]
name = "cookie-factory" name = "cookie-factory"
version = "0.3.3" version = "0.3.3"
@ -674,6 +701,15 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "either"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "equivalent" name = "equivalent"
version = "1.0.1" version = "1.0.1"
@ -773,12 +809,48 @@ dependencies = [
"percent-encoding", "percent-encoding",
] ]
[[package]]
name = "futures"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.31" version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
[[package]]
name = "futures-executor"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.31" version = "0.3.31"
@ -813,12 +885,47 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
] ]
[[package]]
name = "futures-macro"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.90",
]
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.31" version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
[[package]]
name = "futures-task"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
[[package]]
name = "futures-util"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
[[package]] [[package]]
name = "generic-array" name = "generic-array"
version = "0.14.7" version = "0.14.7"
@ -836,8 +943,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys",
"libc", "libc",
"wasi", "wasi",
"wasm-bindgen",
] ]
[[package]] [[package]]
@ -846,6 +955,25 @@ version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
[[package]]
name = "h2"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e"
dependencies = [
"atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
"http",
"indexmap 2.7.0",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.12.3" version = "0.12.3"
@ -885,12 +1013,109 @@ dependencies = [
"digest", "digest",
] ]
[[package]]
name = "http"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "http-body"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
dependencies = [
"bytes",
"http",
]
[[package]]
name = "http-body-util"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
dependencies = [
"bytes",
"futures-util",
"http",
"http-body",
"pin-project-lite",
]
[[package]]
name = "httparse"
version = "1.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946"
[[package]] [[package]]
name = "humantime" name = "humantime"
version = "2.1.0" version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"h2",
"http",
"http-body",
"httparse",
"itoa",
"pin-project-lite",
"smallvec",
"tokio",
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.27.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
dependencies = [
"futures-util",
"http",
"hyper",
"hyper-util",
"rustls",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots",
]
[[package]]
name = "hyper-util"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http",
"http-body",
"hyper",
"pin-project-lite",
"socket2 0.5.8",
"tokio",
"tower-service",
"tracing",
]
[[package]] [[package]]
name = "iana-time-zone" name = "iana-time-zone"
version = "0.1.61" version = "0.1.61"
@ -1111,6 +1336,21 @@ dependencies = [
"windows-sys 0.48.0", "windows-sys 0.48.0",
] ]
[[package]]
name = "ipnet"
version = "2.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708"
[[package]]
name = "iso8601"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153"
dependencies = [
"nom",
]
[[package]] [[package]]
name = "itoa" name = "itoa"
version = "1.0.14" version = "1.0.14"
@ -1127,6 +1367,19 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "jsonwebtoken"
version = "9.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f"
dependencies = [
"base64 0.21.7",
"js-sys",
"ring",
"serde",
"serde_json",
]
[[package]] [[package]]
name = "lapin" name = "lapin"
version = "2.5.0" version = "2.5.0"
@ -1236,18 +1489,64 @@ dependencies = [
"serde-value", "serde-value",
"serde_json", "serde_json",
"serde_yaml", "serde_yaml",
"thiserror", "thiserror 1.0.69",
"thread-id", "thread-id",
"typemap-ors", "typemap-ors",
"winapi", "winapi",
] ]
[[package]]
name = "meilisearch-index-setting-macro"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "056e8c0652af81cc6525e0d9c0e1037ea7bcd77955dcd4aef1a1441be7ad7e55"
dependencies = [
"convert_case",
"proc-macro2",
"quote",
"structmeta",
"syn 2.0.90",
]
[[package]]
name = "meilisearch-sdk"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66958255878d712b4f2dece377a8661b41dc976ff15f564b91bfce8b4a619304"
dependencies = [
"async-trait",
"bytes",
"either",
"futures",
"futures-io",
"iso8601",
"jsonwebtoken",
"log",
"meilisearch-index-setting-macro",
"pin-project-lite",
"reqwest",
"serde",
"serde_json",
"thiserror 1.0.69",
"time",
"uuid",
"wasm-bindgen-futures",
"web-sys",
"yaup",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.7.4" version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "mime"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]] [[package]]
name = "minimal-lexical" name = "minimal-lexical"
version = "0.2.1" version = "0.2.1"
@ -1381,7 +1680,7 @@ dependencies = [
"rc2", "rc2",
"sha1", "sha1",
"sha2", "sha2",
"thiserror", "thiserror 1.0.69",
"x509-parser", "x509-parser",
] ]
@ -1445,6 +1744,12 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff"
[[package]]
name = "pin-utils"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]] [[package]]
name = "pinky-swear" name = "pinky-swear"
version = "6.2.0" version = "6.2.0"
@ -1553,6 +1858,58 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "quinn"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
dependencies = [
"bytes",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustls",
"socket2 0.5.8",
"thiserror 2.0.7",
"tokio",
"tracing",
]
[[package]]
name = "quinn-proto"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
dependencies = [
"bytes",
"getrandom",
"rand",
"ring",
"rustc-hash",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 2.0.7",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.5.8",
"tracing",
"windows-sys 0.59.0",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.37" version = "1.0.37"
@ -1629,7 +1986,52 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
dependencies = [ dependencies = [
"getrandom", "getrandom",
"libredox", "libredox",
"thiserror", "thiserror 1.0.69",
]
[[package]]
name = "reqwest"
version = "0.12.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-rustls",
"hyper-util",
"ipnet",
"js-sys",
"log",
"mime",
"once_cell",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls",
"rustls-pemfile",
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls",
"tokio-util",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"webpki-roots",
"windows-registry",
] ]
[[package]] [[package]]
@ -1653,6 +2055,12 @@ version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
name = "rustc-hash"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497"
[[package]] [[package]]
name = "rusticata-macros" name = "rusticata-macros"
version = "4.1.0" version = "4.1.0"
@ -1743,6 +2151,9 @@ name = "rustls-pki-types"
version = "1.10.1" version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37"
dependencies = [
"web-time",
]
[[package]] [[package]]
name = "rustls-webpki" name = "rustls-webpki"
@ -1870,13 +2281,25 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
dependencies = [
"form_urlencoded",
"itoa",
"ryu",
"serde",
]
[[package]] [[package]]
name = "serde_with" name = "serde_with"
version = "3.11.0" version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"chrono", "chrono",
"hex", "hex",
"indexmap 1.9.3", "indexmap 1.9.3",
@ -2007,6 +2430,29 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "structmeta"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329"
dependencies = [
"proc-macro2",
"quote",
"structmeta-derive",
"syn 2.0.90",
]
[[package]]
name = "structmeta-derive"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.90",
]
[[package]] [[package]]
name = "subtle" name = "subtle"
version = "2.6.1" version = "2.6.1"
@ -2035,6 +2481,15 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
[[package]] [[package]]
name = "synstructure" name = "synstructure"
version = "0.13.1" version = "0.13.1"
@ -2064,7 +2519,16 @@ version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [ dependencies = [
"thiserror-impl", "thiserror-impl 1.0.69",
]
[[package]]
name = "thiserror"
version = "2.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767"
dependencies = [
"thiserror-impl 2.0.7",
] ]
[[package]] [[package]]
@ -2078,6 +2542,17 @@ dependencies = [
"syn 2.0.90", "syn 2.0.90",
] ]
[[package]]
name = "thiserror-impl"
version = "2.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.90",
]
[[package]] [[package]]
name = "thread-id" name = "thread-id"
version = "4.2.2" version = "4.2.2"
@ -2129,6 +2604,21 @@ dependencies = [
"zerovec", "zerovec",
] ]
[[package]]
name = "tinyvec"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.42.0" version = "1.42.0"
@ -2136,6 +2626,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551"
dependencies = [ dependencies = [
"backtrace", "backtrace",
"bytes",
"libc", "libc",
"mio", "mio",
"pin-project-lite", "pin-project-lite",
@ -2180,6 +2671,16 @@ dependencies = [
"tokio-stream", "tokio-stream",
] ]
[[package]]
name = "tokio-rustls"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
dependencies = [
"rustls",
"tokio",
]
[[package]] [[package]]
name = "tokio-stream" name = "tokio-stream"
version = "0.1.17" version = "0.1.17"
@ -2191,6 +2692,19 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "tokio-util"
version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
dependencies = [
"bytes",
"futures-core",
"futures-sink",
"pin-project-lite",
"tokio",
]
[[package]] [[package]]
name = "toml" name = "toml"
version = "0.8.19" version = "0.8.19"
@ -2225,6 +2739,12 @@ dependencies = [
"winnow", "winnow",
] ]
[[package]]
name = "tower-service"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]] [[package]]
name = "tracing" name = "tracing"
version = "0.1.41" version = "0.1.41"
@ -2240,6 +2760,15 @@ name = "tracing-core"
version = "0.1.33" version = "0.1.33"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
dependencies = [
"once_cell",
]
[[package]]
name = "try-lock"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]] [[package]]
name = "typemap-ors" name = "typemap-ors"
@ -2262,6 +2791,12 @@ version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
[[package]]
name = "unicode-segmentation"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
[[package]] [[package]]
name = "unsafe-any-ors" name = "unsafe-any-ors"
version = "1.0.0" version = "1.0.0"
@ -2306,6 +2841,16 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "uuid"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
dependencies = [
"getrandom",
"wasm-bindgen",
]
[[package]] [[package]]
name = "version_check" name = "version_check"
version = "0.9.5" version = "0.9.5"
@ -2318,6 +2863,15 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
[[package]]
name = "want"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
dependencies = [
"try-lock",
]
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.11.0+wasi-snapshot-preview1" version = "0.11.0+wasi-snapshot-preview1"
@ -2349,6 +2903,19 @@ dependencies = [
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2"
dependencies = [
"cfg-if",
"js-sys",
"once_cell",
"wasm-bindgen",
"web-sys",
]
[[package]] [[package]]
name = "wasm-bindgen-macro" name = "wasm-bindgen-macro"
version = "0.2.99" version = "0.2.99"
@ -2378,6 +2945,48 @@ version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
[[package]]
name = "wasm-streams"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.26.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e"
dependencies = [
"rustls-pki-types",
]
[[package]] [[package]]
name = "whisper-worker" name = "whisper-worker"
version = "0.1.0" version = "0.1.0"
@ -2387,12 +2996,14 @@ dependencies = [
"lapin", "lapin",
"log", "log",
"log4rs", "log4rs",
"meilisearch-sdk",
"serde", "serde",
"serde_json", "serde_json",
"serde_with", "serde_with",
"tokio", "tokio",
"tokio-executor-trait", "tokio-executor-trait",
"tokio-reactor-trait", "tokio-reactor-trait",
"uuid",
] ]
[[package]] [[package]]
@ -2426,6 +3037,36 @@ dependencies = [
"windows-targets 0.52.6", "windows-targets 0.52.6",
] ]
[[package]]
name = "windows-registry"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result",
"windows-strings",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-strings"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result",
"windows-targets 0.52.6",
]
[[package]] [[package]]
name = "windows-sys" name = "windows-sys"
version = "0.48.0" version = "0.48.0"
@ -2619,10 +3260,21 @@ dependencies = [
"nom", "nom",
"oid-registry", "oid-registry",
"rusticata-macros", "rusticata-macros",
"thiserror", "thiserror 1.0.69",
"time", "time",
] ]
[[package]]
name = "yaup"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0144f1a16a199846cb21024da74edd930b43443463292f536b7110b4855b5c6"
dependencies = [
"form_urlencoded",
"serde",
"thiserror 1.0.69",
]
[[package]] [[package]]
name = "yoke" name = "yoke"
version = "0.7.5" version = "0.7.5"

View file

@ -4,7 +4,7 @@ version = "0.1.0"
edition = "2021" edition = "2021"
[dependencies] [dependencies]
tokio = { version = "1.42.0", features = ["rt-multi-thread", "rt", "macros"] } tokio = { version = "1.42.0", features = ["rt-multi-thread", "rt", "macros", "time"] }
chrono = {version = "0.4.39", features = ["serde"] } chrono = {version = "0.4.39", features = ["serde"] }
serde = { version = "1.0.216", features = ["derive"] } serde = { version = "1.0.216", features = ["derive"] }
serde_json = "1.0.133" serde_json = "1.0.133"
@ -15,3 +15,5 @@ tokio-executor-trait = "2.1.3"
tokio-reactor-trait = "1.1.0" tokio-reactor-trait = "1.1.0"
log = "0.4.22" log = "0.4.22"
log4rs = "1.3.0" log4rs = "1.3.0"
meilisearch-sdk = "0.27.1"
uuid = { version = "1.11.0", features = ["v4"] }

View file

@ -1,9 +1,10 @@
use serde_with::BoolFromInt; use serde_with::BoolFromInt;
use chrono::serde::ts_milliseconds; use chrono::serde::ts_milliseconds;
use std::env; use std::{env, fs};
use std::fs::File; use std::fs::File;
use std::path::Path; use std::io::Error;
use std::process::Command; use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use lapin::{Connection, ConnectionProperties, Consumer}; use lapin::{Connection, ConnectionProperties, Consumer};
@ -11,7 +12,11 @@ use lapin::message::{Delivery, DeliveryResult};
use lapin::options::{BasicAckOptions, BasicConsumeOptions}; use lapin::options::{BasicAckOptions, BasicConsumeOptions};
use lapin::types::FieldTable; use lapin::types::FieldTable;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use log::{info, warn}; use log::{error, info, trace, warn};
use meilisearch_sdk::client::Client;
use meilisearch_sdk::task_info::TaskInfo;
use meilisearch_sdk::tasks::Task;
use uuid::Uuid;
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
@ -45,6 +50,7 @@ _|"""""|_|"""""|_|"""""|_|"""""|_|"""""|_|"""""|_|"""""| {======|_|"""""|_|"""""
consumer.set_delegate({ consumer.set_delegate({
let processing_lock = Arc::clone(&processing_lock); let processing_lock = Arc::clone(&processing_lock);
move |delivery: DeliveryResult| { move |delivery: DeliveryResult| {
let meilisearch_client = Client::new(&cfg.meilisearch_config.connection_string, Some(&cfg.meilisearch_config.api_key)).expect("Couldn't create meilisearch client");
let processing_lock = Arc::clone(&processing_lock); let processing_lock = Arc::clone(&processing_lock);
async move { async move {
info!("Consuming mq message"); info!("Consuming mq message");
@ -65,39 +71,111 @@ _|"""""|_|"""""|_|"""""|_|"""""|_|"""""|_|"""""|_|"""""| {======|_|"""""|_|"""""
} }
let result = { let transcription_result = {
info!("Waiting for lock");
let _lock = processing_lock.lock(); let _lock = processing_lock.lock();
info!("Acquired lock!"); transcribe_call(path)
transcribe_call(path);
}; };
info!("Lock released");
info!("Call transcription done"); match transcription_result {
Ok(result) if result.status.success() => {
info!("Successfully transcribed {}", &transcription_request.audio_file_path);
if !result.stdout.is_empty() {
trace!("Stdout: {}", String::from_utf8_lossy(&result.stdout));
}
if !result.stderr.is_empty() {
trace!("Stderr: {}", String::from_utf8_lossy(&result.stderr));
}
}
Ok(result) => {
error!("Failed to transcribe {}, Exit code {}", &transcription_request.audio_file_path, result.status.code().unwrap());
error!("Stdout: {}", String::from_utf8_lossy(&result.stdout));
error!("Stderr: {}", String::from_utf8_lossy(&result.stderr));
return delivery.ack(BasicAckOptions::default()).await.unwrap();
}
Err(error) => {
error!("Failed to transcribe {}, {}", &transcription_request.audio_file_path, error);
return delivery.ack(BasicAckOptions::default()).await.unwrap();
}
}
let transcript_path = path.with_extension("txt");
let transcript = match fs::read_to_string(&transcript_path) {
Ok(transcript) => transcript,
Err(_) => {
error!("Failed to read transcript file {}", &transcript_path.display());
return delivery.ack(BasicAckOptions::default()).await.unwrap();
}
};
let meilisearch_task = match write_to_meilisearch(&meilisearch_client, transcription_request.call_metadata, transcript).await {
Ok(task) => task,
Err(error) => {
error!("Failed to send message to meilisearch, {}", error);
return delivery.ack(BasicAckOptions::default()).await.unwrap();
}
};
match wait_for_task_to_complete(&meilisearch_client, &meilisearch_task).await {
Ok(task) if task.is_failure() => {
error!("Failed to send message to meilisearch, {}", task.unwrap_failure());
}
Err(error) => {
error!("Failed to send message to meilisearch, {}", error);
}
_ => {}
}
delivery.ack(BasicAckOptions::default()).await.unwrap() delivery.ack(BasicAckOptions::default()).await.unwrap()
} }
} }
}); });
info!("Waiting for messages"); info!("Startup Complete!");
std::future::pending::<()>().await; std::future::pending::<()>().await;
} }
fn transcribe_call(file_path: &Path) { fn transcribe_call(file_path: &Path) -> Result<Output, Error> {
info!("Transcribing file {}", file_path.display()); info!("Transcribing file {}", file_path.display());
let output_directory = file_path.parent().unwrap(); let output_directory = file_path.parent().unwrap();
let output = Command::new("whisperx") Command::new("whisperx")
.args(["--language", "en"]) .args(["--language", "en"])
.args(["--model", "large-v3"]) .args(["--model", "large-v3"])
.args(["--batch_size", "4"]) .args(["--batch_size", "4"])
.args(["--compute_type", "int8"]) .args(["--compute_type", "int8"])
.args(["--output_format", "txt"]) .args(["--output_format", "txt"])
.args(["--output_dir", output_directory.parent().unwrap().to_str().unwrap()]) .args(["--output_dir", output_directory.to_str().unwrap()])
.arg(file_path) .arg(file_path)
.output().expect("TODO: panic message"); .output()
info!("Transcription done, {}", &output.status); }
info!("Std out: {}", String::from_utf8_lossy(&output.stdout));
info!("Std err: {}", String::from_utf8_lossy(&output.stderr)); async fn write_to_meilisearch(client: &Client, call: Call, transcript: String) -> Result<TaskInfo, meilisearch_sdk::errors::Error> {
let doc = MeilisearchCall {
id: String::from(Uuid::new_v4()),
transcript,
metadata: call
};
client
.index("calls")
.add_documents(&vec![&doc], None)
.await
}
async fn wait_for_task_to_complete(client: &Client, task: &TaskInfo) -> Result<Task, meilisearch_sdk::errors::Error> {
loop {
let result = client.get_task(task).await;
match &result {
Ok(task) if task.is_pending() => {
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
_ => {
return result;
}
}
}
} }
#[derive(Serialize, Deserialize, Default, Debug)] #[derive(Serialize, Deserialize, Default, Debug)]
@ -106,6 +184,13 @@ struct TranscriptionRequest {
call_metadata: Call call_metadata: Call
} }
#[derive(Serialize, Deserialize, Debug)]
struct MeilisearchCall {
id: String,
transcript: String,
metadata: Call
}
#[serde_with::serde_as] #[serde_with::serde_as]
#[derive(Serialize, Deserialize, Default, Debug)] #[derive(Serialize, Deserialize, Default, Debug)]
struct Call { struct Call {
@ -174,7 +259,7 @@ struct CallFrequency {
#[derive(Debug, Serialize, Deserialize, Clone)] #[derive(Debug, Serialize, Deserialize, Clone)]
struct AppConfig { struct AppConfig {
rabbit_mq_config: RabbitMqConfig, rabbit_mq_config: RabbitMqConfig,
file_storage_path: String meilisearch_config: MeilisearchConfig
} }
impl Default for AppConfig { impl Default for AppConfig {
@ -187,3 +272,9 @@ impl Default for AppConfig {
struct RabbitMqConfig { struct RabbitMqConfig {
connection_string: String, connection_string: String,
} }
#[derive(Debug, Serialize, Deserialize, Clone)]
struct MeilisearchConfig {
connection_string: String,
api_key: String,
}