feat: parse refactor document using knuffel

Merge pull request #10 from lavafroth/dependabot/cargo/clap-4.5.45
chore(deps): bump clap from 4.5.41 to 4.5.45
2025-09-02 14:03:16 +05:30 · 2025-09-01 05:29:39 +00:00 · 2025-09-01 05:24:10 +00:00 · 2025-08-26 12:02:58 +05:30 · 2025-08-26 11:59:13 +05:30 · 2025-08-26 11:58:53 +05:30
13 changed files with 453 additions and 189 deletions
--- a/.helix/languages.toml
+++ b/.helix/languages.toml
@@ -1,6 +1,11 @@
 [language-server.silos]
-command = "silos"
+command = "./target/debug/silos"
+args = ["lsp"]

 [[language]]
 name = "go"
-language-servers = [ { name = "silos" } ]
+language-servers = [ { name = "silos" }, "gopls" ]
+
+[[language]]
+name = "rust"
+language-servers = [ ]
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -9,6 +9,7 @@
            ],
            "command": [
                "silos"
+                "lsp"
            ]
        }
    ]
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17,6 +17,20 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"

+[[package]]
+name = "ahash"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
+dependencies = [
+ "cfg-if",
+ "getrandom 0.3.3",
+ "once_cell",
+ "serde",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.1.3"
@@ -26,6 +40,12 @@ dependencies = [
 "memchr",
 ]

+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
 [[package]]
 name = "anstream"
 version = "0.6.19"
@@ -99,7 +119,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -116,7 +136,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -146,6 +166,12 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"

+[[package]]
+name = "base64"
+version = "0.21.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
+
 [[package]]
 name = "base64"
 version = "0.22.1"
@@ -211,7 +237,7 @@ checksum = "7ecc273b49b3205b83d648f0690daa588925572cc5063745bfe547fe7ec8e1a1"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -282,6 +308,15 @@ dependencies = [
 "tracing",
 ]

+[[package]]
+name = "castaway"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "cc"
 version = "1.2.26"
@@ -298,10 +333,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
-name = "clap"
-version = "4.5.41"
+name = "chumsky"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9"
+checksum = "8eebd66744a15ded14960ab4ccdbfb51ad3b81f51f3f04a80adac98c985396c9"
+dependencies = [
+ "hashbrown 0.14.5",
+]
+
+[[package]]
+name = "clap"
+version = "4.5.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318"
 dependencies = [
 "clap_builder",
 "clap_derive",
@@ -309,9 +353,9 @@ dependencies = [

 [[package]]
 name = "clap_builder"
-version = "4.5.41"
+version = "4.5.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d"
+checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8"
 dependencies = [
 "anstream",
 "anstyle",
@@ -321,14 +365,14 @@ dependencies = [

 [[package]]
 name = "clap_derive"
-version = "4.5.41"
+version = "4.5.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491"
+checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6"
 dependencies = [
- "heck",
+ "heck 0.5.0",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -343,6 +387,21 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"

+[[package]]
+name = "compact_str"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "rustversion",
+ "ryu",
+ "serde",
+ "static_assertions",
+]
+
 [[package]]
 name = "console"
 version = "0.15.11"
@@ -433,7 +492,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "strsim",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -444,7 +503,16 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
 dependencies = [
 "darling_core",
 "quote",
- "syn",
+ "syn 2.0.101",
+]
+
+[[package]]
+name = "dary_heap"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728"
+dependencies = [
+ "serde",
 ]

 [[package]]
@@ -468,7 +536,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -489,7 +557,7 @@ dependencies = [
 "darling",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -499,7 +567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
 dependencies = [
 "derive_builder_core",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -519,7 +587,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 "unicode-xid",
 ]

@@ -552,7 +620,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -601,10 +669,10 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc"
 dependencies = [
- "heck",
+ "heck 0.5.0",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -751,7 +819,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -1088,6 +1156,10 @@ name = "hashbrown"
 version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+dependencies = [
+ "ahash",
+ "allocator-api2",
+]

 [[package]]
 name = "hashbrown"
@@ -1095,6 +1167,15 @@ version = "0.15.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"

+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -1424,18 +1505,9 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"

 [[package]]
 name = "itertools"
-version = "0.11.0"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itertools"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
 dependencies = [
 "either",
 ]
@@ -1462,12 +1534,39 @@ version = "6.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "12661358400b02cbbf1fbd05f0a483335490e8a6bd1867620f2eeb78f304a22f"
 dependencies = [
- "miette",
+ "miette 7.6.0",
 "num",
 "thiserror 1.0.69",
 "winnow 0.6.24",
 ]

+[[package]]
+name = "knuffel"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04bee6ddc6071011314b1ce4f7705fef6c009401dba4fd22cb0009db6a177413"
+dependencies = [
+ "base64 0.21.7",
+ "chumsky",
+ "knuffel-derive",
+ "miette 5.10.0",
+ "thiserror 1.0.69",
+ "unicode-width 0.1.14",
+]
+
+[[package]]
+name = "knuffel-derive"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91977f56c49cfb961e3d840e2e7c6e4a56bde7283898cf606861f1421348283d"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -1579,6 +1678,18 @@ dependencies = [
 "stable_deref_trait",
 ]

+[[package]]
+name = "miette"
+version = "5.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e"
+dependencies = [
+ "miette-derive 5.10.0",
+ "once_cell",
+ "thiserror 1.0.69",
+ "unicode-width 0.1.14",
+]
+
 [[package]]
 name = "miette"
 version = "7.6.0"
@@ -1586,10 +1697,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7"
 dependencies = [
 "cfg-if",
- "miette-derive",
+ "miette-derive 7.6.0",
 "unicode-width 0.1.14",
 ]

+[[package]]
+name = "miette-derive"
+version = "5.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
 [[package]]
 name = "miette-derive"
 version = "7.6.0"
@@ -1598,7 +1720,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -1651,7 +1773,7 @@ checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -1794,7 +1916,7 @@ dependencies = [
 "proc-macro-crate",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -1869,7 +1991,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -1944,7 +2066,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -1998,6 +2120,30 @@ dependencies = [
 "toml_edit",
 ]

+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.95"
@@ -2147,12 +2293,12 @@ dependencies = [

 [[package]]
 name = "rayon-cond"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9"
+checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"
 dependencies = [
 "either",
- "itertools 0.11.0",
+ "itertools",
 "rayon",
 ]

@@ -2426,7 +2572,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -2459,7 +2605,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -2491,7 +2637,7 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"

 [[package]]
 name = "silos"
-version = "5.2.2"
+version = "6.0.0"
 dependencies = [
 "anyhow",
 "candle-core",
@@ -2502,6 +2648,7 @@ dependencies = [
 "hf-hub",
 "hora",
 "kdl",
+ "knuffel",
 "serde_json",
 "tokenizers",
 "tokio",
@@ -2570,6 +2717,12 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"

+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "streaming-iterator"
 version = "0.1.9"
@@ -2588,6 +2741,17 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"

+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.101"
@@ -2616,7 +2780,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -2707,7 +2871,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -2718,7 +2882,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -2742,23 +2906,25 @@ dependencies = [

 [[package]]
 name = "tokenizers"
-version = "0.21.1"
+version = "0.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3169b3195f925496c895caee7978a335d49218488ef22375267fba5a46a40bd7"
+checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"
 dependencies = [
+ "ahash",
 "aho-corasick",
+ "compact_str",
+ "dary_heap",
 "derive_builder",
 "esaxx-rs",
- "getrandom 0.2.16",
+ "getrandom 0.3.3",
 "indicatif",
- "itertools 0.13.0",
- "lazy_static",
+ "itertools",
 "log",
 "macro_rules_attribute",
 "monostate",
 "onig",
 "paste",
- "rand 0.8.5",
+ "rand 0.9.1",
 "rayon",
 "rayon-cond",
 "regex",
@@ -2796,7 +2962,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -2933,7 +3099,7 @@ checksum = "84fd902d4e0b9a4b27f2f440108dc034e1758628a9b702f8ec61ad66355422fa"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -2961,7 +3127,7 @@ checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -3265,7 +3431,7 @@ dependencies = [
 "log",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 "wasm-bindgen-shared",
 ]

@@ -3300,7 +3466,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 "wasm-bindgen-backend",
 "wasm-bindgen-shared",
 ]
@@ -3708,7 +3874,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 "synstructure",
 ]

@@ -3720,7 +3886,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 "synstructure",
 ]

@@ -3741,7 +3907,7 @@ checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
@@ -3761,7 +3927,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 "synstructure",
 ]

@@ -3801,7 +3967,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.101",
 ]

 [[package]]
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "silos"
-version = "5.2.2"
+version = "6.0.0"
 edition = "2024"

 [dependencies]
@@ -8,13 +8,13 @@ anyhow = "1.0.98"
 candle-core = "0.9.1"
 candle-nn = "0.9.1"
 candle-transformers = "0.9.1"
-clap = { version = "4.5.41", features = ["derive"] }
+clap = { version = "4.5.45", features = ["derive"] }
 derive_more = { version = "2.0.1", features = ["display", "error"] }
 hf-hub = "0.4.2"
 hora = "0.1.1"
 kdl = "6.3.4"
 serde_json = "1.0.140"
-tokenizers = "0.21.1"
+tokenizers = "0.21.4"
 tracing = "0.1.41"
 tracing-subscriber = "0.3.19"
 tree-sitter = "0.25.8"
@@ -25,3 +25,4 @@ tokio = { version = "1.45.1", features = ["io-std", "macros", "rt", "rt-multi-th
 tower-lsp = "0.20.0"
 tree-sitter-javascript = "0.23.1"
 tree-sitter-cpp = "0.23.4"
+knuffel = "3.2.0"
--- a/README.md
+++ b/README.md
@@ -6,9 +6,7 @@ Dumb, proomptable modular snippet search.

 ## Installation

-### Binary releases
-
-There are no binary releases yet.
+You can download a binary from releases tab or build the project from source.

 ### From source

@@ -86,7 +84,7 @@ This API parses code into an AST (Abstract Syntax Tree) via tree-sitter and can
 ``` kdl
 description "describes the mutation collection"
 mutation {
-  expression "some ((beautiful) @adjective) AST expression"
+  expression "(some ((beautiful) @adjective) AST expression) @root"
  substitute {
    literal "hello"
    capture "adjective"
@@ -95,7 +93,7 @@ mutation {
 }

 mutation {
-  expression "another"
+  expression "(another) @root"
  substitute {
    literal "multiple mutations work"
    literal "as long as their expression"
@@ -106,14 +104,30 @@ mutation {

 - `description`: A textual description of the mutation collection.
 - `mutation`:  Defines individual code changes.
-  - `expression`: Uses tree-sitter to match and capture AST nodes with `@` prefixes, The special `@root` node is reserved for the entire expression.
+  - `expression`: Uses tree-sitter to match and capture AST nodes with `@` prefixes,
+  - The special `@root` node must be specify the expression to be replaced.
  - `substitute`:  Constructs the modified code using literals and captured arguments.

-See the example mutation collection in `./snippets/v2/go/mutations.kdl`.
+See the example mutation collection in `./snippets/v2/go/filepath-parent.kdl`.

 - The API performs a single-pass substitution based on the closest matching mutation.
 - Captured groups are used within the `substitute` block and the mutated code is returned.

+> Every capture group must contain the largest atom to be operated on.
+For example: if you wish to operate on elements of an array, capture each identifier inside the array
+
+Correct way: Here the `array` and `identifier` only hints about where the expression `root` lies.
+
+```
+(array (identifier @root))
+```
+
+Incorrect way: Here the root expression matches the block all the array elements inside the braces, not each element.
+
+```
+(array ((identifier)*) @entire-block-capture) @root
+```
+
 **Further reading**

 - [tree-sitter query snytax](https://tree-sitter.github.io/tree-sitter/using-parsers/queries/1-syntax.html) to create mutation expressions.
--- a/snippets/refactor/go/base64.kdl
+++ b/snippets/refactor/go/base64.kdl
@@ -0,0 +1,13 @@
+description "base64 import"
+mutation {
+	expression "(import_spec_list ((import_spec)* @spec)) @root"
+	substitute {
+		literal "("
+		literal "\n"
+		capture "spec"
+		literal "\n"
+		literal #""base64""#
+		literal "\n"
+		literal ")"
+	}
+}
--- a/snippets/refactor/go/filepath-parent.kdl
+++ b/snippets/refactor/go/filepath-parent.kdl
@@ -4,7 +4,7 @@ mutation {
 	(call_expression
 		function: (_) @func (#eq? @func "filepath.Base")
 		arguments: (_) @args
-	)
+	) @root
 	"""
 	substitute {
 		literal "filepath.Base(filepath.Dir(filepath.Clean"
--- a/src/args.rs
+++ b/src/args.rs
@@ -1,8 +1,15 @@
-use clap::Parser;
+use clap::{Args, Parser, Subcommand};
+use std::path::PathBuf;

 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
-pub(crate) struct Args {
+pub(crate) struct Cli {
+    #[command(subcommand)]
+    pub command: Command,
+}
+
+#[derive(Args, Debug)]
+pub(crate) struct Lsp {
    /// Run on the Nth GPU device.
    #[arg(long)]
    pub(crate) gpu: Option<usize>,
@@ -20,7 +27,35 @@ pub(crate) struct Args {
    pub(crate) snippets: std::path::PathBuf,
 }

-impl Args {
+#[derive(Args, Debug)]
+pub struct DumpExpression {
+    pub path: PathBuf,
+}
+
+#[derive(Args, Debug)]
+pub struct ShowCaptures {
+    pub path: PathBuf,
+    pub expression: String,
+}
+
+#[derive(Subcommand, Debug)]
+pub enum Ast {
+    /// Dump the S expression for a given source file
+    DumpExpression(DumpExpression),
+    /// Show what parts of a source file gets captured by an S expression
+    ShowCaptures(ShowCaptures),
+}
+
+#[derive(Subcommand, Debug)]
+pub enum Command {
+    /// quick actions to dump, modify and verify abstract syntax trees
+    #[command(subcommand)]
+    Ast(Ast),
+    /// spawn a language server for use with a text editor
+    Lsp(Lsp),
+}
+
+impl Lsp {
    pub(crate) fn resolve_model_and_revision(&self) -> (String, String) {
        let default_model = "sentence-transformers/all-MiniLM-L6-v2".to_string();
        let default_revision = "refs/pr/21".to_string();
--- a/src/lsp.rs
+++ b/src/lsp.rs
@@ -60,12 +60,18 @@ impl LanguageServer for Backend {
    }

    async fn did_open(&self, params: DidOpenTextDocumentParams) {
-        self.body.lock().await.insert(params.text_document.uri, params.text_document.text);
+        self.body
+            .lock()
+            .await
+            .insert(params.text_document.uri, params.text_document.text);
    }

    async fn did_change(&self, params: DidChangeTextDocumentParams) {
        if let Some(body) = params.content_changes.into_iter().next() {
-            self.body.lock().await.insert(params.text_document.uri, body.text);
+            self.body
+                .lock()
+                .await
+                .insert(params.text_document.uri, body.text);
        }
    }

--- a/src/main.rs
+++ b/src/main.rs
@@ -3,7 +3,7 @@ use clap::Parser;
 use hora::core::{ann_index::ANNIndex, metrics::Metric::Euclidean};
 use hora::index::hnsw_idx::HNSWIndex;
 use kdl::KdlDocument;
-use state::State;
+use state::{State, dump_expression};
 use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::Mutex;
@@ -13,14 +13,43 @@ mod args;
 mod embed;
 mod lsp;
 mod mutation;
-mod state;
 mod sources;
+mod state;

 #[tokio::main]
 async fn main() -> Result<()> {
    tracing_subscriber::fmt::init();
-    let args = args::Args::parse();
+    let args = match args::Cli::parse().command {
+        args::Command::Ast(ast) => {
+            match ast {
+                args::Ast::DumpExpression(source_file) => {
+                    println!("{}", dump_expression(&source_file.path)?);
+                }
+                args::Ast::ShowCaptures(show_captures) => {
+                    let source = std::fs::read_to_string(&show_captures.path).unwrap();
+                    let source_bytes = source.as_bytes();
+                    let extension = show_captures.path.extension().unwrap().to_str().unwrap();
+                    let langfn = state::Refactor::get_lang(extension).unwrap();
+                    let mut parser = tree_sitter::Parser::new();
+                    parser.set_language(&langfn).unwrap();
+                    let tree = parser.parse(source_bytes, None).unwrap();
+                    let root_node = tree.root_node();
+                    let cooked = mutation::query(
+                        root_node,
+                        &show_captures.expression,
+                        &langfn,
+                        source_bytes,
+                    );
+                    println!("{:#?}", cooked);
+                }
+            }
+            return Ok(());
+        }
+        args::Command::Lsp(lsp) => lsp,
+    };
+
    let (model_id, revision) = args.resolve_model_and_revision();
+
    let embed = embed::Embed::new(args.gpu, &model_id, &revision)?;
    let mut dict = HashMap::default();
    let dimensions = embed.hidden_size;
@@ -64,7 +93,10 @@ async fn main() -> Result<()> {
                .or_insert_with(|| HNSWIndex::new(dimensions, &Default::default()));

            current_lang_index
-                .add(&embed.embed(&mutations.description)?, mutations_collection.len())
+                .add(
+                    &embed.embed(&mutations.description)?,
+                    mutations_collection.len(),
+                )
                .map_err(E::msg)?;
            mutations_collection.push(mutations);
        }
--- a/src/mutation.rs
+++ b/src/mutation.rs
@@ -2,92 +2,34 @@ use std::collections::HashMap;
 use std::path::Path;
 use tracing::debug;
 use tree_sitter::{Language, Node, Query, QueryCursor, StreamingIterator};
+use anyhow::Result;

-use anyhow::{Result, bail};
-use kdl::KdlDocument;
-
-#[derive(Debug)]
-pub struct Mutation {
-    pub expression: String,
-    pub substitute: Vec<Substitute>,
-}
-
+#[derive(knuffel::Decode, Debug)]
 pub struct MutationCollection {
+    #[knuffel(child, unwrap(argument))]
    pub description: String,
-    pub mutations: Vec<Mutation>,
+    #[knuffel(children)]
+    mutations: Vec<Mutation>,
 }

-#[derive(Debug)]
-pub enum Substitute {
-    Literal(String),
-    Capture(String),
+#[derive(knuffel::Decode, Debug)]
+struct Mutation {
+    #[knuffel(child, unwrap(argument))]
+    expression: String,
+    #[knuffel(child, unwrap(children))]
+    substitute: Vec<Substitute>,
+}
+
+#[derive(knuffel::Decode, Debug)]
+enum Substitute {
+    Literal(#[knuffel(argument)] String),
+    Capture(#[knuffel(argument)] String),
 }

 pub fn from_path<P: AsRef<Path>>(path: P) -> Result<MutationCollection> {
-    let contents = std::fs::read_to_string(path)?;
-    let doc: KdlDocument = contents.parse()?;
-    let mut mutations = vec![];
-
-    let mut description = None;
-
-    for node in doc.nodes() {
-        let node_name = node.name().value();
-
-        if node_name != "mutation" && node_name != "description" {
-            bail!(
-                "document root must only contain `mutation` or `description` nodes: got {node_name}"
-            );
-        }
-
-        if node_name == "description" {
-            description.replace(
-                node.entry(0)
-                    .unwrap()
-                    .value()
-                    .as_string()
-                    .unwrap()
-                    .to_string(),
-            );
-            continue;
-        }
-
-        let node = node.children().unwrap();
-        let Some(expression) = node.get_arg("expression").and_then(|v| v.as_string()) else {
-            bail!("mutation node must contain an expression");
-        };
-        let Some(substitute) = node.get("substitute") else {
-            bail!("mutation node must contain an substitute");
-        };
-
-        let children = substitute.children().unwrap().nodes();
-        let mut substitute = vec![];
-        for child in children {
-            let attrib = child.entry(0).unwrap().value().as_string().unwrap();
-            let substitutor = match child.name().value() {
-                "literal" => Substitute::Literal(attrib.to_string()),
-                "capture" => Substitute::Capture(attrib.to_string()),
-                _ => unreachable!(),
-            };
-
-            substitute.push(substitutor);
-        }
-
-        let expression = format!("({expression}) @root");
-
-        mutations.push(Mutation {
-            expression,
-            substitute,
-        })
-    }
-
-    let Some(description) = description else {
-        bail!("mutation collection contains no `description`");
-    };
-
-    Ok(MutationCollection {
-        description,
-        mutations,
-    })
+    let contents = std::fs::read_to_string(&path)?;
+    let val = knuffel::parse(path.as_ref().to_str().unwrap(), &contents)?;
+    Ok(val)
 }

 pub fn apply(
@@ -127,7 +69,7 @@ pub fn apply(
 }

 #[derive(Debug)]
-struct QueryCooked {
+pub struct QueryCooked {
    captures: HashMap<String, String>,
    end: usize,
    start: usize,
@@ -152,7 +94,7 @@ fn split_at_indices<'a>(c: &'a [u8], idx: &[usize]) -> SplitMap<'a> {
    SplitMap { values, indices }
 }

-fn query<'a>(
+pub fn query<'a>(
    node: Node<'a>,
    expr: &'a str,
    lang: &Language,
@@ -164,6 +106,7 @@ fn query<'a>(
    let mut query_matches = qc.matches(&query, node, source_bytes);

    let capture_names = query.capture_names();
+    // println!("names: {capture_names:#?}");

    let mut cooked = vec![];

@@ -171,19 +114,36 @@ fn query<'a>(
        let mut capture_cooked = HashMap::new();
        let mut start = 0;
        let mut end = 0;
-        for cap in matcha.captures {
-            let Some(name) = capture_names.get(cap.index as usize) else {
-                continue;
-            };
-            if *name == "root" {
-                start = cap.node.start_byte();
-                end = cap.node.end_byte();
+        if matcha.captures.is_empty() {
+            continue;
+        }
+        //     println!("match {:#?}", matcha.id());
+
+        for (ix, name) in capture_names.iter().enumerate() {
+            let nodes = matcha.nodes_for_capture_index(ix.try_into().unwrap());
+            let mut start_pos = None;
+            let mut end_pos = None;
+            //         println!("matches for {name}");
+            for node in nodes {
+                if start_pos.is_none() {
+                    start_pos.replace(node.start_byte());
+                }
+                end_pos.replace(node.end_byte());
+                //             println!("hit {node:#?}");
+            }
+            if start_pos.or(end_pos).is_none() {
                continue;
            }
-            capture_cooked.insert(
-                name.to_string(),
-                cap.node.utf8_text(source_bytes).unwrap().to_string(),
-            );
+            if *name == "root" {
+                start = start_pos.unwrap();
+                end = end_pos.unwrap();
+            }
+            let range = start_pos.unwrap()..end_pos.unwrap();
+            //         println!("match range for {name}: {:#?}", range);
+            let text_bytes = &source_bytes[range];
+            let text = std::str::from_utf8(text_bytes).unwrap();
+            //         println!("text: {text}");
+            capture_cooked.insert(name.to_string(), text.to_string());
        }
        cooked.push(QueryCooked {
            start,
--- a/src/sources.rs
+++ b/src/sources.rs
@@ -1,23 +1,32 @@
-use std::{fs, io, path::{Path, PathBuf}, collections::HashMap};
+use std::{
+    collections::HashMap,
+    fs, io,
+    path::{Path, PathBuf},
+};

 pub fn rule_files<P: AsRef<Path>>(path: P) -> io::Result<HashMap<String, Vec<PathBuf>>> {
    let per_language_dirs: Vec<_> = fs::read_dir(path)?
-    .filter_map(|res| res.ok())
-    .map(|direntry| direntry.path())
-    .filter(|dir| dir.is_dir()).collect();
+        .filter_map(|res| res.ok())
+        .map(|direntry| direntry.path())
+        .filter(|dir| dir.is_dir())
+        .collect();

    let mut basename_to_paths = HashMap::new();

    for language_dir in per_language_dirs {
-        let Some(dirname) = language_dir.file_stem().and_then(|v|v.to_str()).map(|v| v.to_string()) else {
+        let Some(dirname) = language_dir
+            .file_stem()
+            .and_then(|v| v.to_str())
+            .map(|v| v.to_string())
+        else {
            continue;
        };
        let rule_file_paths: Vec<_> = fs::read_dir(&language_dir)?
-        .filter_map(|res| res.ok())
-        .map(|entry| entry.path())
-        .filter(|file| file.is_file() && file.extension().is_some_and(|ext| ext == "kdl"))
-        .map(|path| path.to_path_buf())
-        .collect();
+            .filter_map(|res| res.ok())
+            .map(|entry| entry.path())
+            .filter(|file| file.is_file() && file.extension().is_some_and(|ext| ext == "kdl"))
+            .map(|path| path.to_path_buf())
+            .collect();
        basename_to_paths.insert(dirname, rule_file_paths);
    }
    Ok(basename_to_paths)
--- a/src/state.rs
+++ b/src/state.rs
@@ -4,6 +4,7 @@ use derive_more::Error;
 use hora::core::ann_index::ANNIndex;
 use hora::index::hnsw_idx::HNSWIndex;
 use std::collections::HashMap;
+use std::path::Path;
 use tree_sitter::Parser;

 #[derive(Debug, Display, Error)]
@@ -22,7 +23,7 @@ pub struct Refactor {
 }

 impl Refactor {
-    fn get_lang(s: &str) -> Result<tree_sitter::Language, Error> {
+    pub fn get_lang(s: &str) -> Result<tree_sitter::Language, Error> {
        Ok(match s {
            "go" => tree_sitter_go::LANGUAGE,
            "c" | "h" => tree_sitter_c::LANGUAGE,
@@ -81,6 +82,27 @@ impl Refactor {
        Ok(collected)
    }
 }
+
+pub fn dump_expression(path: &Path) -> Result<String, Error> {
+    let Some(lang) = path.extension() else {
+        return Err(Error::UnknownLang);
+    };
+    let lang = lang.to_str().ok_or(Error::UnknownLang)?;
+    let langfn = Refactor::get_lang(lang)?;
+    let mut parser = Parser::new();
+    parser
+        .set_language(&langfn)
+        .map_err(|_| Error::UnknownLang)?;
+
+    let source_code = std::fs::read_to_string(path).map_err(|_| Error::SnippetParsing)?;
+    let source_bytes = source_code.as_bytes();
+    let tree = parser
+        .parse(source_bytes, None)
+        .ok_or(Error::SnippetParsing)?;
+    let root_node = tree.root_node();
+    Ok(root_node.to_sexp().to_string())
+}
+
 pub struct Generate {
    pub dict: HashMap<String, HNSWIndex<f32, String>>,
 }
Author	SHA1	Message	Date
Himadri Bhattacharjee	fcf98006c4	feat: parse refactor document using knuffel	2025-09-02 14:03:16 +05:30
Himadri Bhattacharjee	6eab02575a	Merge pull request #10 from lavafroth/dependabot/cargo/clap-4.5.45 chore(deps): bump clap from 4.5.41 to 4.5.45	2025-09-01 05:29:39 +00:00
Himadri Bhattacharjee	80fe8a3f16	Merge pull request #8 from lavafroth/dependabot/cargo/tokenizers-0.21.4 chore(deps): bump tokenizers from 0.21.1 to 0.21.4	2025-09-01 05:24:10 +00:00
Himadri Bhattacharjee	60256c06cc	feat: update vscode lsp config	2025-08-26 12:02:58 +05:30
Himadri Bhattacharjee	caf4f51d22	Merge branch 'dump-expression'	2025-08-26 11:59:13 +05:30
Himadri Bhattacharjee	a543e80a04	ver: bump version for next release	2025-08-26 11:58:53 +05:30
Himadri Bhattacharjee	f0c137ade4	fix: reintroduce the `root` node for anchoring flexibility	2025-08-26 11:57:15 +05:30
Himadri Bhattacharjee	7b0f818d38	feat: parse capture groups with `+` or `*` wildcards	2025-08-26 10:41:54 +05:30
dependabot[bot]	4195cbb734	chore(deps): bump clap from 4.5.41 to 4.5.45 Bumps [clap](https://github.com/clap-rs/clap) from 4.5.41 to 4.5.45. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.5.41...clap_complete-v4.5.45) --- updated-dependencies: - dependency-name: clap dependency-version: 4.5.45 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2025-08-19 05:05:31 +00:00
Himadri Bhattacharjee	e5602c688c	add subcommands for ast quick actions	2025-08-18 16:57:03 +05:30
Himadri Bhattacharjee	89e4c3b5fb	docs: document binary releases	2025-08-18 10:29:54 +05:30
Himadri Bhattacharjee	7d9c3a448f	feat: cli flag to dump S expression for a source file TODO: move to being a subcommand	2025-08-17 17:34:43 +05:30
dependabot[bot]	e348b9a830	chore(deps): bump tokenizers from 0.21.1 to 0.21.4 Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.21.1 to 0.21.4. - [Release notes](https://github.com/huggingface/tokenizers/releases) - [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md) - [Commits](https://github.com/huggingface/tokenizers/compare/v0.21.1...v0.21.4) --- updated-dependencies: - dependency-name: tokenizers dependency-version: 0.21.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>	2025-08-12 06:07:01 +00:00