Compare commits

...

7 commits

Author SHA1 Message Date
c8b8b56456 cuda: support cudnn manifests
Also changes the schemas a bit (still don't care about migrations),
switches to named_params!, fixes up perf issues when ingesting manifests
and/or hrefs into sqlite, etc. Adds sample queries such as "CudaArtifact
conflicts" to datasette config, which explain some issues associated
with choosing an evalModules schema on the cudaPackages side.
2025-05-09 04:05:20 +00:00
26538edf4f shell: add datasette wrapper with metadata.json 2025-05-09 04:05:20 +00:00
1ac96a4356 sql: name tables as Types, sample success optional
...and do not load blobs over 1M into memory
2025-05-09 04:05:20 +00:00
a8a722b002 sql: rename migrations 2025-04-28 20:42:35 +00:00
cfff120c9b feat(fetch-listing): poc recursive fetching 2025-04-27 16:54:04 +00:00
6df68a7e9c cli: log to stderr 2025-04-20 17:53:54 +00:00
9de0315810 Ingestable: move into subcommand 2025-04-20 09:54:20 +00:00
21 changed files with 2328 additions and 178 deletions

291
Cargo.lock generated
View file

@ -342,6 +342,12 @@ version = "3.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.10.1" version = "1.10.1"
@ -512,6 +518,29 @@ dependencies = [
"typenum", "typenum",
] ]
[[package]]
name = "cssparser"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3"
dependencies = [
"cssparser-macros",
"dtoa-short",
"itoa",
"phf",
"smallvec",
]
[[package]]
name = "cssparser-macros"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
dependencies = [
"quote",
"syn 2.0.100",
]
[[package]] [[package]]
name = "darling" name = "darling"
version = "0.20.11" version = "0.20.11"
@ -563,6 +592,17 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "derive_more"
version = "0.99.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]] [[package]]
name = "digest" name = "digest"
version = "0.10.7" version = "0.10.7"
@ -591,6 +631,27 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "dtoa"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04"
[[package]]
name = "dtoa-short"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
dependencies = [
"dtoa",
]
[[package]]
name = "ego-tree"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8"
[[package]] [[package]]
name = "either" name = "either"
version = "1.15.0" version = "1.15.0"
@ -727,6 +788,16 @@ dependencies = [
"percent-encoding", "percent-encoding",
] ]
[[package]]
name = "futf"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
dependencies = [
"mac",
"new_debug_unreachable",
]
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.31" version = "0.3.31"
@ -816,6 +887,15 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]] [[package]]
name = "gcp_auth" name = "gcp_auth"
version = "0.12.3" version = "0.12.3"
@ -853,6 +933,15 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "getopts"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
dependencies = [
"unicode-width 0.1.14",
]
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.15" version = "0.2.15"
@ -958,6 +1047,18 @@ dependencies = [
"windows-sys 0.59.0", "windows-sys 0.59.0",
] ]
[[package]]
name = "html5ever"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c"
dependencies = [
"log",
"mac",
"markup5ever",
"match_token",
]
[[package]] [[package]]
name = "http" name = "http"
version = "1.3.1" version = "1.3.1"
@ -1429,6 +1530,37 @@ version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "mac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "markup5ever"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18"
dependencies = [
"log",
"phf",
"phf_codegen",
"string_cache",
"string_cache_codegen",
"tendril",
]
[[package]]
name = "match_token"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]] [[package]]
name = "matchers" name = "matchers"
version = "0.1.0" version = "0.1.0"
@ -1509,6 +1641,12 @@ dependencies = [
"tempfile", "tempfile",
] ]
[[package]]
name = "new_debug_unreachable"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
[[package]] [[package]]
name = "nu-ansi-term" name = "nu-ansi-term"
version = "0.46.0" version = "0.46.0"
@ -1685,6 +1823,58 @@ dependencies = [
"indexmap 2.9.0", "indexmap 2.9.0",
] ]
[[package]]
name = "phf"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
dependencies = [
"phf_macros",
"phf_shared",
]
[[package]]
name = "phf_codegen"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
dependencies = [
"phf_generator",
"phf_shared",
]
[[package]]
name = "phf_generator"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared",
"rand 0.8.5",
]
[[package]]
name = "phf_macros"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]]
name = "phf_shared"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
dependencies = [
"siphasher",
]
[[package]] [[package]]
name = "pin-project" name = "pin-project"
version = "1.1.10" version = "1.1.10"
@ -1744,6 +1934,12 @@ dependencies = [
"zerocopy", "zerocopy",
] ]
[[package]]
name = "precomputed-hash"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]] [[package]]
name = "prettyplease" name = "prettyplease"
version = "0.2.32" version = "0.2.32"
@ -2300,6 +2496,21 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "scraper"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "527e65d9d888567588db4c12da1087598d0f6f8b346cc2c5abc91f05fc2dffe2"
dependencies = [
"cssparser",
"ego-tree",
"getopts",
"html5ever",
"precomputed-hash",
"selectors",
"tendril",
]
[[package]] [[package]]
name = "security-framework" name = "security-framework"
version = "2.11.1" version = "2.11.1"
@ -2336,6 +2547,25 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "selectors"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8"
dependencies = [
"bitflags 2.9.0",
"cssparser",
"derive_more",
"fxhash",
"log",
"new_debug_unreachable",
"phf",
"phf_codegen",
"precomputed-hash",
"servo_arc",
"smallvec",
]
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.219" version = "1.0.219"
@ -2431,6 +2661,15 @@ dependencies = [
"syn 2.0.100", "syn 2.0.100",
] ]
[[package]]
name = "servo_arc"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae65c4249478a2647db249fb43e23cec56a2c8974a427e7bd8cb5a1d0964921a"
dependencies = [
"stable_deref_trait",
]
[[package]] [[package]]
name = "sharded-slab" name = "sharded-slab"
version = "0.1.7" version = "0.1.7"
@ -2455,8 +2694,12 @@ dependencies = [
"futures", "futures",
"reqwest", "reqwest",
"rusqlite", "rusqlite",
"scraper",
"serde",
"serde_json",
"snix-castore", "snix-castore",
"tokio", "tokio",
"tokio-stream",
"tokio-util", "tokio-util",
"url", "url",
] ]
@ -2470,6 +2713,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "siphasher"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.9" version = "0.4.9"
@ -2588,6 +2837,31 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "string_cache"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f"
dependencies = [
"new_debug_unreachable",
"parking_lot",
"phf_shared",
"precomputed-hash",
"serde",
]
[[package]]
name = "string_cache_codegen"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro2",
"quote",
]
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.11.1" version = "0.11.1"
@ -2676,6 +2950,17 @@ dependencies = [
"windows-sys 0.59.0", "windows-sys 0.59.0",
] ]
[[package]]
name = "tendril"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
dependencies = [
"futf",
"mac",
"utf-8",
]
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.69" version = "1.0.69"
@ -3129,6 +3414,12 @@ dependencies = [
"percent-encoding", "percent-encoding",
] ]
[[package]]
name = "utf-8"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]] [[package]]
name = "utf16_iter" name = "utf16_iter"
version = "1.0.5" version = "1.0.5"

743
Cargo.nix
View file

@ -1104,6 +1104,19 @@ rec {
}; };
resolvedDefaultFeatures = [ "default" ]; resolvedDefaultFeatures = [ "default" ];
}; };
"byteorder" = rec {
crateName = "byteorder";
version = "1.5.0";
edition = "2021";
sha256 = "0jzncxyf404mwqdbspihyzpkndfgda450l0893pz5xj685cg5l0z";
authors = [
"Andrew Gallant <jamslam@gmail.com>"
];
features = {
"default" = [ "std" ];
};
resolvedDefaultFeatures = [ "default" "std" ];
};
"bytes" = rec { "bytes" = rec {
crateName = "bytes"; crateName = "bytes";
version = "1.10.1"; version = "1.10.1";
@ -1588,6 +1601,65 @@ rec {
"rand_core" = [ "dep:rand_core" ]; "rand_core" = [ "dep:rand_core" ];
}; };
}; };
"cssparser" = rec {
crateName = "cssparser";
version = "0.34.0";
edition = "2018";
sha256 = "1qx3hha392szcl812l6hp0d4029gg8x62cl4nf0byqgdv0f6vimp";
authors = [
"Simon Sapin <simon.sapin@exyr.org>"
];
dependencies = [
{
name = "cssparser-macros";
packageId = "cssparser-macros";
}
{
name = "dtoa-short";
packageId = "dtoa-short";
}
{
name = "itoa";
packageId = "itoa";
}
{
name = "phf";
packageId = "phf";
features = [ "macros" ];
}
{
name = "smallvec";
packageId = "smallvec";
}
];
features = {
"serde" = [ "dep:serde" ];
};
};
"cssparser-macros" = rec {
crateName = "cssparser-macros";
version = "0.6.1";
edition = "2018";
sha256 = "0cfkzj60avrnskdmaf7f8zw6pp3di4ylplk455zrzaf19ax8id8k";
procMacro = true;
libName = "cssparser_macros";
libPath = "lib.rs";
authors = [
"Simon Sapin <simon.sapin@exyr.org>"
];
dependencies = [
{
name = "quote";
packageId = "quote";
}
{
name = "syn";
packageId = "syn 2.0.100";
features = [ "full" "extra-traits" ];
}
];
};
"darling" = rec { "darling" = rec {
crateName = "darling"; crateName = "darling";
version = "0.20.11"; version = "0.20.11";
@ -1731,6 +1803,49 @@ rec {
}; };
resolvedDefaultFeatures = [ "alloc" "powerfmt" "serde" "std" ]; resolvedDefaultFeatures = [ "alloc" "powerfmt" "serde" "std" ];
}; };
"derive_more" = rec {
crateName = "derive_more";
version = "0.99.19";
edition = "2018";
sha256 = "17y6g78dg31fsv7z4p455bzxs670spg476ww2ibg3mj3vww9m8ix";
procMacro = true;
authors = [
"Jelte Fennema <github-tech@jeltef.nl>"
];
dependencies = [
{
name = "proc-macro2";
packageId = "proc-macro2";
}
{
name = "quote";
packageId = "quote";
}
{
name = "syn";
packageId = "syn 2.0.100";
}
];
features = {
"convert_case" = [ "dep:convert_case" ];
"default" = [ "add_assign" "add" "as_mut" "as_ref" "constructor" "deref" "deref_mut" "display" "error" "from" "from_str" "index" "index_mut" "into" "into_iterator" "iterator" "mul_assign" "mul" "not" "sum" "try_into" "is_variant" "unwrap" ];
"display" = [ "syn/extra-traits" ];
"error" = [ "syn/extra-traits" ];
"from" = [ "syn/extra-traits" ];
"generate-parsing-rs" = [ "peg" ];
"into" = [ "syn/extra-traits" ];
"is_variant" = [ "convert_case" ];
"mul" = [ "syn/extra-traits" ];
"mul_assign" = [ "syn/extra-traits" ];
"not" = [ "syn/extra-traits" ];
"peg" = [ "dep:peg" ];
"rustc_version" = [ "dep:rustc_version" ];
"testing-helpers" = [ "rustc_version" ];
"try_into" = [ "syn/extra-traits" ];
"unwrap" = [ "convert_case" "rustc_version" ];
};
resolvedDefaultFeatures = [ "add" "add_assign" ];
};
"digest" = rec { "digest" = rec {
crateName = "digest"; crateName = "digest";
version = "0.10.7"; version = "0.10.7";
@ -1810,6 +1925,49 @@ rec {
features = { features = {
}; };
}; };
"dtoa" = rec {
crateName = "dtoa";
version = "1.0.10";
edition = "2018";
sha256 = "016gid01rarcdv57h049d7nr9daxc2hc2gqzx0mji57krywd7bfn";
authors = [
"David Tolnay <dtolnay@gmail.com>"
];
features = {
"no-panic" = [ "dep:no-panic" ];
};
};
"dtoa-short" = rec {
crateName = "dtoa-short";
version = "0.3.5";
edition = "2015";
sha256 = "11rwnkgql5jilsmwxpx6hjzkgyrbdmx1d71s0jyrjqm5nski25fd";
libName = "dtoa_short";
authors = [
"Xidorn Quan <me@upsuper.org>"
];
dependencies = [
{
name = "dtoa";
packageId = "dtoa";
}
];
};
"ego-tree" = rec {
crateName = "ego-tree";
version = "0.10.0";
edition = "2021";
sha256 = "1n2csy99chk5v5vzjl0ff79vxpxhl76xmcb3aj6brrzzipmjz5xj";
libName = "ego_tree";
authors = [
"June McEnroe <june@causal.agency>"
"Carlo Federico Vescovo <vescovocarlofederico@gmail.com>"
];
features = {
"serde" = [ "dep:serde" ];
};
};
"either" = rec { "either" = rec {
crateName = "either"; crateName = "either";
version = "1.15.0"; version = "1.15.0";
@ -2155,6 +2313,26 @@ rec {
}; };
resolvedDefaultFeatures = [ "alloc" "default" "std" ]; resolvedDefaultFeatures = [ "alloc" "default" "std" ];
}; };
"futf" = rec {
crateName = "futf";
version = "0.1.5";
edition = "2015";
sha256 = "0hvqk2r7v4fnc34hvc3vkri89gn52d5m9ihygmwn75l1hhp0whnz";
authors = [
"Keegan McAllister <kmcallister@mozilla.com>"
];
dependencies = [
{
name = "mac";
packageId = "mac";
}
{
name = "new_debug_unreachable";
packageId = "new_debug_unreachable";
}
];
};
"futures" = rec { "futures" = rec {
crateName = "futures"; crateName = "futures";
version = "0.3.31"; version = "0.3.31";
@ -2433,6 +2611,23 @@ rec {
}; };
resolvedDefaultFeatures = [ "alloc" "async-await" "async-await-macro" "channel" "default" "futures-channel" "futures-io" "futures-macro" "futures-sink" "io" "memchr" "sink" "slab" "std" ]; resolvedDefaultFeatures = [ "alloc" "async-await" "async-await-macro" "channel" "default" "futures-channel" "futures-io" "futures-macro" "futures-sink" "io" "memchr" "sink" "slab" "std" ];
}; };
"fxhash" = rec {
crateName = "fxhash";
version = "0.2.1";
edition = "2015";
sha256 = "037mb9ichariqi45xm6mz0b11pa92gj38ba0409z3iz239sns6y3";
libPath = "lib.rs";
authors = [
"cbreeden <github@u.breeden.cc>"
];
dependencies = [
{
name = "byteorder";
packageId = "byteorder";
}
];
};
"gcp_auth" = rec { "gcp_auth" = rec {
crateName = "gcp_auth"; crateName = "gcp_auth";
version = "0.12.3"; version = "0.12.3";
@ -2565,6 +2760,26 @@ rec {
}; };
resolvedDefaultFeatures = [ "more_lengths" ]; resolvedDefaultFeatures = [ "more_lengths" ];
}; };
"getopts" = rec {
crateName = "getopts";
version = "0.2.21";
edition = "2015";
sha256 = "1mgb3qvivi26gs6ihqqhh8iyhp3vgxri6vwyrwg28w0xqzavznql";
authors = [
"The Rust Project Developers"
];
dependencies = [
{
name = "unicode-width";
packageId = "unicode-width 0.1.14";
}
];
features = {
"core" = [ "dep:core" ];
"rustc-dep-of-std" = [ "unicode-width/rustc-dep-of-std" "std" "core" ];
"std" = [ "dep:std" ];
};
};
"getrandom 0.2.15" = rec { "getrandom 0.2.15" = rec {
crateName = "getrandom"; crateName = "getrandom";
version = "0.2.15"; version = "0.2.15";
@ -2938,6 +3153,35 @@ rec {
]; ];
}; };
"html5ever" = rec {
crateName = "html5ever";
version = "0.29.1";
edition = "2021";
sha256 = "07518h5gbw0c6x7w5br76bgxvgphs6zlrb4q7ii7bg1ww7510x1v";
authors = [
"The html5ever Project Developers"
];
dependencies = [
{
name = "log";
packageId = "log";
}
{
name = "mac";
packageId = "mac";
}
{
name = "markup5ever";
packageId = "markup5ever";
}
{
name = "match_token";
packageId = "match_token";
}
];
features = {
};
};
"http" = rec { "http" = rec {
crateName = "http"; crateName = "http";
version = "1.3.1"; version = "1.3.1";
@ -4471,6 +4715,78 @@ rec {
}; };
resolvedDefaultFeatures = [ "std" ]; resolvedDefaultFeatures = [ "std" ];
}; };
"mac" = rec {
crateName = "mac";
version = "0.1.1";
edition = "2015";
sha256 = "194vc7vrshqff72rl56f9xgb0cazyl4jda7qsv31m5l6xx7hq7n4";
authors = [
"Jonathan Reem <jonathan.reem@gmail.com>"
];
};
"markup5ever" = rec {
crateName = "markup5ever";
version = "0.14.1";
edition = "2021";
sha256 = "063sdq7hwxn2al9ygify8dd96mj57n9c4lig007lr1p128yj39y7";
libPath = "lib.rs";
authors = [
"The html5ever Project Developers"
];
dependencies = [
{
name = "log";
packageId = "log";
}
{
name = "phf";
packageId = "phf";
}
{
name = "string_cache";
packageId = "string_cache";
}
{
name = "tendril";
packageId = "tendril";
}
];
buildDependencies = [
{
name = "phf_codegen";
packageId = "phf_codegen";
}
{
name = "string_cache_codegen";
packageId = "string_cache_codegen";
}
];
};
"match_token" = rec {
crateName = "match_token";
version = "0.1.0";
edition = "2021";
sha256 = "0sx3212vkjqfblfhr556ayabbjflbigjf5j591j9kgs4infniac8";
procMacro = true;
dependencies = [
{
name = "proc-macro2";
packageId = "proc-macro2";
}
{
name = "quote";
packageId = "quote";
}
{
name = "syn";
packageId = "syn 2.0.100";
features = [ "full" ];
}
];
};
"matchers" = rec { "matchers" = rec {
crateName = "matchers"; crateName = "matchers";
version = "0.1.0"; version = "0.1.0";
@ -4717,6 +5033,18 @@ rec {
"vendored" = [ "openssl/vendored" ]; "vendored" = [ "openssl/vendored" ];
}; };
}; };
"new_debug_unreachable" = rec {
crateName = "new_debug_unreachable";
version = "1.0.6";
edition = "2021";
sha256 = "11phpf1mjxq6khk91yzcbd3ympm78m3ivl7xg6lg2c0lf66fy3k5";
libName = "debug_unreachable";
authors = [
"Matt Brubeck <mbrubeck@limpet.net>"
"Jonathan Reem <jonathan.reem@gmail.com>"
];
};
"nu-ansi-term" = rec { "nu-ansi-term" = rec {
crateName = "nu-ansi-term"; crateName = "nu-ansi-term";
version = "0.46.0"; version = "0.46.0";
@ -5280,6 +5608,142 @@ rec {
"unstable" = [ "generate" ]; "unstable" = [ "generate" ];
}; };
}; };
"phf" = rec {
crateName = "phf";
version = "0.11.3";
edition = "2021";
sha256 = "0y6hxp1d48rx2434wgi5g8j1pr8s5jja29ha2b65435fh057imhz";
authors = [
"Steven Fackler <sfackler@gmail.com>"
];
dependencies = [
{
name = "phf_macros";
packageId = "phf_macros";
optional = true;
}
{
name = "phf_shared";
packageId = "phf_shared";
usesDefaultFeatures = false;
}
];
features = {
"default" = [ "std" ];
"macros" = [ "phf_macros" ];
"phf_macros" = [ "dep:phf_macros" ];
"serde" = [ "dep:serde" ];
"std" = [ "phf_shared/std" ];
"uncased" = [ "phf_shared/uncased" ];
"unicase" = [ "phf_macros?/unicase" "phf_shared/unicase" ];
};
resolvedDefaultFeatures = [ "default" "macros" "phf_macros" "std" ];
};
"phf_codegen" = rec {
crateName = "phf_codegen";
version = "0.11.3";
edition = "2021";
sha256 = "0si1n6zr93kzjs3wah04ikw8z6npsr39jw4dam8yi9czg2609y5f";
authors = [
"Steven Fackler <sfackler@gmail.com>"
];
dependencies = [
{
name = "phf_generator";
packageId = "phf_generator";
}
{
name = "phf_shared";
packageId = "phf_shared";
}
];
};
"phf_generator" = rec {
crateName = "phf_generator";
version = "0.11.3";
edition = "2021";
crateBin = [];
sha256 = "0gc4np7s91ynrgw73s2i7iakhb4lzdv1gcyx7yhlc0n214a2701w";
authors = [
"Steven Fackler <sfackler@gmail.com>"
];
dependencies = [
{
name = "phf_shared";
packageId = "phf_shared";
usesDefaultFeatures = false;
}
{
name = "rand";
packageId = "rand 0.8.5";
usesDefaultFeatures = false;
features = [ "small_rng" ];
}
];
features = {
"criterion" = [ "dep:criterion" ];
};
};
"phf_macros" = rec {
crateName = "phf_macros";
version = "0.11.3";
edition = "2021";
sha256 = "05kjfbyb439344rhmlzzw0f9bwk9fp95mmw56zs7yfn1552c0jpq";
procMacro = true;
authors = [
"Steven Fackler <sfackler@gmail.com>"
];
dependencies = [
{
name = "phf_generator";
packageId = "phf_generator";
}
{
name = "phf_shared";
packageId = "phf_shared";
usesDefaultFeatures = false;
}
{
name = "proc-macro2";
packageId = "proc-macro2";
}
{
name = "quote";
packageId = "quote";
}
{
name = "syn";
packageId = "syn 2.0.100";
features = [ "full" ];
}
];
features = {
"unicase" = [ "unicase_" "phf_shared/unicase" ];
"unicase_" = [ "dep:unicase_" ];
};
};
"phf_shared" = rec {
crateName = "phf_shared";
version = "0.11.3";
edition = "2021";
sha256 = "1rallyvh28jqd9i916gk5gk2igdmzlgvv5q0l3xbf3m6y8pbrsk7";
authors = [
"Steven Fackler <sfackler@gmail.com>"
];
dependencies = [
{
name = "siphasher";
packageId = "siphasher";
}
];
features = {
"default" = [ "std" ];
"uncased" = [ "dep:uncased" ];
"unicase" = [ "dep:unicase" ];
};
resolvedDefaultFeatures = [ "default" "std" ];
};
"pin-project" = rec { "pin-project" = rec {
crateName = "pin-project"; crateName = "pin-project";
version = "1.1.10"; version = "1.1.10";
@ -5397,6 +5861,17 @@ rec {
}; };
resolvedDefaultFeatures = [ "simd" "std" ]; resolvedDefaultFeatures = [ "simd" "std" ];
}; };
"precomputed-hash" = rec {
crateName = "precomputed-hash";
version = "0.1.1";
edition = "2015";
sha256 = "075k9bfy39jhs53cb2fpb9klfakx2glxnf28zdw08ws6lgpq6lwj";
libName = "precomputed_hash";
authors = [
"Emilio Cobos Álvarez <emilio@crisal.io>"
];
};
"prettyplease" = rec { "prettyplease" = rec {
crateName = "prettyplease"; crateName = "prettyplease";
version = "0.2.32"; version = "0.2.32";
@ -7485,6 +7960,56 @@ rec {
"default" = [ "use_std" ]; "default" = [ "use_std" ];
}; };
}; };
"scraper" = rec {
crateName = "scraper";
version = "0.23.1";
edition = "2021";
crateBin = [];
sha256 = "1qpz5py0a7y9mg2w4v1lidphz3arhw8dl4jcvf47aml8v3cnazjj";
authors = [
"June McEnroe <june@causal.agency>"
];
dependencies = [
{
name = "cssparser";
packageId = "cssparser";
}
{
name = "ego-tree";
packageId = "ego-tree";
}
{
name = "getopts";
packageId = "getopts";
optional = true;
}
{
name = "html5ever";
packageId = "html5ever";
}
{
name = "precomputed-hash";
packageId = "precomputed-hash";
}
{
name = "selectors";
packageId = "selectors";
}
{
name = "tendril";
packageId = "tendril";
}
];
features = {
"default" = [ "main" "errors" ];
"deterministic" = [ "indexmap" ];
"getopts" = [ "dep:getopts" ];
"indexmap" = [ "dep:indexmap" ];
"main" = [ "getopts" ];
"serde" = [ "dep:serde" ];
};
resolvedDefaultFeatures = [ "default" "errors" "getopts" "main" ];
};
"security-framework 2.11.1" = rec { "security-framework 2.11.1" = rec {
crateName = "security-framework"; crateName = "security-framework";
version = "2.11.1"; version = "2.11.1";
@ -7607,6 +8132,69 @@ rec {
}; };
resolvedDefaultFeatures = [ "OSX_10_10" "OSX_10_11" "OSX_10_12" "OSX_10_9" "default" ]; resolvedDefaultFeatures = [ "OSX_10_10" "OSX_10_11" "OSX_10_12" "OSX_10_9" "default" ];
}; };
"selectors" = rec {
crateName = "selectors";
version = "0.26.0";
edition = "2021";
sha256 = "1s3zv30rqgdvil7mnfr4xq5nb9m8yp0sai42l28y565mkd68lmpx";
libPath = "lib.rs";
authors = [
"The Servo Project Developers"
];
dependencies = [
{
name = "bitflags";
packageId = "bitflags 2.9.0";
}
{
name = "cssparser";
packageId = "cssparser";
}
{
name = "derive_more";
packageId = "derive_more";
usesDefaultFeatures = false;
features = [ "add" "add_assign" ];
}
{
name = "fxhash";
packageId = "fxhash";
}
{
name = "log";
packageId = "log";
}
{
name = "new_debug_unreachable";
packageId = "new_debug_unreachable";
}
{
name = "phf";
packageId = "phf";
}
{
name = "precomputed-hash";
packageId = "precomputed-hash";
}
{
name = "servo_arc";
packageId = "servo_arc";
}
{
name = "smallvec";
packageId = "smallvec";
}
];
buildDependencies = [
{
name = "phf_codegen";
packageId = "phf_codegen";
}
];
features = {
"to_shmem" = [ "dep:to_shmem" "dep:to_shmem_derive" ];
};
};
"serde" = rec { "serde" = rec {
crateName = "serde"; crateName = "serde";
version = "1.0.219"; version = "1.0.219";
@ -7951,6 +8539,26 @@ rec {
features = { features = {
}; };
}; };
"servo_arc" = rec {
crateName = "servo_arc";
version = "0.4.0";
edition = "2021";
sha256 = "06ljch4isnnbv1xpwhjajz4a4mpc7ki47ys9n9yn98kqjhjc8rdf";
libPath = "lib.rs";
authors = [
"The Servo Project Developers"
];
dependencies = [
{
name = "stable_deref_trait";
packageId = "stable_deref_trait";
}
];
features = {
"serde" = [ "dep:serde" ];
"servo" = [ "serde" "track_alloc_size" ];
};
};
"sharded-slab" = rec { "sharded-slab" = rec {
crateName = "sharded-slab"; crateName = "sharded-slab";
version = "0.1.7"; version = "0.1.7";
@ -8021,6 +8629,14 @@ rec {
name = "rusqlite"; name = "rusqlite";
packageId = "rusqlite"; packageId = "rusqlite";
} }
{
name = "scraper";
packageId = "scraper";
}
{
name = "serde_json";
packageId = "serde_json";
}
{ {
name = "snix-castore"; name = "snix-castore";
packageId = "snix-castore"; packageId = "snix-castore";
@ -8058,6 +8674,23 @@ rec {
]; ];
}; };
"siphasher" = rec {
crateName = "siphasher";
version = "1.0.1";
edition = "2018";
sha256 = "17f35782ma3fn6sh21c027kjmd227xyrx06ffi8gw4xzv9yry6an";
authors = [
"Frank Denis <github@pureftpd.org>"
];
features = {
"default" = [ "std" ];
"serde" = [ "dep:serde" ];
"serde_json" = [ "dep:serde_json" ];
"serde_no_std" = [ "serde/alloc" ];
"serde_std" = [ "std" "serde/std" ];
};
resolvedDefaultFeatures = [ "default" "std" ];
};
"slab" = rec { "slab" = rec {
crateName = "slab"; crateName = "slab";
version = "0.4.9"; version = "0.4.9";
@ -8477,7 +9110,75 @@ rec {
"default" = [ "std" ]; "default" = [ "std" ];
"std" = [ "alloc" ]; "std" = [ "alloc" ];
}; };
resolvedDefaultFeatures = [ "alloc" ]; resolvedDefaultFeatures = [ "alloc" "default" "std" ];
};
"string_cache" = rec {
crateName = "string_cache";
version = "0.8.9";
edition = "2018";
sha256 = "03z7km2kzlwiv2r2qifq5riv4g8phazwng9wnvs3py3lzainnxxz";
authors = [
"The Servo Project Developers"
];
dependencies = [
{
name = "new_debug_unreachable";
packageId = "new_debug_unreachable";
}
{
name = "parking_lot";
packageId = "parking_lot";
}
{
name = "phf_shared";
packageId = "phf_shared";
}
{
name = "precomputed-hash";
packageId = "precomputed-hash";
}
{
name = "serde";
packageId = "serde";
optional = true;
}
];
features = {
"default" = [ "serde_support" ];
"malloc_size_of" = [ "dep:malloc_size_of" ];
"serde" = [ "dep:serde" ];
"serde_support" = [ "serde" ];
};
resolvedDefaultFeatures = [ "default" "serde" "serde_support" ];
};
"string_cache_codegen" = rec {
crateName = "string_cache_codegen";
version = "0.5.4";
edition = "2018";
sha256 = "181ir4d6y053s1kka2idpjx5g9d9jgll6fy517jhzzpi2n3r44f7";
libPath = "lib.rs";
authors = [
"The Servo Project Developers"
];
dependencies = [
{
name = "phf_generator";
packageId = "phf_generator";
}
{
name = "phf_shared";
packageId = "phf_shared";
}
{
name = "proc-macro2";
packageId = "proc-macro2";
}
{
name = "quote";
packageId = "quote";
}
];
}; };
"strsim" = rec { "strsim" = rec {
crateName = "strsim"; crateName = "strsim";
@ -8718,6 +9419,35 @@ rec {
}; };
resolvedDefaultFeatures = [ "default" "getrandom" ]; resolvedDefaultFeatures = [ "default" "getrandom" ];
}; };
"tendril" = rec {
crateName = "tendril";
version = "0.4.3";
edition = "2015";
sha256 = "1c3vip59sqwxn148i714nmkrvjzbk7105vj0h92s6r64bw614jnj";
authors = [
"Keegan McAllister <mcallister.keegan@gmail.com>"
"Simon Sapin <simon.sapin@exyr.org>"
"Chris Morgan <me@chrismorgan.info>"
];
dependencies = [
{
name = "futf";
packageId = "futf";
}
{
name = "mac";
packageId = "mac";
}
{
name = "utf-8";
packageId = "utf-8";
}
];
features = {
"encoding" = [ "dep:encoding" ];
"encoding_rs" = [ "dep:encoding_rs" ];
};
};
"thiserror 1.0.69" = rec { "thiserror 1.0.69" = rec {
crateName = "thiserror"; crateName = "thiserror";
version = "1.0.69"; version = "1.0.69";
@ -10325,6 +11055,17 @@ rec {
}; };
resolvedDefaultFeatures = [ "default" "std" ]; resolvedDefaultFeatures = [ "default" "std" ];
}; };
"utf-8" = rec {
crateName = "utf-8";
version = "0.7.6";
edition = "2015";
sha256 = "1a9ns3fvgird0snjkd3wbdhwd3zdpc2h5gpyybrfr6ra5pkqxk09";
libName = "utf8";
authors = [
"Simon Sapin <simon.sapin@exyr.org>"
];
};
"utf16_iter" = rec { "utf16_iter" = rec {
crateName = "utf16_iter"; crateName = "utf16_iter";
version = "1.0.5"; version = "1.0.5";

View file

@ -9,7 +9,11 @@ clap = "4.5.35"
futures = "0.3.31" futures = "0.3.31"
reqwest = "0.12.15" reqwest = "0.12.15"
rusqlite = "0.34.0" rusqlite = "0.34.0"
scraper = "0.23.1"
serde = "1.0.219"
serde_json = "1.0.140"
snix-castore = { version = "0.1.0", git = "https://git.snix.dev/snix/snix.git" } snix-castore = { version = "0.1.0", git = "https://git.snix.dev/snix/snix.git" }
tokio = "1.44.2" tokio = "1.44.2"
tokio-stream = "0.1.17"
tokio-util = "0.7.14" tokio-util = "0.7.14"
url = "2.5.4" url = "2.5.4"

View file

@ -1,7 +1,8 @@
sidx [sidx](https://forge.someonex.net/else/sidx)
=== ===
Work in Progress.
Indexing archives and build outputs. Indexing archives and build outputs.
@ -17,4 +18,8 @@ Roadmap
Approach Approach
--- ---
Vapourware and means to an end. Vapourware and means to an end:
[this project](https://forge.someonex.net/else/sidx) was originally motivated by the needs of maintaining
`cudaPackages` in Nixpkgs.
Specifically, it attempts to answer the question of "what is there to be maintained",
improve [observability and debug-ability of the package set (cf. demo)](https://cuda-index.someonex.net/sidx/UriReference).

View file

@ -47,5 +47,118 @@ lib.makeScope pkgs.newScope (
} }
); );
sidx = self.sidx-crate2nix.rootCrate.build; sidx = self.sidx-crate2nix.rootCrate.build;
datasette-wrapped = self.callPackage (
{
datasette,
datasette-assets,
makeWrapper,
runCommand,
}:
runCommand "datasettew"
{
nativeBuildInputs = [ makeWrapper ];
preferLocalBuild = true;
allowSubstitutes = false;
}
''
mkdir -p "$out/bin"
makeWrapper ${lib.getExe datasette} "$out/bin/datasettew" \
--append-flags --metadata=${datasette-assets}/metadata.json \
--append-flags --static=static:${datasette-assets}/static
''
) { };
datasette-assets = self.callPackage (
{
runCommand,
datasette-metadata,
datasette-settings,
}:
runCommand "datasette-assets"
{
preferLocalBuild = true;
allowSubstitutes = false;
}
''
mkdir "$out"
cp --no-preserve=mode -r ${./static} "$out"/static
cp ${datasette-metadata} "$out"/metadata.json
cp ${datasette-settings} "$out"/settings.json
''
) { };
datasette-settings = self.callPackage (
{ formats }:
(formats.json { }).generate "datasette-settings.json" {
sql_time_limit_ms = 8000;
}
) { };
datasette-metadata = self.callPackage (
{ formats }:
(formats.json { }).generate "datasette-metadata.json" {
title = "CUDA INDEX";
description_html = ''
<p>Visualizing the contents of <a href="https://nixos.org/manual/nixpkgs/unstable/#cuda">Nixpkgs' cudaPackages</a>.
Generated via an <a href="https://forge.someonex.net/else/sidx">ad-hoc indexing tool</a>.
</p>
'';
"extra_css_urls" = [
"/static/some.css"
];
"databases" = {
"sidx" = {
"tables" = {
"Hash" = {
"label_column" = "hash";
};
"CudaArtifact" = {
facets = [
"pname"
"platform"
];
};
};
queries.cuda_conflicts = {
title = "CudaArtifact Conflicts";
description_html = ''
<code>CudaArtifact</code>s (identified by <code>sha256</code>)
claiming the same <code>(pname, version, platform)</code> triple
'';
sql = ''
SELECT
COUNT(DISTINCT sha256) AS conflicts,
pname.str AS pname,
ver.str AS ver,
plat.str AS plat,
GROUP_CONCAT(name.str, char(10)) AS name,
GROUP_CONCAT(tag.str, char(10)) AS tag,
GROUP_CONCAT(h.hash, char(10)) AS sha256
FROM
(
CudaArtifact AS cc,
Str AS name,
Str AS pname,
Str as ver,
Str as plat,
Hash as h
ON cc.name=name.id
AND cc.pname=pname.id
AND cc.version = ver.id
AND cc.platform = plat.id
AND cc.sha256 = h.id
)
LEFT JOIN Str AS tag
ON
cc.compat_tag=tag.id
GROUP BY
cc.pname, cc.version, cc.platform
HAVING
conflicts >= CAST(:min_conflicts AS INTEGER)
ORDER BY conflicts DESC
'';
};
};
};
}
) { };
} }
) )

View file

@ -2,17 +2,20 @@
npins ? import ./npins, npins ? import ./npins,
nixpkgs ? npins.nixpkgs, nixpkgs ? npins.nixpkgs,
pkgs ? import nixpkgs { }, pkgs ? import nixpkgs { },
self ? import ./. { inherit nixpkgs pkgs npins; },
lib ? pkgs.lib, lib ? pkgs.lib,
mkShell ? pkgs.mkShell, mkShell ? pkgs.mkShell,
sqlite ? pkgs.sqlite, sqlite ? pkgs.sqlite,
openssl ? pkgs.openssl, openssl ? pkgs.openssl,
rust-analyzer ? pkgs.rust-analyzer, rust-analyzer ? pkgs.rust-analyzer,
rustc ? pkgs.rustc, rustc ? pkgs.rustc,
rustfmt ? pkgs.rustfmt,
cargo ? pkgs.cargo, cargo ? pkgs.cargo,
pkg-config ? pkgs.pkg-config, pkg-config ? pkgs.pkg-config,
crate2nix ? pkgs.crate2nix, crate2nix ? pkgs.crate2nix,
protobuf ? pkgs.protobuf, protobuf ? pkgs.protobuf,
datasette ? pkgs.datasette, datasette-wrapped ? self.datasette-wrapped,
datasette-assets ? self.datasette-assets,
... ...
}: }:
mkShell { mkShell {
@ -22,13 +25,19 @@ mkShell {
cargo cargo
crate2nix crate2nix
rustc rustc
rustfmt
rust-analyzer rust-analyzer
pkg-config pkg-config
protobuf protobuf
datasette datasette-wrapped
]; ];
buildInputs = [ buildInputs = [
openssl openssl
sqlite sqlite
]; ];
DATASETTE_ASSETS = datasette-assets; # uploaded to cuda-index.someonex.net in bulk...
shellHook = ''
export DATABASE_PATH="$HOME/.local/share/sidx/sidx.db"
unset out outputs phases
'';
} }

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,17 @@
INSERT INTO
CudaArtifact(manifest, sha256, md5, name, pname, license_name, license_path, version, platform, compat_tag, relative_path, n_bytes)
VALUES (
(SELECT id FROM Hash WHERE hash=:manifest LIMIT 1),
(SELECT id FROM Hash WHERE hash=:sha256 LIMIT 1),
(SELECT id FROM Hash WHERE hash=:md5 LIMIT 1),
(SELECT id FROM Str WHERE str=:name LIMIT 1),
(SELECT id FROM Str WHERE str=:pname LIMIT 1),
(SELECT id FROM Str WHERE str=:license_name LIMIT 1),
(SELECT id FROM Str WHERE str=:license_path LIMIT 1),
(SELECT id FROM Str WHERE str=:version LIMIT 1),
(SELECT id FROM Str WHERE str=:platform LIMIT 1),
(SELECT id FROM Str WHERE str=:compat_tag LIMIT 1),
(SELECT id FROM Str WHERE str=:relative_path LIMIT 1),
:n_bytes
)
ON CONFLICT DO NOTHING

View file

@ -0,0 +1,9 @@
INSERT INTO
CudaManifest(id, release_date, release_label, release_product)
VALUES (
(SELECT id FROM Hash WHERE hash=:hash LIMIT 1),
(SELECT id FROM Str WHERE str=:release_date LIMIT 1),
(SELECT id FROM Str WHERE str=:release_label LIMIT 1),
(SELECT id FROM Str WHERE str=:release_product LIMIT 1)
)
ON CONFLICT DO NOTHING

View file

@ -1,21 +1,8 @@
INSERT INTO sidx_uri_sample(uri_id, blake3_id) INSERT INTO SidxUriSample(uri, hash, http_code, content_type)
VALUES ( VALUES (
( ( SELECT id FROM Str WHERE str = :uri LIMIT 1),
SELECT ( SELECT id FROM Hash WHERE hash = :hash LIMIT 1 ),
id :http_code,
FROM ( SELECT id FROM Str WHERE str = :content_type LIMIT 1)
sidx_uri
WHERE
uri = ?
LIMIT 1
),
(
SELECT
id
FROM
sidx_blake3
WHERE
blake3 = ?
)
) )
RETURNING id, epoch; RETURNING id, epoch;

5
src/q/add-str.sql Normal file
View file

@ -0,0 +1,5 @@
INSERT INTO Str(str)
VALUES
(?)
ON CONFLICT DO NOTHING;

7
src/q/add-uri-ref.sql Normal file
View file

@ -0,0 +1,7 @@
INSERT INTO UriReference(content, target, why)
VALUES (
(SELECT id FROM Hash WHERE hash=:source LIMIT 1),
(SELECT id FROM Str WHERE str=:target LIMIT 1),
(SELECT id FROM Str WHERE str=:why LIMIT 1)
)
ON CONFLICT DO UPDATE SET why=excluded.why;

49
src/q/cuda-init.sql Normal file
View file

@ -0,0 +1,49 @@
CREATE TABLE IF NOT EXISTS CudaManifest(
id INTEGER, /* Blake3/ca-node of the JSON */
release_date INTEGER, /* E.g. "2025-03-06" */
release_label INTEGER, /* E.g. "12.8.1" */
release_product INTEGER, /* E.g. "cuda" */
PRIMARY KEY(id),
FOREIGN KEY(id) REFERENCES Hash(id),
FOREIGN KEY(release_date) REFERENCES Str(id),
FOREIGN KEY(release_label) REFERENCES Str(id),
FOREIGN KEY(release_product) REFERENCES Str(id)
) STRICT;
CREATE TABLE IF NOT EXISTS CudaArtifact(
manifest INTEGER NOT NULL,
name INTEGER, /* E.g. "cuda_nvcc" */
pname INTEGER, /* E.g. "CUDA NVCC" */
license_name INTEGER, /* E.g. "CUDA Toolkit" */
license_path INTEGER, /* E.g. "cuda_cccl/LICENSE.txt" */
version INTEGER NOT NULL, /* E.g. "12.8.90" */
/* Consider making external */
compat_tag INTEGER, /* E.g. "cuda12" in cudnn */
sha256 INTEGER,
md5 INTEGER,
platform INTEGER, /* E.g. "linux-x86_64" */
/* E.g. "cuda_cccl/linux-x86_64/cuda_cccl-linux-x86_64-12.8.90-archive.tar.xz" */
relative_path INTEGER,
n_bytes INTEGER, /* May be a string in the JSON */
/* Tempting to have
* PRIMARY KEY(manifest, name, platform),
* however that's not unique at least because of `compat_tag`,
* which might also be `NULL`.
*/
PRIMARY KEY(sha256, manifest),
FOREIGN KEY(manifest) REFERENCES CudaManifest(id),
FOREIGN KEY(manifest) REFERENCES Hash(id),
FOREIGN KEY(name) REFERENCES Str(id),
FOREIGN KEY(pname) REFERENCES Str(id),
FOREIGN KEY(license_name) REFERENCES Str(id),
FOREIGN KEY(license_path) REFERENCES Str(id),
FOREIGN KEY(version) REFERENCES Str(id),
FOREIGN KEY(compat_tag) REFERENCES Str(id),
FOREIGN KEY(sha256) REFERENCES Hash(id),
FOREIGN KEY(md5) REFERENCES Hash(id),
FOREIGN KEY(platform) REFERENCES Str(id),
FOREIGN KEY(relative_path) REFERENCES Str(id)
) STRICT;
CREATE UNIQUE INDEX IF NOT EXISTS CudaArtifactIdx
ON CudaArtifact(pname, platform, version, compat_tag, name, manifest);

View file

@ -0,0 +1,15 @@
SELECT
uri.str AS uri, h.hash, cm.id AS manifest
FROM
SidxUriSample AS s
INNER JOIN Str AS uri
INNER JOIN (Hash AS h LEFT JOIN CudaManifest AS cm ON h.id=cm.id)
ON
s.uri=uri.id
AND s.hash=h.id
WHERE
uri.str LIKE 'https://developer.download.nvidia.com/compute/%.json'
AND (:include_finished OR cm.id IS NULL)
GROUP BY
s.hash
ORDER BY uri.str, s.id DESC;

View file

@ -1,22 +0,0 @@
CREATE TABLE IF NOT EXISTS sidx_uri(
id INTEGER,
uri TEXT UNIQUE,
PRIMARY KEY(id)
);
CREATE TABLE IF NOT EXISTS sidx_blake3(
id INTEGER,
blake3 TEXT UNIQUE, /* snix-castore node */
n_bytes INTEGER NOT NULL,
PRIMARY KEY(id)
);
CREATE TABLE IF NOT EXISTS sidx_uri_sample(
id INTEGER,
uri_id INTEGER NOT NULL,
blake3_id INTEGER,
epoch INTEGER NOT NULL DEFAULT (unixepoch()),
PRIMARY KEY(id),
FOREIGN KEY(uri_id) REFERENCES sidx_uri(id),
FOREIGN KEY(blake3_id) REFERENCES sidx_blake3(id)
);
CREATE INDEX IF NOT EXISTS sidx_uri_blake3_idx
ON sidx_uri_sample(uri_id, blake3_id, epoch);

View file

@ -1,16 +1,19 @@
SELECT SELECT
s.id AS sample_id, s.id AS sample_id,
b.blake3, h.hash,
h.n_bytes,
s.http_code,
s.epoch s.epoch
FROM FROM
sidx_uri_sample AS s, SidxUriSample AS s,
sidx_uri AS u, Str AS u,
sidx_blake3 AS b Hash AS h
ON ON
s.uri_id = u.id s.uri = u.id
AND s.blake3_id = b.id AND s.hash = h.id
WHERE WHERE
u.uri = ? u.str = ?
AND s.hash IS NOT NULL
ORDER BY ORDER BY
s.epoch DESC s.epoch DESC
LIMIT 1; LIMIT 1;

37
src/q/sidx-init.sql Normal file
View file

@ -0,0 +1,37 @@
CREATE TABLE IF NOT EXISTS Hash(
id INTEGER,
hash TEXT UNIQUE, /* snix-castore node */
n_bytes INTEGER,
PRIMARY KEY(id)
) STRICT; /* Essentially random strings */
CREATE TABLE IF NOT EXISTS Str(
id INTEGER,
str TEXT UNIQUE,
PRIMARY KEY(id)
) STRICT; /* "Naturally occuring" strings */
CREATE TABLE IF NOT EXISTS SidxUriSample(
id INTEGER,
uri INTEGER NOT NULL,
hash INTEGER,
epoch INTEGER NOT NULL DEFAULT (unixepoch()),
http_code INTEGER DEFAULT NULL,
content_type INTEGER DEFAULT NULL,
PRIMARY KEY(id),
FOREIGN KEY(uri) REFERENCES Str(id),
FOREIGN KEY(hash) REFERENCES Hash(id),
FOREIGN KEY(content_type) REFERENCES Str(id)
) STRICT;
CREATE INDEX IF NOT EXISTS SidxUriHashIdx
ON SidxUriSample(uri, hash, epoch);
CREATE TABLE IF NOT EXISTS "UriReference" (
"id" INTEGER,
"content" INTEGER NOT NULL,
"target" INTEGER NOT NULL,
"why" INTEGER,
PRIMARY KEY("id"),
CONSTRAINT "NoDupRefs" UNIQUE("content","target","why"),
FOREIGN KEY("content") REFERENCES "Hash"("id"),
FOREIGN KEY("target") REFERENCES "Str"("id"),
FOREIGN KEY("why") REFERENCES "Str"("id")
) STRICT;

View file

@ -1,4 +1,4 @@
INSERT INTO sidx_blake3(blake3, n_bytes) INSERT INTO Hash(hash, n_bytes)
VALUES VALUES
(?, ?) (?, ?)
ON CONFLICT DO NOTHING; ON CONFLICT DO NOTHING;

View file

@ -1,4 +1,4 @@
INSERT INTO sidx_uri(uri) INSERT INTO Str(str)
VALUES VALUES
(?) (?)
ON CONFLICT DO NOTHING; ON CONFLICT DO NOTHING;

16
src/q/uris-of-hash.sql Normal file
View file

@ -0,0 +1,16 @@
SELECT
uri.str AS uri
FROM
SidxUriSample AS s
INNER JOIN Str AS uri
INNER JOIN Hash AS h
ON
s.uri=uri.id
AND s.hash=h.id
WHERE
h.hash=:hash
ORDER BY
s.epoch DESC
LIMIT
:limit
;

48
static/some.css Normal file
View file

@ -0,0 +1,48 @@
.index {
font-family: "Source Serif Pro", "Linux Libertine", monospace;
}
.db-table > h3 {
font-variant: small-caps;
}
h1, nav {
font-variant: small-caps;
font-family: "Inconsolata", monospace;
}
h2 {
font-variant: small-caps;
}
th {
font-variant: small-caps;
}
header {
background-color: black;
}
.ft {
background-color: black;
}
form input[type="submit"] {
background-color: black;
}
a:link {
color: #404040;
}
a:visited {
color: darkgrey;
}
a:hover {
color: black;
}
.rows-and-columns > tbody > tr:nth-child(even) {
background-color: #F5F5F5;
}