)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"40cc70e755dba0ec1cf39c2f7bfc802e5d0233eb","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"2cf037b2_e0c4c4a2","updated":"2023-11-04 11:46:01.000000000","message":"This should probably also have a default.nix, so CI builds it.","commit_id":"3b527a088cffb85f588b7c9914643821e58d867f"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"27496a28e29547adec7acce14fa328106c59f383","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":2,"id":"7b867f64_3220440d","in_reply_to":"2cf037b2_e0c4c4a2","updated":"2023-11-04 11:46:19.000000000","message":"unresolved","commit_id":"3b527a088cffb85f588b7c9914643821e58d867f"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"badd01cb91bb1f15deb6d15baeae77951e9f3576","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"4c8b3ef4_d104f2b2","in_reply_to":"7b867f64_3220440d","updated":"2023-11-07 13:08:37.000000000","message":"Done","commit_id":"3b527a088cffb85f588b7c9914643821e58d867f"},{"author":{"_account_id":1000001,"name":"tazjin","email":"tazjin@tvl.su","username":"tazjin"},"change_message_id":"9c29d67eaca6f6f301d5daf188bcb5b24ec69049","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"cf42b15a_d05ba053","updated":"2024-10-12 22:39:48.000000000","message":"This should\u0027ve probably had a commit message ... I\u0027m moving this out of //tvix.","commit_id":"bdda10a2f54974053b1d78b801469f076e90a6da"}],"tvix/tools/turbofetch/.gitignore":[{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"40cc70e755dba0ec1cf39c2f7bfc802e5d0233eb","unresolved":true,"context_lines":[{"line_number":1,"context_line":"/target"}],"source_content_type":"application/octet-stream","patch_set":2,"id":"d792a2ef_9b42b6c2","line":1,"updated":"2023-11-04 11:46:01.000000000","message":"isn\u0027t there already a gitignore for this further up?","commit_id":"3b527a088cffb85f588b7c9914643821e58d867f"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"badd01cb91bb1f15deb6d15baeae77951e9f3576","unresolved":false,"context_lines":[{"line_number":1,"context_line":"/target"}],"source_content_type":"application/octet-stream","patch_set":2,"id":"b5d497a4_749b4e85","line":1,"in_reply_to":"d792a2ef_9b42b6c2","updated":"2023-11-07 13:08:37.000000000","message":"Done","commit_id":"3b527a088cffb85f588b7c9914643821e58d867f"}],"tvix/tools/turbofetch/Cargo.lock":[{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"40cc70e755dba0ec1cf39c2f7bfc802e5d0233eb","unresolved":true,"context_lines":[{"line_number":1,"context_line":"# This file is automatically @generated by Cargo."},{"line_number":2,"context_line":"# It is not intended for manual editing."},{"line_number":3,"context_line":"version \u003d 3"},{"line_number":4,"context_line":""}],"source_content_type":"application/octet-stream","patch_set":2,"id":"2eb52a64_9713d142","line":1,"updated":"2023-11-04 11:46:01.000000000","message":"please add this to the tvix/Cargo.toml workspace, and do the crate2nix generate step as documented in the readme there.","commit_id":"3b527a088cffb85f588b7c9914643821e58d867f"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"badd01cb91bb1f15deb6d15baeae77951e9f3576","unresolved":false,"context_lines":[{"line_number":1,"context_line":"# This file is automatically @generated by Cargo."},{"line_number":2,"context_line":"# It is not intended for manual editing."},{"line_number":3,"context_line":"version \u003d 3"},{"line_number":4,"context_line":""}],"source_content_type":"application/octet-stream","patch_set":2,"id":"97193b6b_d738ecf4","line":1,"in_reply_to":"2eb52a64_9713d142","updated":"2023-11-07 13:08:37.000000000","message":"Done","commit_id":"3b527a088cffb85f588b7c9914643821e58d867f"}],"tvix/tools/turbofetch/src/buffer.rs":[{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":1,"context_line":"use magic_buffer::MagicBuffer;"},{"line_number":2,"context_line":"use std::cell::Cell;"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"pub struct Buffer {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"7193d962_b8424ff3","line":1,"updated":"2023-11-08 11:53:42.000000000","message":"Either the module or pub struct Buffer should have a comment describing what\u0027s the purpose of it, and what it adds on top of MagicBuffer.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":1,"context_line":"use magic_buffer::MagicBuffer;"},{"line_number":2,"context_line":"use std::cell::Cell;"},{"line_number":3,"context_line":""},{"line_number":4,"context_line":"pub struct Buffer {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"990b0a21_db59ad8b","line":1,"in_reply_to":"7193d962_b8424ff3","updated":"2023-11-10 15:13:04.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":9,"context_line":"    tail: usize,"},{"line_number":10,"context_line":"}"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"unsafe impl Send for Buffer {}"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"impl Buffer {"},{"line_number":15,"context_line":"    pub fn new(len: usize) -\u003e Buffer {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"68501e75_f8a8562a","line":12,"updated":"2023-11-08 11:53:42.000000000","message":"Why is this Send? Needs a SAFETY comment.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"8cb4d8ef09d909c3dd8a0098512916cb8cffdb1e","unresolved":false,"context_lines":[{"line_number":9,"context_line":"    tail: usize,"},{"line_number":10,"context_line":"}"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"unsafe impl Send for Buffer {}"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"impl Buffer {"},{"line_number":15,"context_line":"    pub fn new(len: usize) -\u003e Buffer {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"89591f7f_5479805f","line":12,"in_reply_to":"68501e75_f8a8562a","updated":"2023-11-09 11:07:21.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"}],"tvix/tools/turbofetch/src/lib.rs":[{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":4,"context_line":"pub use buffer::Buffer;"},{"line_number":5,"context_line":"mod buffer;"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"async fn slurp(buffer: \u0026mut Buffer, sock: \u0026mut (impl AsyncRead + Unpin)) -\u003e io::Result\u003c()\u003e {"},{"line_number":8,"context_line":"    match buffer.space() {"},{"line_number":9,"context_line":"        [] \u003d\u003e Err(io::Error::new(io::ErrorKind::OutOfMemory, \"buffer filled\")),"},{"line_number":10,"context_line":"        buf \u003d\u003e {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"6d70a8ff_e689956a","line":7,"updated":"2023-11-08 11:53:42.000000000","message":"doc-comment","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"8cb4d8ef09d909c3dd8a0098512916cb8cffdb1e","unresolved":false,"context_lines":[{"line_number":4,"context_line":"pub use buffer::Buffer;"},{"line_number":5,"context_line":"mod buffer;"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"async fn slurp(buffer: \u0026mut Buffer, sock: \u0026mut (impl AsyncRead + Unpin)) -\u003e io::Result\u003c()\u003e {"},{"line_number":8,"context_line":"    match buffer.space() {"},{"line_number":9,"context_line":"        [] \u003d\u003e Err(io::Error::new(io::ErrorKind::OutOfMemory, \"buffer filled\")),"},{"line_number":10,"context_line":"        buf \u003d\u003e {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"3b1b58c2_16701cdd","line":7,"in_reply_to":"6d70a8ff_e689956a","updated":"2023-11-09 11:07:21.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":19,"context_line":"    }"},{"line_number":20,"context_line":"}"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"fn get_content_length(headers: \u0026[httparse::Header]) -\u003e io::Result\u003cusize\u003e {"},{"line_number":23,"context_line":"    for header in headers {"},{"line_number":24,"context_line":"        if header.name \u003d\u003d \"Content-Length\" {"},{"line_number":25,"context_line":"            let value \u003d str::from_utf8(header.value).unwrap();"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"f032bf8c_31a12c0b","line":22,"updated":"2023-11-08 11:53:42.000000000","message":"We don\u0027t use this to ingest NAR files (yet), only narinfo files, but it\u0027s probably okay to just make this u64?","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"8cb4d8ef09d909c3dd8a0098512916cb8cffdb1e","unresolved":false,"context_lines":[{"line_number":19,"context_line":"    }"},{"line_number":20,"context_line":"}"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"fn get_content_length(headers: \u0026[httparse::Header]) -\u003e io::Result\u003cusize\u003e {"},{"line_number":23,"context_line":"    for header in headers {"},{"line_number":24,"context_line":"        if header.name \u003d\u003d \"Content-Length\" {"},{"line_number":25,"context_line":"            let value \u003d str::from_utf8(header.value).unwrap();"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"9f1a798f_3f50ec3c","line":22,"in_reply_to":"f032bf8c_31a12c0b","updated":"2023-11-09 11:07:21.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":36,"context_line":"    ))"},{"line_number":37,"context_line":"}"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"pub async fn parse_response\u003c\u0027a\u003e("},{"line_number":40,"context_line":"    sock: \u0026mut (impl AsyncRead + Unpin),"},{"line_number":41,"context_line":"    buffer: \u0026\u0027a mut Buffer,"},{"line_number":42,"context_line":") -\u003e io::Result\u003c\u0026\u0027a [u8]\u003e {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"24d8b8e3_682b6af1","line":39,"updated":"2023-11-08 11:53:42.000000000","message":"This parses (parts of) the response headers, and consumes the response data into the buffer passed in.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"8cb4d8ef09d909c3dd8a0098512916cb8cffdb1e","unresolved":false,"context_lines":[{"line_number":36,"context_line":"    ))"},{"line_number":37,"context_line":"}"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"pub async fn parse_response\u003c\u0027a\u003e("},{"line_number":40,"context_line":"    sock: \u0026mut (impl AsyncRead + Unpin),"},{"line_number":41,"context_line":"    buffer: \u0026\u0027a mut Buffer,"},{"line_number":42,"context_line":") -\u003e io::Result\u003c\u0026\u0027a [u8]\u003e {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"ffd3eb75_fa3fbc77","line":39,"in_reply_to":"24d8b8e3_682b6af1","updated":"2023-11-09 11:07:21.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"}],"tvix/tools/turbofetch/src/main.rs":[{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":1,"context_line":"//! turbofetch retrieves narinfo files from cache.nixos.org as fast as possible."},{"line_number":2,"context_line":"//! It is designed to run many instances in parallel, each writing to a common S3 bucket."},{"line_number":3,"context_line":"//! The list of files to retrieve is read from a file consisting of nixbase32 format Nix store path hashes, without separators."},{"line_number":4,"context_line":"//! Each instance receives a range in this file (specified as indices in the listing file), and writes them to the target bucket after zstd compression."},{"line_number":5,"context_line":"//! Empirically, the resulting compression ratio is about 4.7x, on chunks of 20k narinfo files sorted by creation time."}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"526b3ff4_eb9bb2e4","line":2,"updated":"2023-11-08 11:53:42.000000000","message":"each writing what to a common S3 bucket?","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":1,"context_line":"//! turbofetch retrieves narinfo files from cache.nixos.org as fast as possible."},{"line_number":2,"context_line":"//! It is designed to run many instances in parallel, each writing to a common S3 bucket."},{"line_number":3,"context_line":"//! The list of files to retrieve is read from a file consisting of nixbase32 format Nix store path hashes, without separators."},{"line_number":4,"context_line":"//! Each instance receives a range in this file (specified as indices in the listing file), and writes them to the target bucket after zstd compression."},{"line_number":5,"context_line":"//! Empirically, the resulting compression ratio is about 4.7x, on chunks of 20k narinfo files sorted by creation time."}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"fd44d3eb_5087d5bc","line":2,"in_reply_to":"526b3ff4_eb9bb2e4","updated":"2023-11-10 15:13:04.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":1,"context_line":"//! turbofetch retrieves narinfo files from cache.nixos.org as fast as possible."},{"line_number":2,"context_line":"//! It is designed to run many instances in parallel, each writing to a common S3 bucket."},{"line_number":3,"context_line":"//! The list of files to retrieve is read from a file consisting of nixbase32 format Nix store path hashes, without separators."},{"line_number":4,"context_line":"//! Each instance receives a range in this file (specified as indices in the listing file), and writes them to the target bucket after zstd compression."},{"line_number":5,"context_line":"//! Empirically, the resulting compression ratio is about 4.7x, on chunks of 20k narinfo files sorted by creation time."},{"line_number":6,"context_line":"//! No additional metadata is included, and the files within each group are concatenated in arbitrary order."}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"5e13e293_7c7a7996","line":3,"updated":"2023-11-08 11:53:42.000000000","message":"where\u0027s that list of files to retrieve coming from / parsed?","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":1,"context_line":"//! turbofetch retrieves narinfo files from cache.nixos.org as fast as possible."},{"line_number":2,"context_line":"//! It is designed to run many instances in parallel, each writing to a common S3 bucket."},{"line_number":3,"context_line":"//! The list of files to retrieve is read from a file consisting of nixbase32 format Nix store path hashes, without separators."},{"line_number":4,"context_line":"//! Each instance receives a range in this file (specified as indices in the listing file), and writes them to the target bucket after zstd compression."},{"line_number":5,"context_line":"//! Empirically, the resulting compression ratio is about 4.7x, on chunks of 20k narinfo files sorted by creation time."},{"line_number":6,"context_line":"//! No additional metadata is included, and the files within each group are concatenated in arbitrary order."}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"37193fd8_c9f47b36","line":3,"in_reply_to":"5e13e293_7c7a7996","updated":"2023-11-10 15:13:04.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":6,"context_line":"//! No additional metadata is included, and the files within each group are concatenated in arbitrary order."},{"line_number":7,"context_line":"//! The narinfo format already starts each file off with a `StorePath` line, so this poses no practical issues."},{"line_number":8,"context_line":"//!"},{"line_number":9,"context_line":"//! TODO(edef): any retries/error handling whatsoever"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"use bytes::Bytes;"},{"line_number":12,"context_line":"use futures::{stream::FuturesUnordered, Stream, TryStreamExt};"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"b7c902d1_f54ec743","line":9,"updated":"2023-11-08 11:53:42.000000000","message":"Wouldnt\u0027t this be a matter of conveying the list of successful/failing requests in the response?","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":6,"context_line":"//! No additional metadata is included, and the files within each group are concatenated in arbitrary order."},{"line_number":7,"context_line":"//! The narinfo format already starts each file off with a `StorePath` line, so this poses no practical issues."},{"line_number":8,"context_line":"//!"},{"line_number":9,"context_line":"//! TODO(edef): any retries/error handling whatsoever"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"use bytes::Bytes;"},{"line_number":12,"context_line":"use futures::{stream::FuturesUnordered, Stream, TryStreamExt};"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"0b707b95_08f80040","line":9,"in_reply_to":"7c7bd39f_742a3a6d","updated":"2023-11-10 15:13:04.000000000","message":"saving this for a followup, marking resolved for this round.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"8cb4d8ef09d909c3dd8a0098512916cb8cffdb1e","unresolved":true,"context_lines":[{"line_number":6,"context_line":"//! No additional metadata is included, and the files within each group are concatenated in arbitrary order."},{"line_number":7,"context_line":"//! The narinfo format already starts each file off with a `StorePath` line, so this poses no practical issues."},{"line_number":8,"context_line":"//!"},{"line_number":9,"context_line":"//! TODO(edef): any retries/error handling whatsoever"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"use bytes::Bytes;"},{"line_number":12,"context_line":"use futures::{stream::FuturesUnordered, Stream, TryStreamExt};"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"7c7bd39f_742a3a6d","line":9,"in_reply_to":"b7c902d1_f54ec743","updated":"2023-11-09 11:07:21.000000000","message":"no, because we write files for *ranges*. we want a complete range for every file","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":19,"context_line":"    net::TcpStream,"},{"line_number":20,"context_line":"};"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"/// `keys` must be a slice of Nix path hashes in nixbase32 format"},{"line_number":23,"context_line":"/// file contents is concatenated and returned in ~arbitrary order"},{"line_number":24,"context_line":"/// any network error at all fails the entire batch, and there is no rate limiting"},{"line_number":25,"context_line":"fn fetch(keys: \u0026[[u8; 32]]) -\u003e impl Stream\u003cItem \u003d io::Result\u003cBytes\u003e\u003e {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"1cc28bbc_3939d3ef","line":22,"updated":"2023-11-08 11:53:42.000000000","message":"Please also document what this function does (on a high-level), not just the shape of the inputs and outputs.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":19,"context_line":"    net::TcpStream,"},{"line_number":20,"context_line":"};"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"/// `keys` must be a slice of Nix path hashes in nixbase32 format"},{"line_number":23,"context_line":"/// file contents is concatenated and returned in ~arbitrary order"},{"line_number":24,"context_line":"/// any network error at all fails the entire batch, and there is no rate limiting"},{"line_number":25,"context_line":"fn fetch(keys: \u0026[[u8; 32]]) -\u003e impl Stream\u003cItem \u003d io::Result\u003cBytes\u003e\u003e {"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"19248021_ab00bfab","line":22,"in_reply_to":"1cc28bbc_3939d3ef","updated":"2023-11-10 15:13:04.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":41,"context_line":"            (request, chunk.len())"},{"line_number":42,"context_line":"        })"},{"line_number":43,"context_line":"        .map(|(request, n)| async move {"},{"line_number":44,"context_line":"            let (mut read, mut write) \u003d TcpStream::connect(\"s3.amazonaws.com:80\")"},{"line_number":45,"context_line":"                .await?"},{"line_number":46,"context_line":"                .into_split();"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"abd1dcf1_fd5449a5","line":44,"updated":"2023-11-08 11:53:42.000000000","message":"Do we want to pass the host:port into this function? There\u0027s nothing specific to S3 here…","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"8cb4d8ef09d909c3dd8a0098512916cb8cffdb1e","unresolved":true,"context_lines":[{"line_number":41,"context_line":"            (request, chunk.len())"},{"line_number":42,"context_line":"        })"},{"line_number":43,"context_line":"        .map(|(request, n)| async move {"},{"line_number":44,"context_line":"            let (mut read, mut write) \u003d TcpStream::connect(\"s3.amazonaws.com:80\")"},{"line_number":45,"context_line":"                .await?"},{"line_number":46,"context_line":"                .into_split();"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"f223c5c5_0739a7b7","line":44,"in_reply_to":"abd1dcf1_fd5449a5","updated":"2023-11-09 11:07:21.000000000","message":"This is pretty S3-specific, and it\u0027ll only get more so when we follow up with auth and retries.\nFor example, the 100-sized chunks specifically match S3\u0027s limits.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":41,"context_line":"            (request, chunk.len())"},{"line_number":42,"context_line":"        })"},{"line_number":43,"context_line":"        .map(|(request, n)| async move {"},{"line_number":44,"context_line":"            let (mut read, mut write) \u003d TcpStream::connect(\"s3.amazonaws.com:80\")"},{"line_number":45,"context_line":"                .await?"},{"line_number":46,"context_line":"                .into_split();"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"7ecb30a4_28ee49be","line":44,"in_reply_to":"f223c5c5_0739a7b7","updated":"2023-11-10 15:13:04.000000000","message":"marking resolved (i should have done so on the previous round)","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":62,"context_line":"        })"},{"line_number":63,"context_line":"        .collect::\u003cFuturesUnordered\u003c_\u003e\u003e()"},{"line_number":64,"context_line":"}"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"async fn get_range("},{"line_number":67,"context_line":"    s3: \u0026\u0027static S3Client,"},{"line_number":68,"context_line":"    bucket: \u0026str,"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"303bd102_6e396783","line":65,"updated":"2023-11-08 11:53:42.000000000","message":"This needs a comment explaining what it does (and I\u0027d probably make the key a function arg that\u0027s passed in from the outside).","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":62,"context_line":"        })"},{"line_number":63,"context_line":"        .collect::\u003cFuturesUnordered\u003c_\u003e\u003e()"},{"line_number":64,"context_line":"}"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"async fn get_range("},{"line_number":67,"context_line":"    s3: \u0026\u0027static S3Client,"},{"line_number":68,"context_line":"    bucket: \u0026str,"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"a91284e0_a327e75d","line":65,"in_reply_to":"303bd102_6e396783","updated":"2023-11-10 15:13:04.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":91,"context_line":"}"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"fn exact_chunks(mut buf: Box\u003c[u8]\u003e) -\u003e Option\u003cBox\u003c[[u8; 32]]\u003e\u003e {"},{"line_number":94,"context_line":"    unsafe {"},{"line_number":95,"context_line":"        let ptr \u003d buf.as_mut_ptr();"},{"line_number":96,"context_line":"        let len \u003d buf.len();"},{"line_number":97,"context_line":""}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"8f737363_c566d242","line":94,"updated":"2023-11-08 11:53:42.000000000","message":"Needs SAFETY comment. Also, considering this is only used /once/ (before the fan-out), it\u0027s probably fine for this to be a bit slower (but safe).","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"8cb4d8ef09d909c3dd8a0098512916cb8cffdb1e","unresolved":false,"context_lines":[{"line_number":91,"context_line":"}"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"fn exact_chunks(mut buf: Box\u003c[u8]\u003e) -\u003e Option\u003cBox\u003c[[u8; 32]]\u003e\u003e {"},{"line_number":94,"context_line":"    unsafe {"},{"line_number":95,"context_line":"        let ptr \u003d buf.as_mut_ptr();"},{"line_number":96,"context_line":"        let len \u003d buf.len();"},{"line_number":97,"context_line":""}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"9e223706_5e4897b8","line":94,"in_reply_to":"8f737363_c566d242","updated":"2023-11-09 11:07:21.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":108,"context_line":"}"},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"#[tokio::main(flavor \u003d \"current_thread\")]"},{"line_number":111,"context_line":"async fn main() -\u003e Result\u003c(), lambda_runtime::Error\u003e {"},{"line_number":112,"context_line":"    let s3 \u003d S3Client::new(rusoto_core::Region::UsEast1);"},{"line_number":113,"context_line":"    let s3 \u003d \u0026*Box::leak(Box::new(s3));"},{"line_number":114,"context_line":""}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"1c3a4248_ac2edfba","line":111,"updated":"2023-11-08 11:53:42.000000000","message":"Can we have a second, non-lambda entrypoint into this? See also the other comment.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":108,"context_line":"}"},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"#[tokio::main(flavor \u003d \"current_thread\")]"},{"line_number":111,"context_line":"async fn main() -\u003e Result\u003c(), lambda_runtime::Error\u003e {"},{"line_number":112,"context_line":"    let s3 \u003d S3Client::new(rusoto_core::Region::UsEast1);"},{"line_number":113,"context_line":"    let s3 \u003d \u0026*Box::leak(Box::new(s3));"},{"line_number":114,"context_line":""}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"04f13bb6_4e1deb59","line":111,"in_reply_to":"1c3a4248_ac2edfba","updated":"2023-11-10 15:13:04.000000000","message":"saving this for a followup, marking resolved for this round.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":130,"context_line":"        .await"},{"line_number":131,"context_line":"}"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"#[derive(Debug, Deserialize)]"},{"line_number":134,"context_line":"struct Params {"},{"line_number":135,"context_line":"    bucket: String,"},{"line_number":136,"context_line":"    start: u64,"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"d0c06005_309b7e3a","line":133,"updated":"2023-11-08 11:53:42.000000000","message":"The fact of the lambda payload being a json serialization of these 3 values is an implementation detail of the lambda-specifics. We should probably add a comment explaining that here.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":130,"context_line":"        .await"},{"line_number":131,"context_line":"}"},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"#[derive(Debug, Deserialize)]"},{"line_number":134,"context_line":"struct Params {"},{"line_number":135,"context_line":"    bucket: String,"},{"line_number":136,"context_line":"    start: u64,"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"640cf95d_ef6b83d5","line":133,"in_reply_to":"d0c06005_309b7e3a","updated":"2023-11-10 15:13:04.000000000","message":"Done","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000036,"name":"flokli","email":"flokli@flokli.de","username":"flokli"},"change_message_id":"c8269c6ad47ea613e3307ee3ca31a4df329a3e1a","unresolved":true,"context_lines":[{"line_number":144,"context_line":"        aws_lambda_events::lambda_function_urls::LambdaFunctionUrlRequest,"},{"line_number":145,"context_line":"    \u003e,"},{"line_number":146,"context_line":") -\u003e Result\u003chyper::Response\u003cString\u003e, lambda_runtime::Error\u003e {"},{"line_number":147,"context_line":"    let mut params \u003d event.payload.body.ok_or(\"no body\")?;"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"    if event.payload.is_base64_encoded {"},{"line_number":150,"context_line":"        params \u003d String::from_utf8(data_encoding::BASE64.decode(params.as_bytes())?)?;"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"ab5b817e_ff03a0c3","line":147,"updated":"2023-11-08 11:53:42.000000000","message":"Can we factor out the lambda-specific parsing of request parameters into one function, and call a inner function that receives bucket/start/end, or Params directly? Then probably the remaining outside plumbing is small enough to just be kept inline in the lambda-specific `async fn main`.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"},{"author":{"_account_id":1000005,"name":"edef","email":"edef@edef.eu","username":"edef"},"change_message_id":"22a045285d82243eff45f6fcc6ba0b3efa189c0d","unresolved":false,"context_lines":[{"line_number":144,"context_line":"        aws_lambda_events::lambda_function_urls::LambdaFunctionUrlRequest,"},{"line_number":145,"context_line":"    \u003e,"},{"line_number":146,"context_line":") -\u003e Result\u003chyper::Response\u003cString\u003e, lambda_runtime::Error\u003e {"},{"line_number":147,"context_line":"    let mut params \u003d event.payload.body.ok_or(\"no body\")?;"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"    if event.payload.is_base64_encoded {"},{"line_number":150,"context_line":"        params \u003d String::from_utf8(data_encoding::BASE64.decode(params.as_bytes())?)?;"}],"source_content_type":"text/x-rustsrc","patch_set":5,"id":"f04e9321_4f8ddd8f","line":147,"in_reply_to":"ab5b817e_ff03a0c3","updated":"2023-11-10 15:13:04.000000000","message":"saving this for a followup, marking resolved for this round.","commit_id":"38e401a0c97fd7000e8578228cead52b4d381793"}]}