#52555 [RFC PATCH 0/3] Decentralized substitute distribution with ERIS

Message #157 received at 52555@debbugs.gnu.org (full text, mbox, reply):

Received: (at 52555) by debbugs.gnu.org; 29 Dec 2022 18:14:37 +0000
From debbugs-submit-bounces@debbugs.gnu.org Thu Dec 29 13:14:37 2022
Received: from localhost ([127.0.0.1]:32810 helo=debbugs.gnu.org)
	by debbugs.gnu.org with esmtp (Exim 4.84_2)
	(envelope-from <debbugs-submit-bounces@debbugs.gnu.org>)
	id 1pAxQL-0007Pf-6l
	for submit@debbugs.gnu.org; Thu, 29 Dec 2022 13:14:37 -0500
Received: from mout01.posteo.de ([185.67.36.65]:54713)
 by debbugs.gnu.org with esmtp (Exim 4.84_2)
 (envelope-from <pukkamustard@posteo.net>) id 1pAxQD-0007OK-34
 for 52555@debbugs.gnu.org; Thu, 29 Dec 2022 13:14:30 -0500
Received: from submission (posteo.de [185.67.36.169]) 
 by mout01.posteo.de (Postfix) with ESMTPS id 54D32240149
 for <52555@debbugs.gnu.org>; Thu, 29 Dec 2022 19:14:23 +0100 (CET)
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=posteo.net; s=2017;
 t=1672337663; bh=HALOp6BmLbWkF/uCoyDNMO6MNjP132bCXeHMxPGwckE=;
 h=From:To:Cc:Subject:Date:From;
 b=M9s4m7gqOxDtn1oXZWShSq/dECdvB4UuAcJivvUt4Lgh+/JAMhn0rPDOXrILXnR6E
 akjsOL1WZGk4ob7R6pHkfkjdh9UCiWAzbW88qZqoE7WVTDG2OutBQEivnJjdkbQ5ox
 de0LfvMGr2myvEMwb8G2e2E+XQiMUjYt3ronDb0K0yw/G+8FNcwYqSU3zDTce74FQ5
 zC1Jz9xVH8Kg8zALAGZr6gdJZhe/awXoBnbYY135OLf/uSNzF9FgaqOkNX4FcwulIx
 N55XqyozUIgaRn2T6RCDyy27zxYldju+PIjb2HYxhRHyVJ2sHQMd2SxSXXBveShYJ6
 xpyP/PHugameQ==
Received: from customer (localhost [127.0.0.1])
 by submission (posteo.de) with ESMTPSA id 4Njc3f5tmNz6tn8;
 Thu, 29 Dec 2022 19:14:22 +0100 (CET)
From: pukkamustard <pukkamustard@posteo.net>
To: 52555@debbugs.gnu.org
Subject: [PATCH v3 4/8] WIP: substitute: Fetch substitutes using ERIS.
Date: Thu, 29 Dec 2022 18:13:23 +0000
Message-Id: <20221229181327.758-5-pukkamustard@posteo.net>
In-Reply-To: <20221229181327.758-1-pukkamustard@posteo.net>
References: <20221229181327.758-1-pukkamustard@posteo.net>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-Spam-Score: -2.3 (--)
X-Debbugs-Envelope-To: 52555
Cc: pukkamustard <pukkamustard@posteo.net>
X-BeenThere: debbugs-submit@debbugs.gnu.org
X-Mailman-Version: 2.1.18
Precedence: list
List-Id: <debbugs-submit.debbugs.gnu.org>
List-Unsubscribe: <https://debbugs.gnu.org/cgi-bin/mailman/options/debbugs-submit>, 
 <mailto:debbugs-submit-request@debbugs.gnu.org?subject=unsubscribe>
List-Archive: <https://debbugs.gnu.org/cgi-bin/mailman/private/debbugs-submit/>
List-Post: <mailto:debbugs-submit@debbugs.gnu.org>
List-Help: <mailto:debbugs-submit-request@debbugs.gnu.org?subject=help>
List-Subscribe: <https://debbugs.gnu.org/cgi-bin/mailman/listinfo/debbugs-submit>, 
 <mailto:debbugs-submit-request@debbugs.gnu.org?subject=subscribe>
Errors-To: debbugs-submit-bounces@debbugs.gnu.org
Sender: "Debbugs-submit" <debbugs-submit-bounces@debbugs.gnu.org>
X-Spam-Score: -3.3 (---)

* guix/scripts/substitute.scm (process-substitution): Fetch substitutes using ERIS.
* guix/eris.scm (call-with-eris-block-ref): New procedure.

TODO:

- When to set prefer-eris? in scripts/substitute.scm?
---
 guix/eris.scm               |  7 +++++++
 guix/scripts/substitute.scm | 42 +++++++++++++++++++++++++++++--------
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/guix/eris.scm b/guix/eris.scm
index 29d5e7b1db..d9a0914b67 100644
--- a/guix/eris.scm
+++ b/guix/eris.scm
@@ -23,6 +23,7 @@ (define-module (guix eris)
   #:use-module (guix eris fs-store)
 
   #:export (guix-eris-block-reducer
+            call-with-eris-block-ref
 
             %eris-block-store-directory))
 
@@ -34,3 +35,9 @@ (define %eris-block-store-directory
 (define (guix-eris-block-reducer)
   "Returns a block reducer that stores blocks of ERIS encoded content."
   (eris-fs-store-reducer (%eris-block-store-directory)))
+
+(define (call-with-eris-block-ref f)
+  (let ((fs-store-block-ref
+         (eris-fs-store-ref
+          (%eris-block-store-directory))))
+    (f fs-store-block-ref)))
diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm
index 0efa61b0d7..8cf011d7e6 100755
--- a/guix/scripts/substitute.scm
+++ b/guix/scripts/substitute.scm
@@ -55,12 +55,16 @@ (define-module (guix scripts substitute)
   #:use-module (ice-9 ftw)
   #:use-module (rnrs bytevectors)
   #:use-module (srfi srfi-1)
+  #:use-module (srfi srfi-2)
   #:use-module (srfi srfi-19)
   #:use-module (srfi srfi-26)
   #:use-module (srfi srfi-34)
   #:use-module (srfi srfi-35)
   #:use-module (srfi srfi-71)
   #:use-module (web uri)
+  #:use-module (eris)
+  #:use-module (eris read-capability)
+  #:use-module (guix eris)
   #:use-module (guix http-client)
   #:export (%allow-unauthenticated-substitutes?
             %reply-file-descriptor
@@ -439,11 +443,12 @@ (define-syntax-rule (with-cached-connection uri port exp ...)
 
 (define* (download-nar narinfo destination
                        #:key status-port
-                       deduplicate? print-build-trace?)
+                       deduplicate? print-build-trace?
+                       prefer-eris?)
   "Download the nar prescribed in NARINFO, which is assumed to be authentic
 and authorized, and write it to DESTINATION.  When DEDUPLICATE? is true, and
-if DESTINATION is in the store, deduplicate its files.  Print a status line to
-STATUS-PORT."
+if DESTINATION is in the store, deduplicate its files.  When PREFER-ERIS? is
+true, attempt to ERIS to get the nar. Print a status line to STATUS-PORT."
   (define destination-in-store?
     (string-prefix? (string-append (%store-prefix) "/")
                     destination))
@@ -474,14 +479,29 @@ (define (fetch uri)
                        #:port port
                        #:keep-alive? #t
                        #:buffered? #f))))
+      ((urn)
+       (let ((read-capability (->eris-read-capability uri)))
+         (if (eris-read-capability? read-capability)
+             (call-with-eris-block-ref
+              (lambda (block-ref)
+                (values (open-eris-input-port
+                         read-capability
+                         #:block-ref block-ref)
+                        #f)))
+             (leave (G_ "unsupported substitute URI scheme: ~a~%")
+                    (uri->string uri)))))
       (else
        (leave (G_ "unsupported substitute URI scheme: ~a~%")
               (uri->string uri)))))
 
   (let ((uri compression file-size
-             (narinfo-best-uri narinfo
-                               #:fast-decompression?
-                               %prefer-fast-decompression?)))
+             (if (and prefer-eris? (narinfo-eris-urn narinfo))
+                 (values (narinfo-eris-urn narinfo) "zstd" #f)
+                 (narinfo-best-uri narinfo
+                                   #:fast-decompression?
+                                   %prefer-fast-decompression?))))
+
+
     (unless print-build-trace?
       (format (current-error-port)
               (G_ "Downloading ~a...~%") (uri->string uri)))
@@ -631,7 +651,8 @@ (define* (process-substitution/fallback port narinfo destination
 
 (define* (process-substitution port store-item destination
                                #:key cache-urls acl
-                               deduplicate? print-build-trace?)
+                               deduplicate? print-build-trace?
+                               prefer-eris?)
   "Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to
 DESTINATION as a nar file.  Verify the substitute against ACL, and verify its
 hash against what appears in the narinfo.  When DEDUPLICATE? is true, and if
@@ -660,7 +681,8 @@ (define narinfo
     (download-nar narinfo destination
                   #:status-port port
                   #:deduplicate? deduplicate?
-                  #:print-build-trace? print-build-trace?)))
+                  #:print-build-trace? print-build-trace?
+                  #:prefer-eris? prefer-eris?)))
 
 
 ;;;
@@ -858,7 +880,9 @@ (define reply-port
                                      #:acl (current-acl)
                                      #:deduplicate? deduplicate?
                                      #:print-build-trace?
-                                     print-build-trace?)
+                                     print-build-trace?
+                                     ;; TODO when to prefer ERIS?
+                                     #:prefer-eris? #t)
                (loop))))))
        (opts
         (leave (G_ "~a: unrecognized options~%") opts))))))
-- 
2.38.1

Send a report that this bug log contains spam.