cmd/swarm, swarm: added access control functionality (#17404)

Co-authored-by: Janos Guljas <janos@resenje.org>
Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>
Co-authored-by: Balint Gabor <balint.g@gmail.com>

cmd/swarm/access.go

@@ -0,0 +1,219 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+package main
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/api/client"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var salt = make([]byte, 32)
+
+func init() {
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
+		panic("reading from crypto/rand failed: " + err.Error())
+	}
+}
+
+func accessNewPass(ctx *cli.Context) {
+	args := ctx.Args()
+	if len(args) != 1 {
+		utils.Fatalf("Expected 1 argument - the ref")
+	}
+
+	var (
+		ae        *api.AccessEntry
+		accessKey []byte
+		err       error
+		ref       = args[0]
+		password  = getPassPhrase("", 0, makePasswordList(ctx))
+		dryRun    = ctx.Bool(SwarmDryRunFlag.Name)
+	)
+	accessKey, ae, err = api.DoPasswordNew(ctx, password, salt)
+	if err != nil {
+		utils.Fatalf("error getting session key: %v", err)
+	}
+	m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
+	if dryRun {
+		err = printManifests(m, nil)
+		if err != nil {
+			utils.Fatalf("had an error printing the manifests: %v", err)
+		}
+	} else {
+		utils.Fatalf("uploading manifests")
+		err = uploadManifests(ctx, m, nil)
+		if err != nil {
+			utils.Fatalf("had an error uploading the manifests: %v", err)
+		}
+	}
+}
+
+func accessNewPK(ctx *cli.Context) {
+	args := ctx.Args()
+	if len(args) != 1 {
+		utils.Fatalf("Expected 1 argument - the ref")
+	}
+
+	var (
+		ae               *api.AccessEntry
+		sessionKey       []byte
+		err              error
+		ref              = args[0]
+		privateKey       = getPrivKey(ctx)
+		granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name)
+		dryRun           = ctx.Bool(SwarmDryRunFlag.Name)
+	)
+	sessionKey, ae, err = api.DoPKNew(ctx, privateKey, granteePublicKey, salt)
+	if err != nil {
+		utils.Fatalf("error getting session key: %v", err)
+	}
+	m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
+	if dryRun {
+		err = printManifests(m, nil)
+		if err != nil {
+			utils.Fatalf("had an error printing the manifests: %v", err)
+		}
+	} else {
+		err = uploadManifests(ctx, m, nil)
+		if err != nil {
+			utils.Fatalf("had an error uploading the manifests: %v", err)
+		}
+	}
+}
+
+func accessNewACT(ctx *cli.Context) {
+	args := ctx.Args()
+	if len(args) != 1 {
+		utils.Fatalf("Expected 1 argument - the ref")
+	}
+
+	var (
+		ae          *api.AccessEntry
+		actManifest *api.Manifest
+		accessKey   []byte
+		err         error
+		ref         = args[0]
+		grantees    = []string{}
+		actFilename = ctx.String(SwarmAccessGrantKeysFlag.Name)
+		privateKey  = getPrivKey(ctx)
+		dryRun      = ctx.Bool(SwarmDryRunFlag.Name)
+	)
+
+	bytes, err := ioutil.ReadFile(actFilename)
+	if err != nil {
+		utils.Fatalf("had an error reading the grantee public key list")
+	}
+	grantees = strings.Split(string(bytes), "\n")
+	accessKey, ae, actManifest, err = api.DoACTNew(ctx, privateKey, salt, grantees)
+	if err != nil {
+		utils.Fatalf("error generating ACT manifest: %v", err)
+	}
+
+	if err != nil {
+		utils.Fatalf("error getting session key: %v", err)
+	}
+	m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
+	if err != nil {
+		utils.Fatalf("error generating root access manifest: %v", err)
+	}
+
+	if dryRun {
+		err = printManifests(m, actManifest)
+		if err != nil {
+			utils.Fatalf("had an error printing the manifests: %v", err)
+		}
+	} else {
+		err = uploadManifests(ctx, m, actManifest)
+		if err != nil {
+			utils.Fatalf("had an error uploading the manifests: %v", err)
+		}
+	}
+}
+
+func printManifests(rootAccessManifest, actManifest *api.Manifest) error {
+	js, err := json.Marshal(rootAccessManifest)
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(js))
+
+	if actManifest != nil {
+		js, err := json.Marshal(actManifest)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(js))
+	}
+	return nil
+}
+
+func uploadManifests(ctx *cli.Context, rootAccessManifest, actManifest *api.Manifest) error {
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+	client := client.NewClient(bzzapi)
+
+	var (
+		key string
+		err error
+	)
+	if actManifest != nil {
+		key, err = client.UploadManifest(actManifest, false)
+		if err != nil {
+			return err
+		}
+
+		rootAccessManifest.Entries[0].Access.Act = key
+	}
+	key, err = client.UploadManifest(rootAccessManifest, false)
+	if err != nil {
+		return err
+	}
+	fmt.Println(key)
+	return nil
+}
+
+// makePasswordList reads password lines from the file specified by the global --password flag
+// and also by the same subcommand --password flag.
+// This function ia a fork of utils.MakePasswordList to lookup cli context for subcommand.
+// Function ctx.SetGlobal is not setting the global flag value that can be accessed
+// by ctx.GlobalString using the current version of cli package.
+func makePasswordList(ctx *cli.Context) []string {
+	path := ctx.GlobalString(utils.PasswordFileFlag.Name)
+	if path == "" {
+		path = ctx.String(utils.PasswordFileFlag.Name)
+		if path == "" {
+			return nil
+		}
+	}
+	text, err := ioutil.ReadFile(path)
+	if err != nil {
+		utils.Fatalf("Failed to read password file: %v", err)
+	}
+	lines := strings.Split(string(text), "\n")
+	// Sanitise DOS line endings.
+	for i := range lines {
+		lines[i] = strings.TrimRight(lines[i], "\r")
+	}
+	return lines
+}

cmd/swarm/access_test.go

@@ -0,0 +1,581 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	gorand "math/rand"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+)
+
+// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
+// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
+// The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
+// is then fetched through 2nd node. since the tested code is not key-aware - we can just
+// fetch from the 2nd node using HTTP BasicAuth
+func TestAccessPassword(t *testing.T) {
+	cluster := newTestCluster(t, 1)
+	defer cluster.Shutdown()
+	proxyNode := cluster.Nodes[0]
+
+	// create a tmp file
+	tmp, err := ioutil.TempDir("", "swarm-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+
+	// write data to file
+	data := "notsorandomdata"
+	dataFilename := filepath.Join(tmp, "data.txt")
+
+	err = ioutil.WriteFile(dataFilename, []byte(data), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hashRegexp := `[a-f\d]{128}`
+
+	// upload the file with 'swarm up' and expect a hash
+	up := runSwarm(t,
+		"--bzzapi",
+		proxyNode.URL, //it doesn't matter through which node we upload content
+		"up",
+		"--encrypt",
+		dataFilename)
+	_, matches := up.ExpectRegexp(hashRegexp)
+	up.ExpectExit()
+
+	if len(matches) < 1 {
+		t.Fatal("no matches found")
+	}
+
+	ref := matches[0]
+
+	password := "smth"
+	passwordFilename := filepath.Join(tmp, "password.txt")
+
+	err = ioutil.WriteFile(passwordFilename, []byte(password), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	up = runSwarm(t,
+		"access",
+		"new",
+		"pass",
+		"--dry-run",
+		"--password",
+		passwordFilename,
+		ref,
+	)
+
+	_, matches = up.ExpectRegexp(".+")
+	up.ExpectExit()
+
+	if len(matches) == 0 {
+		t.Fatalf("stdout not matched")
+	}
+
+	var m api.Manifest
+
+	err = json.Unmarshal([]byte(matches[0]), &m)
+	if err != nil {
+		t.Fatalf("unmarshal manifest: %v", err)
+	}
+
+	if len(m.Entries) != 1 {
+		t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
+	}
+
+	e := m.Entries[0]
+
+	ct := "application/bzz-manifest+json"
+	if e.ContentType != ct {
+		t.Errorf("expected %q content type, got %q", ct, e.ContentType)
+	}
+
+	if e.Access == nil {
+		t.Fatal("manifest access is nil")
+	}
+
+	a := e.Access
+
+	if a.Type != "pass" {
+		t.Errorf(`got access type %q, expected "pass"`, a.Type)
+	}
+	if len(a.Salt) < 32 {
+		t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
+	}
+	if a.KdfParams == nil {
+		t.Fatal("manifest access kdf params is nil")
+	}
+
+	client := swarm.NewClient(cluster.Nodes[0].URL)
+
+	hash, err := client.UploadManifest(&m, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	httpClient := &http.Client{}
+
+	url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
+	response, err := httpClient.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response.StatusCode != http.StatusUnauthorized {
+		t.Fatal("should be a 401")
+	}
+	authHeader := response.Header.Get("WWW-Authenticate")
+	if authHeader == "" {
+		t.Fatal("should be something here")
+	}
+
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.SetBasicAuth("", password)
+
+	response, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		t.Errorf("expected status %v, got %v", http.StatusOK, response.StatusCode)
+	}
+	d, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(d) != data {
+		t.Errorf("expected decrypted data %q, got %q", data, string(d))
+	}
+
+	wrongPasswordFilename := filepath.Join(tmp, "password-wrong.txt")
+
+	err = ioutil.WriteFile(wrongPasswordFilename, []byte("just wr0ng"), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//download file with 'swarm down' with wrong password
+	up = runSwarm(t,
+		"--bzzapi",
+		proxyNode.URL,
+		"down",
+		"bzz:/"+hash,
+		tmp,
+		"--password",
+		wrongPasswordFilename)
+
+	_, matches = up.ExpectRegexp("unauthorized")
+	if len(matches) != 1 && matches[0] != "unauthorized" {
+		t.Fatal(`"unauthorized" not found in output"`)
+	}
+	up.ExpectExit()
+}
+
+// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
+// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
+// The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
+// Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
+// the test will fail if the proxy's given private key is not granted on the ACT.
+func TestAccessPK(t *testing.T) {
+	// Setup Swarm and upload a test file to it
+	cluster := newTestCluster(t, 1)
+	defer cluster.Shutdown()
+
+	// create a tmp file
+	tmp, err := ioutil.TempFile("", "swarm-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tmp.Close()
+	defer os.Remove(tmp.Name())
+
+	// write data to file
+	data := "notsorandomdata"
+	_, err = io.WriteString(tmp, data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hashRegexp := `[a-f\d]{128}`
+
+	// upload the file with 'swarm up' and expect a hash
+	up := runSwarm(t,
+		"--bzzapi",
+		cluster.Nodes[0].URL,
+		"up",
+		"--encrypt",
+		tmp.Name())
+	_, matches := up.ExpectRegexp(hashRegexp)
+	up.ExpectExit()
+
+	if len(matches) < 1 {
+		t.Fatal("no matches found")
+	}
+
+	ref := matches[0]
+
+	pk := cluster.Nodes[0].PrivateKey
+	granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
+
+	publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	passFile, err := ioutil.TempFile("", "swarm-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer passFile.Close()
+	defer os.Remove(passFile.Name())
+	_, err = io.WriteString(passFile, testPassphrase)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, publisherAccount := getTestAccount(t, publisherDir)
+	up = runSwarm(t,
+		"--bzzaccount",
+		publisherAccount.Address.String(),
+		"--password",
+		passFile.Name(),
+		"--datadir",
+		publisherDir,
+		"--bzzapi",
+		cluster.Nodes[0].URL,
+		"access",
+		"new",
+		"pk",
+		"--dry-run",
+		"--grant-key",
+		hex.EncodeToString(granteePubKey),
+		ref,
+	)
+
+	_, matches = up.ExpectRegexp(".+")
+	up.ExpectExit()
+
+	if len(matches) == 0 {
+		t.Fatalf("stdout not matched")
+	}
+
+	var m api.Manifest
+
+	err = json.Unmarshal([]byte(matches[0]), &m)
+	if err != nil {
+		t.Fatalf("unmarshal manifest: %v", err)
+	}
+
+	if len(m.Entries) != 1 {
+		t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
+	}
+
+	e := m.Entries[0]
+
+	ct := "application/bzz-manifest+json"
+	if e.ContentType != ct {
+		t.Errorf("expected %q content type, got %q", ct, e.ContentType)
+	}
+
+	if e.Access == nil {
+		t.Fatal("manifest access is nil")
+	}
+
+	a := e.Access
+
+	if a.Type != "pk" {
+		t.Errorf(`got access type %q, expected "pk"`, a.Type)
+	}
+	if len(a.Salt) < 32 {
+		t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
+	}
+	if a.KdfParams != nil {
+		t.Fatal("manifest access kdf params should be nil")
+	}
+
+	client := swarm.NewClient(cluster.Nodes[0].URL)
+
+	hash, err := client.UploadManifest(&m, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	httpClient := &http.Client{}
+
+	url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
+	response, err := httpClient.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response.StatusCode != http.StatusOK {
+		t.Fatal("should be a 200")
+	}
+	d, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(d) != data {
+		t.Errorf("expected decrypted data %q, got %q", data, string(d))
+	}
+}
+
+// TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control
+// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
+// set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through
+// one of the nodes then disappears.
+func TestAccessACT(t *testing.T) {
+	// Setup Swarm and upload a test file to it
+	cluster := newTestCluster(t, 3)
+	defer cluster.Shutdown()
+
+	var uploadThroughNode = cluster.Nodes[0]
+	client := swarm.NewClient(uploadThroughNode.URL)
+
+	r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
+	nodeToSkip := r1.Intn(3) // a number between 0 and 2 (node indices in `cluster`)
+	// create a tmp file
+	tmp, err := ioutil.TempFile("", "swarm-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tmp.Close()
+	defer os.Remove(tmp.Name())
+
+	// write data to file
+	data := "notsorandomdata"
+	_, err = io.WriteString(tmp, data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hashRegexp := `[a-f\d]{128}`
+
+	// upload the file with 'swarm up' and expect a hash
+	up := runSwarm(t,
+		"--bzzapi",
+		cluster.Nodes[0].URL,
+		"up",
+		"--encrypt",
+		tmp.Name())
+	_, matches := up.ExpectRegexp(hashRegexp)
+	up.ExpectExit()
+
+	if len(matches) < 1 {
+		t.Fatal("no matches found")
+	}
+
+	ref := matches[0]
+	grantees := []string{}
+	for i, v := range cluster.Nodes {
+		if i == nodeToSkip {
+			continue
+		}
+		pk := v.PrivateKey
+		granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
+		grantees = append(grantees, hex.EncodeToString(granteePubKey))
+	}
+
+	granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list.csv")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer granteesPubkeyListFile.Close()
+	defer os.Remove(granteesPubkeyListFile.Name())
+
+	publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	passFile, err := ioutil.TempFile("", "swarm-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer passFile.Close()
+	defer os.Remove(passFile.Name())
+	_, err = io.WriteString(passFile, testPassphrase)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, publisherAccount := getTestAccount(t, publisherDir)
+	up = runSwarm(t,
+		"--bzzaccount",
+		publisherAccount.Address.String(),
+		"--password",
+		passFile.Name(),
+		"--datadir",
+		publisherDir,
+		"--bzzapi",
+		cluster.Nodes[0].URL,
+		"access",
+		"new",
+		"act",
+		"--grant-keys",
+		granteesPubkeyListFile.Name(),
+		ref,
+	)
+
+	_, matches = up.ExpectRegexp(`[a-f\d]{64}`)
+	up.ExpectExit()
+
+	if len(matches) == 0 {
+		t.Fatalf("stdout not matched")
+	}
+	hash := matches[0]
+	m, _, err := client.DownloadManifest(hash)
+	if err != nil {
+		t.Fatalf("unmarshal manifest: %v", err)
+	}
+
+	if len(m.Entries) != 1 {
+		t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
+	}
+
+	e := m.Entries[0]
+
+	ct := "application/bzz-manifest+json"
+	if e.ContentType != ct {
+		t.Errorf("expected %q content type, got %q", ct, e.ContentType)
+	}
+
+	if e.Access == nil {
+		t.Fatal("manifest access is nil")
+	}
+
+	a := e.Access
+
+	if a.Type != "act" {
+		t.Fatalf(`got access type %q, expected "act"`, a.Type)
+	}
+	if len(a.Salt) < 32 {
+		t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
+	}
+	if a.KdfParams != nil {
+		t.Fatal("manifest access kdf params should be nil")
+	}
+
+	httpClient := &http.Client{}
+
+	// all nodes except the skipped node should be able to decrypt the content
+	for i, node := range cluster.Nodes {
+		log.Debug("trying to fetch from node", "node index", i)
+
+		url := node.URL + "/" + "bzz:/" + hash
+		response, err := httpClient.Get(url)
+		if err != nil {
+			t.Fatal(err)
+		}
+		log.Debug("got response from node", "response code", response.StatusCode)
+
+		if i == nodeToSkip {
+			log.Debug("reached node to skip", "status code", response.StatusCode)
+
+			if response.StatusCode != http.StatusUnauthorized {
+				t.Fatalf("should be a 401")
+			}
+
+			continue
+		}
+
+		if response.StatusCode != http.StatusOK {
+			t.Fatal("should be a 200")
+		}
+		d, err := ioutil.ReadAll(response.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if string(d) != data {
+			t.Errorf("expected decrypted data %q, got %q", data, string(d))
+		}
+	}
+}
+
+// TestKeypairSanity is a sanity test for the crypto scheme for ACT. it asserts the correct shared secret according to
+// the specs at https://github.com/ethersphere/swarm-docs/blob/eb857afda906c6e7bb90d37f3f334ccce5eef230/act.md
+func TestKeypairSanity(t *testing.T) {
+	salt := make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
+		t.Fatalf("reading from crypto/rand failed: %v", err.Error())
+	}
+	sharedSecret := "a85586744a1ddd56a7ed9f33fa24f40dd745b3a941be296a0d60e329dbdb896d"
+
+	for i, v := range []struct {
+		publisherPriv string
+		granteePub    string
+	}{
+		{
+			publisherPriv: "ec5541555f3bc6376788425e9d1a62f55a82901683fd7062c5eddcc373a73459",
+			granteePub:    "0226f213613e843a413ad35b40f193910d26eb35f00154afcde9ded57479a6224a",
+		},
+		{
+			publisherPriv: "70c7a73011aa56584a0009ab874794ee7e5652fd0c6911cd02f8b6267dd82d2d",
+			granteePub:    "02e6f8d5e28faaa899744972bb847b6eb805a160494690c9ee7197ae9f619181db",
+		},
+	} {
+		b, _ := hex.DecodeString(v.granteePub)
+		granteePub, _ := crypto.DecompressPubkey(b)
+		publisherPrivate, _ := crypto.HexToECDSA(v.publisherPriv)
+
+		ssKey, err := api.NewSessionKeyPK(publisherPrivate, granteePub, salt)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		hasher := sha3.NewKeccak256()
+		hasher.Write(salt)
+		shared, err := hex.DecodeString(sharedSecret)
+		if err != nil {
+			t.Fatal(err)
+		}
+		hasher.Write(shared)
+		sum := hasher.Sum(nil)
+
+		if !bytes.Equal(ssKey, sum) {
+			t.Fatalf("%d: got a session key mismatch", i)
+		}
+	}
+}

cmd/swarm/config.go

@@ -78,6 +78,7 @@ const (
 	SWARM_ENV_STORE_PATH           = "SWARM_STORE_PATH"
 	SWARM_ENV_STORE_CAPACITY       = "SWARM_STORE_CAPACITY"
 	SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
+	SWARM_ACCESS_PASSWORD          = "SWARM_ACCESS_PASSWORD"
 	GETH_ENV_DATADIR               = "GETH_DATADIR"
 )
 

cmd/swarm/download.go

@@ -68,18 +68,36 @@ func download(ctx *cli.Context) {
 		utils.Fatalf("could not parse uri argument: %v", err)
 	}
 
+	dl := func(credentials string) error {
 		// assume behaviour according to --recursive switch
 		if isRecursive {
-		if err := client.DownloadDirectory(uri.Addr, uri.Path, dest); err != nil {
+			if err := client.DownloadDirectory(uri.Addr, uri.Path, dest, credentials); err != nil {
-			utils.Fatalf("encoutered an error while downloading directory: %v", err)
+				if err == swarm.ErrUnauthorized {
+					return err
+				}
+				return fmt.Errorf("directory %s: %v", uri.Path, err)
 			}
 		} else {
 			// we are downloading a file
-		log.Debug(fmt.Sprintf("downloading file/path from a manifest. hash: %s, path:%s", uri.Addr, uri.Path))
+			log.Debug("downloading file/path from a manifest", "uri.Addr", uri.Addr, "uri.Path", uri.Path)
 
-		err := client.DownloadFile(uri.Addr, uri.Path, dest)
+			err := client.DownloadFile(uri.Addr, uri.Path, dest, credentials)
 			if err != nil {
-			utils.Fatalf("could not download %s from given address: %s. error: %v", uri.Path, uri.Addr, err)
+				if err == swarm.ErrUnauthorized {
+					return err
+				}
+				return fmt.Errorf("file %s from address: %s: %v", uri.Path, uri.Addr, err)
 			}
 		}
+		return nil
+	}
+	if passwords := makePasswordList(ctx); passwords != nil {
+		password := getPassPhrase(fmt.Sprintf("Downloading %s is restricted", uri), 0, passwords)
+		err = dl(password)
+	} else {
+		err = dl("")
+	}
+	if err != nil {
+		utils.Fatalf("download: %v", err)
+	}
 }

cmd/swarm/list.go

@@ -44,7 +44,7 @@ func list(ctx *cli.Context) {
 
 	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 	client := swarm.NewClient(bzzapi)
-	list, err := client.List(manifest, prefix)
+	list, err := client.List(manifest, prefix, "")
 	if err != nil {
 		utils.Fatalf("Failed to generate file and directory list: %s", err)
 	}

cmd/swarm/main.go

@@ -155,6 +155,14 @@ var (
 		Name:  "defaultpath",
 		Usage: "path to file served for empty url path (none)",
 	}
+	SwarmAccessGrantKeyFlag = cli.StringFlag{
+		Name:  "grant-key",
+		Usage: "grants a given public key access to an ACT",
+	}
+	SwarmAccessGrantKeysFlag = cli.StringFlag{
+		Name:  "grant-keys",
+		Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
+	}
 	SwarmUpFromStdinFlag = cli.BoolFlag{
 		Name:  "stdin",
 		Usage: "reads data to be uploaded from stdin",
@@ -167,6 +175,15 @@ var (
 		Name:  "encrypt",
 		Usage: "use encrypted upload",
 	}
+	SwarmAccessPasswordFlag = cli.StringFlag{
+		Name:   "password",
+		Usage:  "Password",
+		EnvVar: SWARM_ACCESS_PASSWORD,
+	}
+	SwarmDryRunFlag = cli.BoolFlag{
+		Name:  "dry-run",
+		Usage: "dry-run",
+	}
 	CorsStringFlag = cli.StringFlag{
 		Name:   "corsdomain",
 		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
@@ -252,6 +269,61 @@ func init() {
 			Flags:              []cli.Flag{SwarmEncryptedFlag},
 			Description:        "uploads a file or directory to swarm using the HTTP API and prints the root hash",
 		},
+		{
+			CustomHelpTemplate: helpTemplate,
+			Name:               "access",
+			Usage:              "encrypts a reference and embeds it into a root manifest",
+			ArgsUsage:          "<ref>",
+			Description:        "encrypts a reference and embeds it into a root manifest",
+			Subcommands: []cli.Command{
+				{
+					CustomHelpTemplate: helpTemplate,
+					Name:               "new",
+					Usage:              "encrypts a reference and embeds it into a root manifest",
+					ArgsUsage:          "<ref>",
+					Description:        "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					Subcommands: []cli.Command{
+						{
+							Action:             accessNewPass,
+							CustomHelpTemplate: helpTemplate,
+							Flags: []cli.Flag{
+								utils.PasswordFileFlag,
+								SwarmDryRunFlag,
+							},
+							Name:        "pass",
+							Usage:       "encrypts a reference with a password and embeds it into a root manifest",
+							ArgsUsage:   "<ref>",
+							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+						},
+						{
+							Action:             accessNewPK,
+							CustomHelpTemplate: helpTemplate,
+							Flags: []cli.Flag{
+								utils.PasswordFileFlag,
+								SwarmDryRunFlag,
+								SwarmAccessGrantKeyFlag,
+							},
+							Name:        "pk",
+							Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+							ArgsUsage:   "<ref>",
+							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+						},
+						{
+							Action:             accessNewACT,
+							CustomHelpTemplate: helpTemplate,
+							Flags: []cli.Flag{
+								SwarmAccessGrantKeysFlag,
+								SwarmDryRunFlag,
+							},
+							Name:        "act",
+							Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+							ArgsUsage:   "<ref>",
+							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+						},
+					},
+				},
+			},
+		},
 		{
 			CustomHelpTemplate: helpTemplate,
 			Name:               "resource",
@@ -306,14 +378,11 @@ func init() {
 		{
 			Action:      download,
 			Name:        "down",
-			Flags:     []cli.Flag{SwarmRecursiveFlag},
+			Flags:       []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
 			Usage:       "downloads a swarm manifest or a file inside a manifest",
 			ArgsUsage:   " <uri> [<dir>]",
-			Description: `
+			Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
-Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.
-`,
 		},
-
 		{
 			Name:               "manifest",
 			CustomHelpTemplate: helpTemplate,
@@ -413,16 +482,14 @@ pv(1) tool to get a progress bar:
 					Name:               "import",
 					Usage:              "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
 					ArgsUsage:          "<chunkdb> <file>",
-					Description: `
+					Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
-Import chunks from a tar archive into a local chunk database (use - to read from stdin).
 
     swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
 
 The import may be quite large, consider piping the input through the Unix
 pv(1) tool to get a progress bar:
 
-    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -
+    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
-`,
 				},
 				{
 					Action:             dbClean,
@@ -535,6 +602,7 @@ func version(ctx *cli.Context) error {
 func bzzd(ctx *cli.Context) error {
 	//build a valid bzzapi.Config from all available sources:
 	//default config, file config, command line and env vars
+
 	bzzconfig, err := buildConfig(ctx)
 	if err != nil {
 		utils.Fatalf("unable to configure swarm: %v", err)
@@ -557,6 +625,7 @@ func bzzd(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf("can't create node: %v", err)
 	}
+
 	//a few steps need to be done after the config phase is completed,
 	//due to overriding behavior
 	initSwarmNode(bzzconfig, stack, ctx)

cmd/swarm/run_test.go

@@ -18,10 +18,12 @@ package main
 
 import (
 	"context"
+	"crypto/ecdsa"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"sync"
@@ -181,6 +183,7 @@ type testNode struct {
 	Enode      string
 	Dir        string
 	IpcPath    string
+	PrivateKey *ecdsa.PrivateKey
 	Client     *rpc.Client
 	Cmd        *cmdtest.TestCmd
 }
@@ -289,7 +292,11 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 func newTestNode(t *testing.T, dir string) *testNode {
 
 	conf, account := getTestAccount(t, dir)
-	node := &testNode{Dir: dir}
+	ks := keystore.NewKeyStore(path.Join(dir, "keystore"), 1<<18, 1)
+
+	pk := decryptStoreAccount(ks, account.Address.Hex(), []string{testPassphrase})
+
+	node := &testNode{Dir: dir, PrivateKey: pk}
 
 	// assign ports
 	ports, err := getAvailableTCPPorts(2)

swarm/api/act.go

@@ -0,0 +1,468 @@
+package api
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/ecies"
+	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/swarm/log"
+	"github.com/ethereum/go-ethereum/swarm/sctx"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"golang.org/x/crypto/scrypt"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var (
+	ErrDecrypt                = errors.New("cant decrypt - forbidden")
+	ErrUnknownAccessType      = errors.New("unknown access type (or not implemented)")
+	ErrDecryptDomainForbidden = errors.New("decryption request domain forbidden - can only decrypt on localhost")
+	AllowedDecryptDomains     = []string{
+		"localhost",
+		"127.0.0.1",
+	}
+)
+
+const EMPTY_CREDENTIALS = ""
+
+type AccessEntry struct {
+	Type      AccessType
+	Publisher string
+	Salt      []byte
+	Act       string
+	KdfParams *KdfParams
+}
+
+type DecryptFunc func(*ManifestEntry) error
+
+func (a *AccessEntry) MarshalJSON() (out []byte, err error) {
+
+	return json.Marshal(struct {
+		Type      AccessType `json:"type,omitempty"`
+		Publisher string     `json:"publisher,omitempty"`
+		Salt      string     `json:"salt,omitempty"`
+		Act       string     `json:"act,omitempty"`
+		KdfParams *KdfParams `json:"kdf_params,omitempty"`
+	}{
+		Type:      a.Type,
+		Publisher: a.Publisher,
+		Salt:      hex.EncodeToString(a.Salt),
+		Act:       a.Act,
+		KdfParams: a.KdfParams,
+	})
+
+}
+
+func (a *AccessEntry) UnmarshalJSON(value []byte) error {
+	v := struct {
+		Type      AccessType `json:"type,omitempty"`
+		Publisher string     `json:"publisher,omitempty"`
+		Salt      string     `json:"salt,omitempty"`
+		Act       string     `json:"act,omitempty"`
+		KdfParams *KdfParams `json:"kdf_params,omitempty"`
+	}{}
+
+	err := json.Unmarshal(value, &v)
+	if err != nil {
+		return err
+	}
+	a.Act = v.Act
+	a.KdfParams = v.KdfParams
+	a.Publisher = v.Publisher
+	a.Salt, err = hex.DecodeString(v.Salt)
+	if err != nil {
+		return err
+	}
+	if len(a.Salt) != 32 {
+		return errors.New("salt should be 32 bytes long")
+	}
+	a.Type = v.Type
+	return nil
+}
+
+type KdfParams struct {
+	N int `json:"n"`
+	P int `json:"p"`
+	R int `json:"r"`
+}
+
+type AccessType string
+
+const AccessTypePass = AccessType("pass")
+const AccessTypePK = AccessType("pk")
+const AccessTypeACT = AccessType("act")
+
+func NewAccessEntryPassword(salt []byte, kdfParams *KdfParams) (*AccessEntry, error) {
+	if len(salt) != 32 {
+		return nil, fmt.Errorf("salt should be 32 bytes long")
+	}
+	return &AccessEntry{
+		Type:      AccessTypePass,
+		Salt:      salt,
+		KdfParams: kdfParams,
+	}, nil
+}
+
+func NewAccessEntryPK(publisher string, salt []byte) (*AccessEntry, error) {
+	if len(publisher) != 66 {
+		return nil, fmt.Errorf("publisher should be 66 characters long, got %d", len(publisher))
+	}
+	if len(salt) != 32 {
+		return nil, fmt.Errorf("salt should be 32 bytes long")
+	}
+	return &AccessEntry{
+		Type:      AccessTypePK,
+		Publisher: publisher,
+		Salt:      salt,
+	}, nil
+}
+
+func NewAccessEntryACT(publisher string, salt []byte, act string) (*AccessEntry, error) {
+	if len(salt) != 32 {
+		return nil, fmt.Errorf("salt should be 32 bytes long")
+	}
+	if len(publisher) != 66 {
+		return nil, fmt.Errorf("publisher should be 66 characters long")
+	}
+
+	return &AccessEntry{
+		Type:      AccessTypeACT,
+		Publisher: publisher,
+		Salt:      salt,
+		Act:       act,
+	}, nil
+}
+
+func NOOPDecrypt(*ManifestEntry) error {
+	return nil
+}
+
+var DefaultKdfParams = NewKdfParams(262144, 1, 8)
+
+func NewKdfParams(n, p, r int) *KdfParams {
+
+	return &KdfParams{
+		N: n,
+		P: p,
+		R: r,
+	}
+}
+
+// NewSessionKeyPassword creates a session key based on a shared secret (password) and the given salt
+// and kdf parameters in the access entry
+func NewSessionKeyPassword(password string, accessEntry *AccessEntry) ([]byte, error) {
+	if accessEntry.Type != AccessTypePass {
+		return nil, errors.New("incorrect access entry type")
+	}
+	return scrypt.Key(
+		[]byte(password),
+		accessEntry.Salt,
+		accessEntry.KdfParams.N,
+		accessEntry.KdfParams.R,
+		accessEntry.KdfParams.P,
+		32,
+	)
+}
+
+// NewSessionKeyPK creates a new ACT Session Key using an ECDH shared secret for the given key pair and the given salt value
+func NewSessionKeyPK(private *ecdsa.PrivateKey, public *ecdsa.PublicKey, salt []byte) ([]byte, error) {
+	granteePubEcies := ecies.ImportECDSAPublic(public)
+	privateKey := ecies.ImportECDSA(private)
+
+	bytes, err := privateKey.GenerateShared(granteePubEcies, 16, 16)
+	if err != nil {
+		return nil, err
+	}
+	bytes = append(salt, bytes...)
+	sessionKey := crypto.Keccak256(bytes)
+	return sessionKey, nil
+}
+
+func (a *API) NodeSessionKey(privateKey *ecdsa.PrivateKey, publicKey *ecdsa.PublicKey, salt []byte) ([]byte, error) {
+	return NewSessionKeyPK(privateKey, publicKey, salt)
+}
+func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.PrivateKey) DecryptFunc {
+	return func(m *ManifestEntry) error {
+		if m.Access == nil {
+			return nil
+		}
+
+		allowed := false
+		requestDomain := sctx.GetHost(ctx)
+		for _, v := range AllowedDecryptDomains {
+			if strings.Contains(requestDomain, v) {
+				allowed = true
+			}
+		}
+
+		if !allowed {
+			return ErrDecryptDomainForbidden
+		}
+
+		switch m.Access.Type {
+		case "pass":
+			if credentials != "" {
+				key, err := NewSessionKeyPassword(credentials, m.Access)
+				if err != nil {
+					return err
+				}
+
+				ref, err := hex.DecodeString(m.Hash)
+				if err != nil {
+					return err
+				}
+
+				enc := NewRefEncryption(len(ref) - 8)
+				decodedRef, err := enc.Decrypt(ref, key)
+				if err != nil {
+					return ErrDecrypt
+				}
+
+				m.Hash = hex.EncodeToString(decodedRef)
+				m.Access = nil
+				return nil
+			}
+			return ErrDecrypt
+		case "pk":
+			publisherBytes, err := hex.DecodeString(m.Access.Publisher)
+			if err != nil {
+				return ErrDecrypt
+			}
+			publisher, err := crypto.DecompressPubkey(publisherBytes)
+			if err != nil {
+				return ErrDecrypt
+			}
+			key, err := a.NodeSessionKey(pk, publisher, m.Access.Salt)
+			if err != nil {
+				return ErrDecrypt
+			}
+			ref, err := hex.DecodeString(m.Hash)
+			if err != nil {
+				return err
+			}
+
+			enc := NewRefEncryption(len(ref) - 8)
+			decodedRef, err := enc.Decrypt(ref, key)
+			if err != nil {
+				return ErrDecrypt
+			}
+
+			m.Hash = hex.EncodeToString(decodedRef)
+			m.Access = nil
+			return nil
+		case "act":
+			publisherBytes, err := hex.DecodeString(m.Access.Publisher)
+			if err != nil {
+				return ErrDecrypt
+			}
+			publisher, err := crypto.DecompressPubkey(publisherBytes)
+			if err != nil {
+				return ErrDecrypt
+			}
+
+			sessionKey, err := a.NodeSessionKey(pk, publisher, m.Access.Salt)
+			if err != nil {
+				return ErrDecrypt
+			}
+
+			hasher := sha3.NewKeccak256()
+			hasher.Write(append(sessionKey, 0))
+			lookupKey := hasher.Sum(nil)
+
+			hasher.Reset()
+
+			hasher.Write(append(sessionKey, 1))
+			accessKeyDecryptionKey := hasher.Sum(nil)
+
+			lk := hex.EncodeToString(lookupKey)
+			list, err := a.GetManifestList(ctx, NOOPDecrypt, storage.Address(common.Hex2Bytes(m.Access.Act)), lk)
+
+			found := ""
+			for _, v := range list.Entries {
+				if v.Path == lk {
+					found = v.Hash
+				}
+			}
+
+			if found == "" {
+				return ErrDecrypt
+			}
+
+			v, err := hex.DecodeString(found)
+			if err != nil {
+				return err
+			}
+			enc := NewRefEncryption(len(v) - 8)
+			decodedRef, err := enc.Decrypt(v, accessKeyDecryptionKey)
+			if err != nil {
+				return ErrDecrypt
+			}
+
+			ref, err := hex.DecodeString(m.Hash)
+			if err != nil {
+				return err
+			}
+
+			enc = NewRefEncryption(len(ref) - 8)
+			decodedMainRef, err := enc.Decrypt(ref, decodedRef)
+			if err != nil {
+				return ErrDecrypt
+			}
+			m.Hash = hex.EncodeToString(decodedMainRef)
+			m.Access = nil
+			return nil
+		}
+		return ErrUnknownAccessType
+	}
+}
+
+func GenerateAccessControlManifest(ctx *cli.Context, ref string, accessKey []byte, ae *AccessEntry) (*Manifest, error) {
+	refBytes, err := hex.DecodeString(ref)
+	if err != nil {
+		return nil, err
+	}
+	// encrypt ref with accessKey
+	enc := NewRefEncryption(len(refBytes))
+	encrypted, err := enc.Encrypt(refBytes, accessKey)
+	if err != nil {
+		return nil, err
+	}
+
+	m := &Manifest{
+		Entries: []ManifestEntry{
+			{
+				Hash:        hex.EncodeToString(encrypted),
+				ContentType: ManifestType,
+				ModTime:     time.Now(),
+				Access:      ae,
+			},
+		},
+	}
+
+	return m, nil
+}
+
+func DoPKNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) {
+	if granteePublicKey == "" {
+		return nil, nil, errors.New("need a grantee Public Key")
+	}
+	b, err := hex.DecodeString(granteePublicKey)
+	if err != nil {
+		log.Error("error decoding grantee public key", "err", err)
+		return nil, nil, err
+	}
+
+	granteePub, err := crypto.DecompressPubkey(b)
+	if err != nil {
+		log.Error("error decompressing grantee public key", "err", err)
+		return nil, nil, err
+	}
+
+	sessionKey, err = NewSessionKeyPK(privateKey, granteePub, salt)
+	if err != nil {
+		log.Error("error getting session key", "err", err)
+		return nil, nil, err
+	}
+
+	ae, err = NewAccessEntryPK(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt)
+	if err != nil {
+		log.Error("error generating access entry", "err", err)
+		return nil, nil, err
+	}
+
+	return sessionKey, ae, nil
+}
+
+func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees []string) (accessKey []byte, ae *AccessEntry, actManifest *Manifest, err error) {
+	if len(grantees) == 0 {
+		return nil, nil, nil, errors.New("did not get any grantee public keys")
+	}
+
+	publisherPub := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey))
+	grantees = append(grantees, publisherPub)
+
+	accessKey = make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
+		panic("reading from crypto/rand failed: " + err.Error())
+	}
+	if _, err := io.ReadFull(rand.Reader, accessKey); err != nil {
+		panic("reading from crypto/rand failed: " + err.Error())
+	}
+
+	lookupPathEncryptedAccessKeyMap := make(map[string]string)
+	i := 0
+	for _, v := range grantees {
+		i++
+		if v == "" {
+			return nil, nil, nil, errors.New("need a grantee Public Key")
+		}
+		b, err := hex.DecodeString(v)
+		if err != nil {
+			log.Error("error decoding grantee public key", "err", err)
+			return nil, nil, nil, err
+		}
+
+		granteePub, err := crypto.DecompressPubkey(b)
+		if err != nil {
+			log.Error("error decompressing grantee public key", "err", err)
+			return nil, nil, nil, err
+		}
+		sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt)
+
+		hasher := sha3.NewKeccak256()
+		hasher.Write(append(sessionKey, 0))
+		lookupKey := hasher.Sum(nil)
+
+		hasher.Reset()
+		hasher.Write(append(sessionKey, 1))
+
+		accessKeyEncryptionKey := hasher.Sum(nil)
+
+		enc := NewRefEncryption(len(accessKey))
+		encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey)
+
+		lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey)
+	}
+
+	m := &Manifest{
+		Entries: []ManifestEntry{},
+	}
+
+	for k, v := range lookupPathEncryptedAccessKeyMap {
+		m.Entries = append(m.Entries, ManifestEntry{
+			Path:        k,
+			Hash:        v,
+			ContentType: "text/plain",
+		})
+	}
+
+	ae, err = NewAccessEntryACT(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt, "")
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return accessKey, ae, m, nil
+}
+
+func DoPasswordNew(ctx *cli.Context, password string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) {
+	ae, err = NewAccessEntryPassword(salt, DefaultKdfParams)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	sessionKey, err = NewSessionKeyPassword(password, ae)
+	if err != nil {
+		return nil, nil, err
+	}
+	return sessionKey, ae, nil
+}

swarm/api/api.go

@@ -19,6 +19,9 @@ package api
 import (
 	"archive/tar"
 	"context"
+	"crypto/ecdsa"
+	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"math/big"
@@ -43,6 +46,10 @@ import (
 	opentracing "github.com/opentracing/opentracing-go"
 )
 
+var (
+	ErrNotFound = errors.New("not found")
+)
+
 var (
 	apiResolveCount        = metrics.NewRegisteredCounter("api.resolve.count", nil)
 	apiResolveFail         = metrics.NewRegisteredCounter("api.resolve.fail", nil)
@@ -227,14 +234,18 @@ type API struct {
 	resource  *mru.Handler
 	fileStore *storage.FileStore
 	dns       Resolver
+	Decryptor func(context.Context, string) DecryptFunc
 }
 
 // NewAPI the api constructor initialises a new API instance.
-func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler) (self *API) {
+func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler, pk *ecdsa.PrivateKey) (self *API) {
 	self = &API{
 		fileStore: fileStore,
 		dns:       dns,
 		resource:  resourceHandler,
+		Decryptor: func(ctx context.Context, credentials string) DecryptFunc {
+			return self.doDecrypt(ctx, credentials, pk)
+		},
 	}
 	return
 }
@@ -260,8 +271,30 @@ func (a *API) Store(ctx context.Context, data io.Reader, size int64, toEncrypt b
 // ErrResolve is returned when an URI cannot be resolved from ENS.
 type ErrResolve error
 
+// Resolve a name into a content-addressed hash
+// where address could be an ENS name, or a content addressed hash
+func (a *API) Resolve(ctx context.Context, address string) (storage.Address, error) {
+	// if DNS is not configured, return an error
+	if a.dns == nil {
+		if hashMatcher.MatchString(address) {
+			return common.Hex2Bytes(address), nil
+		}
+		apiResolveFail.Inc(1)
+		return nil, fmt.Errorf("no DNS to resolve name: %q", address)
+	}
+	// try and resolve the address
+	resolved, err := a.dns.Resolve(address)
+	if err != nil {
+		if hashMatcher.MatchString(address) {
+			return common.Hex2Bytes(address), nil
+		}
+		return nil, err
+	}
+	return resolved[:], nil
+}
+
 // Resolve resolves a URI to an Address using the MultiResolver.
-func (a *API) Resolve(ctx context.Context, uri *URI) (storage.Address, error) {
+func (a *API) ResolveURI(ctx context.Context, uri *URI, credentials string) (storage.Address, error) {
 	apiResolveCount.Inc(1)
 	log.Trace("resolving", "uri", uri.Addr)
 
@@ -280,28 +313,44 @@ func (a *API) Resolve(ctx context.Context, uri *URI) (storage.Address, error) {
 		return key, nil
 	}
 
-	// if DNS is not configured, check if the address is a hash
+	addr, err := a.Resolve(ctx, uri.Addr)
-	if a.dns == nil {
+	if err != nil {
-		key := uri.Address()
-		if key == nil {
-			apiResolveFail.Inc(1)
-			return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr)
-		}
-		return key, nil
-	}
-
-	// try and resolve the address
-	resolved, err := a.dns.Resolve(uri.Addr)
-	if err == nil {
-		return resolved[:], nil
-	}
-
-	key := uri.Address()
-	if key == nil {
-		apiResolveFail.Inc(1)
 		return nil, err
 	}
-	return key, nil
+
+	if uri.Path == "" {
+		return addr, nil
+	}
+	walker, err := a.NewManifestWalker(ctx, addr, a.Decryptor(ctx, credentials), nil)
+	if err != nil {
+		return nil, err
+	}
+	var entry *ManifestEntry
+	walker.Walk(func(e *ManifestEntry) error {
+		// if the entry matches the path, set entry and stop
+		// the walk
+		if e.Path == uri.Path {
+			entry = e
+			// return an error to cancel the walk
+			return errors.New("found")
+		}
+		// ignore non-manifest files
+		if e.ContentType != ManifestType {
+			return nil
+		}
+		// if the manifest's path is a prefix of the
+		// requested path, recurse into it by returning
+		// nil and continuing the walk
+		if strings.HasPrefix(uri.Path, e.Path) {
+			return nil
+		}
+		return ErrSkipManifest
+	})
+	if entry == nil {
+		return nil, errors.New("not found")
+	}
+	addr = storage.Address(common.Hex2Bytes(entry.Hash))
+	return addr, nil
 }
 
 // Put provides singleton manifest creation on top of FileStore store
@@ -332,10 +381,10 @@ func (a *API) Put(ctx context.Context, content string, contentType string, toEnc
 // Get uses iterative manifest retrieval and prefix matching
 // to resolve basePath to content using FileStore retrieve
 // it returns a section reader, mimeType, status, the key of the actual content and an error
-func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) {
+func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) {
 	log.Debug("api.get", "key", manifestAddr, "path", path)
 	apiGetCount.Inc(1)
-	trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil)
+	trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt)
 	if err != nil {
 		apiGetNotFound.Inc(1)
 		status = http.StatusNotFound
@@ -347,6 +396,16 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string
 
 	if entry != nil {
 		log.Debug("trie got entry", "key", manifestAddr, "path", path, "entry.Hash", entry.Hash)
+
+		if entry.ContentType == ManifestType {
+			log.Debug("entry is manifest", "key", manifestAddr, "new key", entry.Hash)
+			adr, err := hex.DecodeString(entry.Hash)
+			if err != nil {
+				return nil, "", 0, nil, err
+			}
+			return a.Get(ctx, decrypt, adr, entry.Path)
+		}
+
 		// we need to do some extra work if this is a mutable resource manifest
 		if entry.ContentType == ResourceContentType {
 
@@ -398,7 +457,7 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string
 				log.Trace("resource is multihash", "key", manifestAddr)
 
 				// get the manifest the multihash digest points to
-				trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil)
+				trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt)
 				if err != nil {
 					apiGetNotFound.Inc(1)
 					status = http.StatusNotFound
@@ -451,7 +510,7 @@ func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Add
 		apiDeleteFail.Inc(1)
 		return nil, err
 	}
-	key, err := a.Resolve(ctx, uri)
+	key, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS)
 
 	if err != nil {
 		return nil, err
@@ -470,13 +529,13 @@ func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Add
 
 // GetDirectoryTar fetches a requested directory as a tarstream
 // it returns an io.Reader and an error. Do not forget to Close() the returned ReadCloser
-func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, error) {
+func (a *API) GetDirectoryTar(ctx context.Context, decrypt DecryptFunc, uri *URI) (io.ReadCloser, error) {
 	apiGetTarCount.Inc(1)
-	addr, err := a.Resolve(ctx, uri)
+	addr, err := a.Resolve(ctx, uri.Addr)
 	if err != nil {
 		return nil, err
 	}
-	walker, err := a.NewManifestWalker(ctx, addr, nil)
+	walker, err := a.NewManifestWalker(ctx, addr, decrypt, nil)
 	if err != nil {
 		apiGetTarFail.Inc(1)
 		return nil, err
@@ -542,9 +601,9 @@ func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, err
 
 // GetManifestList lists the manifest entries for the specified address and prefix
 // and returns it as a ManifestList
-func (a *API) GetManifestList(ctx context.Context, addr storage.Address, prefix string) (list ManifestList, err error) {
+func (a *API) GetManifestList(ctx context.Context, decryptor DecryptFunc, addr storage.Address, prefix string) (list ManifestList, err error) {
 	apiManifestListCount.Inc(1)
-	walker, err := a.NewManifestWalker(ctx, addr, nil)
+	walker, err := a.NewManifestWalker(ctx, addr, decryptor, nil)
 	if err != nil {
 		apiManifestListFail.Inc(1)
 		return ManifestList{}, err
@@ -631,7 +690,7 @@ func (a *API) UpdateManifest(ctx context.Context, addr storage.Address, update f
 func (a *API) Modify(ctx context.Context, addr storage.Address, path, contentHash, contentType string) (storage.Address, error) {
 	apiModifyCount.Inc(1)
 	quitC := make(chan bool)
-	trie, err := loadManifest(ctx, a.fileStore, addr, quitC)
+	trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt)
 	if err != nil {
 		apiModifyFail.Inc(1)
 		return nil, err
@@ -663,7 +722,7 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content []
 		apiAddFileFail.Inc(1)
 		return nil, "", err
 	}
-	mkey, err := a.Resolve(ctx, uri)
+	mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS)
 	if err != nil {
 		apiAddFileFail.Inc(1)
 		return nil, "", err
@@ -770,7 +829,7 @@ func (a *API) RemoveFile(ctx context.Context, mhash string, path string, fname s
 		apiRmFileFail.Inc(1)
 		return "", err
 	}
-	mkey, err := a.Resolve(ctx, uri)
+	mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS)
 	if err != nil {
 		apiRmFileFail.Inc(1)
 		return "", err
@@ -837,7 +896,7 @@ func (a *API) AppendFile(ctx context.Context, mhash, path, fname string, existin
 		apiAppendFileFail.Inc(1)
 		return nil, "", err
 	}
-	mkey, err := a.Resolve(ctx, uri)
+	mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS)
 	if err != nil {
 		apiAppendFileFail.Inc(1)
 		return nil, "", err
@@ -891,13 +950,13 @@ func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver
 	if err != nil {
 		return nil, nil, err
 	}
-	addr, err = a.Resolve(ctx, uri)
+	addr, err = a.Resolve(ctx, uri.Addr)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	quitC := make(chan bool)
-	rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC)
+	rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt)
 	if err != nil {
 		return nil, nil, fmt.Errorf("can't load manifest %v: %v", addr.String(), err)
 	}
@@ -955,7 +1014,7 @@ func (a *API) ResourceHashSize() int {
 
 // ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the address of the metadata chunk.
 func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (storage.Address, error) {
-	trie, err := loadManifest(ctx, a.fileStore, addr, nil)
+	trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt)
 	if err != nil {
 		return nil, fmt.Errorf("cannot load resource manifest: %v", err)
 	}

swarm/api/api_test.go

@@ -19,6 +19,7 @@ package api
 import (
 	"context"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -28,10 +29,17 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/swarm/log"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/sctx"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
+func init() {
+	loglevel := flag.Int("loglevel", 2, "loglevel")
+	flag.Parse()
+	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
+}
+
 func testAPI(t *testing.T, f func(*API, bool)) {
 	datadir, err := ioutil.TempDir("", "bzz-test")
 	if err != nil {
@@ -42,7 +50,7 @@ func testAPI(t *testing.T, f func(*API, bool)) {
 	if err != nil {
 		return
 	}
-	api := NewAPI(fileStore, nil, nil)
+	api := NewAPI(fileStore, nil, nil, nil)
 	f(api, false)
 	f(api, true)
 }
@@ -85,7 +93,7 @@ func expResponse(content string, mimeType string, status int) *Response {
 
 func testGet(t *testing.T, api *API, bzzhash, path string) *testResponse {
 	addr := storage.Address(common.Hex2Bytes(bzzhash))
-	reader, mimeType, status, _, err := api.Get(context.TODO(), addr, path)
+	reader, mimeType, status, _, err := api.Get(context.TODO(), NOOPDecrypt, addr, path)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -229,7 +237,7 @@ func TestAPIResolve(t *testing.T) {
 			if x.immutable {
 				uri.Scheme = "bzz-immutable"
 			}
-			res, err := api.Resolve(context.TODO(), uri)
+			res, err := api.ResolveURI(context.TODO(), uri, "")
 			if err == nil {
 				if x.expectErr != nil {
 					t.Fatalf("expected error %q, got result %q", x.expectErr, res)
@@ -373,3 +381,55 @@ func TestMultiResolver(t *testing.T) {
 		})
 	}
 }
+
+func TestDecryptOriginForbidden(t *testing.T) {
+	ctx := context.TODO()
+	ctx = sctx.SetHost(ctx, "swarm-gateways.net")
+
+	me := &ManifestEntry{
+		Access: &AccessEntry{Type: AccessTypePass},
+	}
+
+	api := NewAPI(nil, nil, nil, nil)
+
+	f := api.Decryptor(ctx, "")
+	err := f(me)
+	if err != ErrDecryptDomainForbidden {
+		t.Fatalf("should fail with ErrDecryptDomainForbidden, got %v", err)
+	}
+}
+
+func TestDecryptOrigin(t *testing.T) {
+	for _, v := range []struct {
+		host        string
+		expectError error
+	}{
+		{
+			host:        "localhost",
+			expectError: ErrDecrypt,
+		},
+		{
+			host:        "127.0.0.1",
+			expectError: ErrDecrypt,
+		},
+		{
+			host:        "swarm-gateways.net",
+			expectError: ErrDecryptDomainForbidden,
+		},
+	} {
+		ctx := context.TODO()
+		ctx = sctx.SetHost(ctx, v.host)
+
+		me := &ManifestEntry{
+			Access: &AccessEntry{Type: AccessTypePass},
+		}
+
+		api := NewAPI(nil, nil, nil, nil)
+
+		f := api.Decryptor(ctx, "")
+		err := f(me)
+		if err != v.expectError {
+			t.Fatalf("should fail with %v, got %v", v.expectError, err)
+		}
+	}
+}

swarm/api/client/client.go

@@ -43,6 +43,10 @@ var (
 	DefaultClient  = NewClient(DefaultGateway)
 )
 
+var (
+	ErrUnauthorized = errors.New("unauthorized")
+)
+
 func NewClient(gateway string) *Client {
 	return &Client{
 		Gateway: gateway,
@@ -188,7 +192,7 @@ func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bo
 
 // DownloadDirectory downloads the files contained in a swarm manifest under
 // the given path into a local directory (existing files will be overwritten)
-func (c *Client) DownloadDirectory(hash, path, destDir string) error {
+func (c *Client) DownloadDirectory(hash, path, destDir, credentials string) error {
 	stat, err := os.Stat(destDir)
 	if err != nil {
 		return err
@@ -201,13 +205,20 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error {
 	if err != nil {
 		return err
 	}
+	if credentials != "" {
+		req.SetBasicAuth("", credentials)
+	}
 	req.Header.Set("Accept", "application/x-tar")
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return err
 	}
 	defer res.Body.Close()
-	if res.StatusCode != http.StatusOK {
+	switch res.StatusCode {
+	case http.StatusOK:
+	case http.StatusUnauthorized:
+		return ErrUnauthorized
+	default:
 		return fmt.Errorf("unexpected HTTP status: %s", res.Status)
 	}
 	tr := tar.NewReader(res.Body)
@@ -248,7 +259,7 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error {
 // DownloadFile downloads a single file into the destination directory
 // if the manifest entry does not specify a file name - it will fallback
 // to the hash of the file as a filename
-func (c *Client) DownloadFile(hash, path, dest string) error {
+func (c *Client) DownloadFile(hash, path, dest, credentials string) error {
 	hasDestinationFilename := false
 	if stat, err := os.Stat(dest); err == nil {
 		hasDestinationFilename = !stat.IsDir()
@@ -261,9 +272,9 @@ func (c *Client) DownloadFile(hash, path, dest string) error {
 		}
 	}
 
-	manifestList, err := c.List(hash, path)
+	manifestList, err := c.List(hash, path, credentials)
 	if err != nil {
-		return fmt.Errorf("could not list manifest: %v", err)
+		return err
 	}
 
 	switch len(manifestList.Entries) {
@@ -280,13 +291,19 @@ func (c *Client) DownloadFile(hash, path, dest string) error {
 	if err != nil {
 		return err
 	}
+	if credentials != "" {
+		req.SetBasicAuth("", credentials)
+	}
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return err
 	}
 	defer res.Body.Close()
-
+	switch res.StatusCode {
-	if res.StatusCode != http.StatusOK {
+	case http.StatusOK:
+	case http.StatusUnauthorized:
+		return ErrUnauthorized
+	default:
 		return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode)
 	}
 	filename := ""
@@ -367,13 +384,24 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) {
 // - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
 //
 // where entries ending with "/" are common prefixes.
-func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
+func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, error) {
-	res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix)
+	req, err := http.NewRequest(http.MethodGet, c.Gateway+"/bzz-list:/"+hash+"/"+prefix, nil)
+	if err != nil {
+		return nil, err
+	}
+	if credentials != "" {
+		req.SetBasicAuth("", credentials)
+	}
+	res, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
 	defer res.Body.Close()
-	if res.StatusCode != http.StatusOK {
+	switch res.StatusCode {
+	case http.StatusOK:
+	case http.StatusUnauthorized:
+		return nil, ErrUnauthorized
+	default:
 		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
 	}
 	var list api.ManifestList

swarm/api/client/client_test.go

@@ -228,7 +228,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmp)
-	if err := client.DownloadDirectory(hash, "", tmp); err != nil {
+	if err := client.DownloadDirectory(hash, "", tmp, ""); err != nil {
 		t.Fatal(err)
 	}
 	for _, file := range testDirFiles {
@@ -265,7 +265,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) {
 	}
 
 	ls := func(prefix string) []string {
-		list, err := client.List(hash, prefix)
+		list, err := client.List(hash, prefix, "")
 		if err != nil {
 			t.Fatal(err)
 		}

swarm/api/encrypt.go

@@ -0,0 +1,76 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"encoding/binary"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/swarm/storage/encryption"
+)
+
+type RefEncryption struct {
+	spanEncryption encryption.Encryption
+	dataEncryption encryption.Encryption
+	span           []byte
+}
+
+func NewRefEncryption(refSize int) *RefEncryption {
+	span := make([]byte, 8)
+	binary.LittleEndian.PutUint64(span, uint64(refSize))
+	return &RefEncryption{
+		spanEncryption: encryption.New(0, uint32(refSize/32), sha3.NewKeccak256),
+		dataEncryption: encryption.New(refSize, 0, sha3.NewKeccak256),
+		span:           span,
+	}
+}
+
+func (re *RefEncryption) Encrypt(ref []byte, key []byte) ([]byte, error) {
+	encryptedSpan, err := re.spanEncryption.Encrypt(re.span, key)
+	if err != nil {
+		return nil, err
+	}
+	encryptedData, err := re.dataEncryption.Encrypt(ref, key)
+	if err != nil {
+		return nil, err
+	}
+	encryptedRef := make([]byte, len(ref)+8)
+	copy(encryptedRef[:8], encryptedSpan)
+	copy(encryptedRef[8:], encryptedData)
+
+	return encryptedRef, nil
+}
+
+func (re *RefEncryption) Decrypt(ref []byte, key []byte) ([]byte, error) {
+	decryptedSpan, err := re.spanEncryption.Decrypt(ref[:8], key)
+	if err != nil {
+		return nil, err
+	}
+
+	size := binary.LittleEndian.Uint64(decryptedSpan)
+	if size != uint64(len(ref)-8) {
+		return nil, errors.New("invalid span in encrypted reference")
+	}
+
+	decryptedRef, err := re.dataEncryption.Decrypt(ref[8:], key)
+	if err != nil {
+		return nil, err
+	}
+
+	return decryptedRef, nil
+}

swarm/api/filesystem.go

@@ -191,7 +191,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error {
 	if err != nil {
 		return err
 	}
-	addr, err := fs.api.Resolve(context.TODO(), uri)
+	addr, err := fs.api.Resolve(context.TODO(), uri.Addr)
 	if err != nil {
 		return err
 	}
@@ -202,7 +202,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error {
 	}
 
 	quitC := make(chan bool)
-	trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC)
+	trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC, NOOPDecrypt)
 	if err != nil {
 		log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err))
 		return err

swarm/api/filesystem_test.go

@@ -64,7 +64,7 @@ func TestApiDirUpload0(t *testing.T) {
 		checkResponse(t, resp, exp)
 
 		addr := storage.Address(common.Hex2Bytes(bzzhash))
-		_, _, _, _, err = api.Get(context.TODO(), addr, "")
+		_, _, _, _, err = api.Get(context.TODO(), NOOPDecrypt, addr, "")
 		if err == nil {
 			t.Fatalf("expected error: %v", err)
 		}
@@ -143,7 +143,7 @@ func TestApiDirUploadModify(t *testing.T) {
 		exp = expResponse(content, "text/css", 0)
 		checkResponse(t, resp, exp)
 
-		_, _, _, _, err = api.Get(context.TODO(), addr, "")
+		_, _, _, _, err = api.Get(context.TODO(), nil, addr, "")
 		if err == nil {
 			t.Errorf("expected error: %v", err)
 		}

swarm/api/http/middleware.go

@@ -9,6 +9,7 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/log"
+	"github.com/ethereum/go-ethereum/swarm/sctx"
 	"github.com/ethereum/go-ethereum/swarm/spancontext"
 	"github.com/pborman/uuid"
 )
@@ -35,6 +36,15 @@ func SetRequestID(h http.Handler) http.Handler {
 	})
 }
 
+func SetRequestHost(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r = r.WithContext(sctx.SetHost(r.Context(), r.Host))
+		log.Info("setting request host", "ruid", GetRUID(r.Context()), "host", sctx.GetHost(r.Context()))
+
+		h.ServeHTTP(w, r)
+	})
+}
+
 func ParseURI(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
@@ -87,7 +97,7 @@ func RecoverPanic(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer func() {
 			if err := recover(); err != nil {
-				log.Error("panic recovery!", "stack trace", debug.Stack(), "url", r.URL.String(), "headers", r.Header)
+				log.Error("panic recovery!", "stack trace", string(debug.Stack()), "url", r.URL.String(), "headers", r.Header)
 			}
 		}()
 		h.ServeHTTP(w, r)

swarm/api/http/response.go

@@ -79,7 +79,7 @@ func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg s
 }
 
 func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) {
-	log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code)
 	RespondTemplate(w, r, "error", msg, code)
 }
 

swarm/api/http/server.go

@@ -23,7 +23,6 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -97,6 +96,7 @@ func NewServer(api *api.API, corsString string) *Server {
 	defaultMiddlewares := []Adapter{
 		RecoverPanic,
 		SetRequestID,
+		SetRequestHost,
 		InitLoggingResponseWriter,
 		ParseURI,
 		InstrumentOpenTracing,
@@ -169,6 +169,7 @@ func NewServer(api *api.API, corsString string) *Server {
 }
 
 func (s *Server) ListenAndServe(addr string) error {
+	s.listenAddr = addr
 	return http.ListenAndServe(addr, s)
 }
 
@@ -179,15 +180,23 @@ func (s *Server) ListenAndServe(addr string) error {
 type Server struct {
 	http.Handler
 	api        *api.API
+	listenAddr string
 }
 
 func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()))
+	log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()), "uri", r.RequestURI)
 	if r.Header.Get("Accept") == "application/x-tar" {
 		uri := GetURI(r.Context())
-		reader, err := s.api.GetDirectoryTar(r.Context(), uri)
+		_, credentials, _ := r.BasicAuth()
+		reader, err := s.api.GetDirectoryTar(r.Context(), s.api.Decryptor(r.Context(), credentials), uri)
 		if err != nil {
+			if isDecryptError(err) {
+				w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", uri.Address().String()))
+				RespondError(w, r, err.Error(), http.StatusUnauthorized)
+				return
+			}
 			RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
+			return
 		}
 		defer reader.Close()
 
@@ -287,7 +296,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
 
 	var addr storage.Address
 	if uri.Addr != "" && uri.Addr != "encrypt" {
-		addr, err = s.api.Resolve(r.Context(), uri)
+		addr, err = s.api.Resolve(r.Context(), uri.Addr)
 		if err != nil {
 			postFilesFail.Inc(1)
 			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError)
@@ -563,7 +572,7 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) {
 	// resolve the content key.
 	manifestAddr := uri.Address()
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(r.Context(), uri)
+		manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr)
 		if err != nil {
 			getFail.Inc(1)
 			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
@@ -682,62 +691,21 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) {
 	uri := GetURI(r.Context())
 	log.Debug("handle.get", "ruid", ruid, "uri", uri)
 	getCount.Inc(1)
+	_, pass, _ := r.BasicAuth()
 
-	var err error
+	addr, err := s.api.ResolveURI(r.Context(), uri, pass)
-	addr := uri.Address()
-	if addr == nil {
-		addr, err = s.api.Resolve(r.Context(), uri)
 	if err != nil {
 		getFail.Inc(1)
 		RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
 		return
 	}
-	} else {
 	w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
-	}
 
 	log.Debug("handle.get: resolved", "ruid", ruid, "key", addr)
 
 	// if path is set, interpret <key> as a manifest and return the
 	// raw entry at the given path
-	if uri.Path != "" {
-		walker, err := s.api.NewManifestWalker(r.Context(), addr, nil)
-		if err != nil {
-			getFail.Inc(1)
-			RespondError(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
-			return
-		}
-		var entry *api.ManifestEntry
-		walker.Walk(func(e *api.ManifestEntry) error {
-			// if the entry matches the path, set entry and stop
-			// the walk
-			if e.Path == uri.Path {
-				entry = e
-				// return an error to cancel the walk
-				return errors.New("found")
-			}
 
-			// ignore non-manifest files
-			if e.ContentType != api.ManifestType {
-				return nil
-			}
-
-			// if the manifest's path is a prefix of the
-			// requested path, recurse into it by returning
-			// nil and continuing the walk
-			if strings.HasPrefix(uri.Path, e.Path) {
-				return nil
-			}
-
-			return api.ErrSkipManifest
-		})
-		if entry == nil {
-			getFail.Inc(1)
-			RespondError(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound)
-			return
-		}
-		addr = storage.Address(common.Hex2Bytes(entry.Hash))
-	}
 	etag := common.Bytes2Hex(addr)
 	noneMatchEtag := r.Header.Get("If-None-Match")
 	w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key.
@@ -781,6 +749,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) {
 func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
 	ruid := GetRUID(r.Context())
 	uri := GetURI(r.Context())
+	_, credentials, _ := r.BasicAuth()
 	log.Debug("handle.get.list", "ruid", ruid, "uri", uri)
 	getListCount.Inc(1)
 
@@ -790,7 +759,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	addr, err := s.api.Resolve(r.Context(), uri)
+	addr, err := s.api.Resolve(r.Context(), uri.Addr)
 	if err != nil {
 		getListFail.Inc(1)
 		RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
@@ -798,9 +767,14 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
 	}
 	log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr)
 
-	list, err := s.api.GetManifestList(r.Context(), addr, uri.Path)
+	list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), addr, uri.Path)
 	if err != nil {
 		getListFail.Inc(1)
+		if isDecryptError(err) {
+			w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", addr.String()))
+			RespondError(w, r, err.Error(), http.StatusUnauthorized)
+			return
+		}
 		RespondError(w, r, err.Error(), http.StatusInternalServerError)
 		return
 	}
@@ -833,7 +807,8 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
 func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
 	ruid := GetRUID(r.Context())
 	uri := GetURI(r.Context())
-	log.Debug("handle.get.file", "ruid", ruid)
+	_, credentials, _ := r.BasicAuth()
+	log.Debug("handle.get.file", "ruid", ruid, "uri", r.RequestURI)
 	getFileCount.Inc(1)
 
 	// ensure the root path has a trailing slash so that relative URLs work
@@ -845,7 +820,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
 	manifestAddr := uri.Address()
 
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(r.Context(), uri)
+		manifestAddr, err = s.api.ResolveURI(r.Context(), uri, credentials)
 		if err != nil {
 			getFileFail.Inc(1)
 			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
@@ -856,7 +831,8 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
 	}
 
 	log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr)
-	reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, uri.Path)
+
+	reader, contentType, status, contentKey, err := s.api.Get(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path)
 
 	etag := common.Bytes2Hex(contentKey)
 	noneMatchEtag := r.Header.Get("If-None-Match")
@@ -869,6 +845,12 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err != nil {
+		if isDecryptError(err) {
+			w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr))
+			RespondError(w, r, err.Error(), http.StatusUnauthorized)
+			return
+		}
+
 		switch status {
 		case http.StatusNotFound:
 			getFileNotFound.Inc(1)
@@ -883,9 +865,14 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
 	//the request results in ambiguous files
 	//e.g. /read with readme.md and readinglist.txt available in manifest
 	if status == http.StatusMultipleChoices {
-		list, err := s.api.GetManifestList(r.Context(), manifestAddr, uri.Path)
+		list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path)
 		if err != nil {
 			getFileFail.Inc(1)
+			if isDecryptError(err) {
+				w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr))
+				RespondError(w, r, err.Error(), http.StatusUnauthorized)
+				return
+			}
 			RespondError(w, r, err.Error(), http.StatusInternalServerError)
 			return
 		}
@@ -951,3 +938,7 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
 	lrw.statusCode = code
 	lrw.ResponseWriter.WriteHeader(code)
 }
+
+func isDecryptError(err error) bool {
+	return strings.Contains(err.Error(), api.ErrDecrypt.Error())
+}

swarm/api/manifest.go

@@ -53,6 +53,7 @@ type ManifestEntry struct {
 	Size        int64        `json:"size,omitempty"`
 	ModTime     time.Time    `json:"mod_time,omitempty"`
 	Status      int          `json:"status,omitempty"`
+	Access      *AccessEntry `json:"access,omitempty"`
 }
 
 // ManifestList represents the result of listing files in a manifest
@@ -98,7 +99,7 @@ type ManifestWriter struct {
 }
 
 func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWriter, error) {
-	trie, err := loadManifest(ctx, a.fileStore, addr, quitC)
+	trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt)
 	if err != nil {
 		return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
 	}
@@ -141,8 +142,8 @@ type ManifestWalker struct {
 	quitC chan bool
 }
 
-func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWalker, error) {
+func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, decrypt DecryptFunc, quitC chan bool) (*ManifestWalker, error) {
-	trie, err := loadManifest(ctx, a.fileStore, addr, quitC)
+	trie, err := loadManifest(ctx, a.fileStore, addr, quitC, decrypt)
 	if err != nil {
 		return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
 	}
@@ -194,6 +195,7 @@ type manifestTrie struct {
 	entries   [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry
 	ref       storage.Address         // if ref != nil, it is stored
 	encrypted bool
+	decrypt   DecryptFunc
 }
 
 func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
@@ -209,15 +211,15 @@ type manifestTrieEntry struct {
 	subtrie *manifestTrie
 }
 
-func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
+func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
 	log.Trace("manifest lookup", "key", hash)
 	// retrieve manifest via FileStore
 	manifestReader, isEncrypted := fileStore.Retrieve(ctx, hash)
 	log.Trace("reader retrieved", "key", hash)
-	return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC)
+	return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC, decrypt)
 }
 
-func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
+func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
 
 	// TODO check size for oversized manifests
 	size, err := mr.Size(mr.Context(), quitC)
@@ -258,26 +260,41 @@ func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore
 	trie = &manifestTrie{
 		fileStore: fileStore,
 		encrypted: isEncrypted,
+		decrypt:   decrypt,
 	}
 	for _, entry := range man.Entries {
-		trie.addEntry(entry, quitC)
+		err = trie.addEntry(entry, quitC)
+		if err != nil {
+			return
+		}
 	}
 	return
 }
 
-func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
+func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) error {
 	mt.ref = nil // trie modified, hash needs to be re-calculated on demand
 
+	if entry.ManifestEntry.Access != nil {
+		if mt.decrypt == nil {
+			return errors.New("dont have decryptor")
+		}
+
+		err := mt.decrypt(&entry.ManifestEntry)
+		if err != nil {
+			return err
+		}
+	}
+
 	if len(entry.Path) == 0 {
 		mt.entries[256] = entry
-		return
+		return nil
 	}
 
 	b := entry.Path[0]
 	oldentry := mt.entries[b]
 	if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) {
 		mt.entries[b] = entry
-		return
+		return nil
 	}
 
 	cpl := 0
@@ -287,12 +304,12 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 
 	if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
 		if mt.loadSubTrie(oldentry, quitC) != nil {
-			return
+			return nil
 		}
 		entry.Path = entry.Path[cpl:]
 		oldentry.subtrie.addEntry(entry, quitC)
 		oldentry.Hash = ""
-		return
+		return nil
 	}
 
 	commonPrefix := entry.Path[:cpl]
@@ -310,6 +327,7 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 		Path:        commonPrefix,
 		ContentType: ManifestType,
 	}, subtrie)
+	return nil
 }
 
 func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
@@ -398,9 +416,20 @@ func (mt *manifestTrie) recalcAndStore() error {
 }
 
 func (mt *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
+	if entry.ManifestEntry.Access != nil {
+		if mt.decrypt == nil {
+			return errors.New("dont have decryptor")
+		}
+
+		err := mt.decrypt(&entry.ManifestEntry)
+		if err != nil {
+			return err
+		}
+	}
+
 	if entry.subtrie == nil {
 		hash := common.Hex2Bytes(entry.Hash)
-		entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC)
+		entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC, mt.decrypt)
 		entry.Hash = "" // might not match, should be recalculated
 	}
 	return

swarm/api/manifest_test.go

@@ -44,7 +44,7 @@ func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...stri
 	quitC := make(chan bool)
 	fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
 	ref := make([]byte, fileStore.HashSize())
-	trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC)
+	trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC, NOOPDecrypt)
 	if err != nil {
 		t.Errorf("unexpected error making manifest: %v", err)
 	}
@@ -101,7 +101,7 @@ func TestExactMatch(t *testing.T) {
 	mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map")
 	fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
 	ref := make([]byte, fileStore.HashSize())
-	trie, err := readManifest(mf, ref, fileStore, false, quitC)
+	trie, err := readManifest(mf, ref, fileStore, false, quitC, nil)
 	if err != nil {
 		t.Errorf("unexpected error making manifest: %v", err)
 	}
@@ -134,7 +134,7 @@ func TestAddFileWithManifestPath(t *testing.T) {
 	}
 	fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
 	ref := make([]byte, fileStore.HashSize())
-	trie, err := readManifest(reader, ref, fileStore, false, nil)
+	trie, err := readManifest(reader, ref, fileStore, false, nil, NOOPDecrypt)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -161,7 +161,7 @@ func TestReadManifestOverSizeLimit(t *testing.T) {
 	reader := &storage.LazyTestSectionReader{
 		SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))),
 	}
-	_, err := readManifest(reader, storage.Address{}, nil, false, nil)
+	_, err := readManifest(reader, storage.Address{}, nil, false, nil, NOOPDecrypt)
 	if err == nil {
 		t.Fatal("got no error from readManifest")
 	}

swarm/api/storage.go

@@ -63,11 +63,11 @@ func (s *Storage) Get(ctx context.Context, bzzpath string) (*Response, error) {
 	if err != nil {
 		return nil, err
 	}
-	addr, err := s.api.Resolve(ctx, uri)
+	addr, err := s.api.Resolve(ctx, uri.Addr)
 	if err != nil {
 		return nil, err
 	}
-	reader, mimeType, status, _, err := s.api.Get(ctx, addr, uri.Path)
+	reader, mimeType, status, _, err := s.api.Get(ctx, nil, addr, uri.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +93,7 @@ func (s *Storage) Modify(ctx context.Context, rootHash, path, contentHash, conte
 	if err != nil {
 		return "", err
 	}
-	addr, err := s.api.Resolve(ctx, uri)
+	addr, err := s.api.Resolve(ctx, uri.Addr)
 	if err != nil {
 		return "", err
 	}

swarm/api/uri.go

@@ -53,6 +53,19 @@ type URI struct {
 	Path string
 }
 
+func (u *URI) MarshalJSON() (out []byte, err error) {
+	return []byte(`"` + u.String() + `"`), nil
+}
+
+func (u *URI) UnmarshalJSON(value []byte) error {
+	uri, err := Parse(string(value))
+	if err != nil {
+		return err
+	}
+	*u = *uri
+	return nil
+}
+
 // Parse parses rawuri into a URI struct, where rawuri is expected to have one
 // of the following formats:
 //

swarm/fuse/swarmfs_test.go

@@ -1650,7 +1650,7 @@ func TestFUSE(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	ta := &testAPI{api: api.NewAPI(fileStore, nil, nil)}
+	ta := &testAPI{api: api.NewAPI(fileStore, nil, nil, nil)}
 
 	//run a short suite of tests
 	//approx time: 28s

swarm/network_test.go

@@ -445,7 +445,7 @@ func retrieve(
 
 				log.Debug("api get: check file", "node", id.String(), "key", f.addr.String(), "total files found", atomic.LoadUint64(totalFoundCount))
 
-				r, _, _, _, err := swarm.api.Get(context.TODO(), f.addr, "/")
+				r, _, _, _, err := swarm.api.Get(context.TODO(), api.NOOPDecrypt, f.addr, "/")
 				if err != nil {
 					errc <- fmt.Errorf("api get: node %s, key %s, kademlia %s: %v", id, f.addr, swarm.bzz.Hive, err)
 					return

swarm/sctx/sctx.go

@@ -1,7 +1,22 @@
 package sctx
 
+import "context"
+
 type ContextKey int
 
 const (
 	HTTPRequestIDKey ContextKey = iota
+	requestHostKey
 )
+
+func SetHost(ctx context.Context, domain string) context.Context {
+	return context.WithValue(ctx, requestHostKey, domain)
+}
+
+func GetHost(ctx context.Context) string {
+	v, ok := ctx.Value(requestHostKey).(string)
+	if ok {
+		return v
+	}
+	return ""
+}

swarm/swarm.go

@@ -85,14 +85,12 @@ type Swarm struct {
 type SwarmAPI struct {
 	Api     *api.API
 	Backend chequebook.Backend
-	PrvKey  *ecdsa.PrivateKey
 }
 
 func (self *Swarm) API() *SwarmAPI {
 	return &SwarmAPI{
 		Api:     self.api,
 		Backend: self.backend,
-		PrvKey:  self.privateKey,
 	}
 }
 
@@ -217,7 +215,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
 		pss.SetHandshakeController(self.ps, pss.NewHandshakeParams())
 	}
 
-	self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler)
+	self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler, self.privateKey)
 	// Manifests for Smart Hosting
 	log.Debug(fmt.Sprintf("-> Web3 virtual server API"))
 

swarm/testutil/http.go

@@ -77,7 +77,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer) *Tes
 		t.Fatal(err)
 	}
 
-	a := api.NewAPI(fileStore, nil, rh.Handler)
+	a := api.NewAPI(fileStore, nil, rh.Handler, nil)
 	srv := httptest.NewServer(serverFunc(a))
 	return &TestSwarmServer{
 		Server:            srv,