From b319f027a0be232a9cb307336b0349b36737c7f1 Mon Sep 17 00:00:00 2001 From: Lewis Marshall Date: Tue, 4 Apr 2017 23:20:07 +0100 Subject: [PATCH] cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742) This adds a swarm ls command which lists files and directories stored in a manifest. Rather than listing all files, it uses "directory prefixes" in case there are a lot of files in a manifest but you just want to traverse it. This also includes some refactoring to the tests and the introduction of a swarm/api/client package to make things easier to test. --- cmd/swarm/list.go | 58 +++++++ cmd/swarm/main.go | 9 + cmd/swarm/manifest.go | 51 +++--- cmd/swarm/upload.go | 163 +---------------- swarm/api/client/client.go | 298 ++++++++++++++++++++++++++++++++ swarm/api/client/client_test.go | 105 +++++++++++ swarm/api/http/server.go | 42 +++-- swarm/api/http/server_test.go | 52 +----- swarm/swarm.go | 5 +- swarm/testutil/http.go | 56 ++++++ 10 files changed, 594 insertions(+), 245 deletions(-) create mode 100644 cmd/swarm/list.go create mode 100644 swarm/api/client/client.go create mode 100644 swarm/api/client/client_test.go create mode 100644 swarm/testutil/http.go diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go new file mode 100644 index 0000000000..3a68fef030 --- /dev/null +++ b/cmd/swarm/list.go @@ -0,0 +1,58 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "strings" + "text/tabwriter" + + "github.com/ethereum/go-ethereum/cmd/utils" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" + "gopkg.in/urfave/cli.v1" +) + +func list(ctx *cli.Context) { + args := ctx.Args() + + if len(args) < 1 { + utils.Fatalf("Please supply a manifest reference as the first argument") + } else if len(args) > 2 { + utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'") + } + manifest := args[0] + + var prefix string + if len(args) == 2 { + prefix = args[1] + } + + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) + entries, err := client.ManifestFileList(manifest, prefix) + if err != nil { + utils.Fatalf("Failed to generate file and directory list: %s", err) + } + + w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0) + defer w.Flush() + fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH") + for _, entry := range entries { + fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path) + } +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 3c82497e5d..731c300f82 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -145,6 +145,15 @@ The output of this command is supposed to be machine-readable. ArgsUsage: " ", Description: ` "upload a file or directory to swarm using the HTTP API and prints the root hash", +`, + }, + { + Action: list, + Name: "ls", + Usage: "list files and directories contained in a manifest", + ArgsUsage: " []", + Description: ` +Lists files and directories contained in a manifest. `, }, { diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index 2b6b023135..698b8ddb88 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/cmd/utils" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" "gopkg.in/urfave/cli.v1" ) @@ -41,7 +42,7 @@ func add(ctx *cli.Context) { ctype string wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot manifest + mroot swarm.Manifest ) if len(args) > 3 { @@ -75,7 +76,7 @@ func update(ctx *cli.Context) { ctype string wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot manifest + mroot swarm.Manifest ) if len(args) > 3 { ctype = args[3] @@ -105,7 +106,7 @@ func remove(ctx *cli.Context) { path = args[1] wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot manifest + mroot swarm.Manifest ) newManifest := removeEntryFromManifest(ctx, mhash, path) @@ -123,21 +124,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin var ( bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = &client{api: bzzapi} - longestPathEntry = manifestEntry{ + client = swarm.NewClient(bzzapi) + longestPathEntry = swarm.ManifestEntry{ Path: "", Hash: "", ContentType: "", } ) - mroot, err := client.downloadManifest(mhash) + mroot, err := client.DownloadManifest(mhash) if err != nil { utils.Fatalf("Manifest download failed: %v", err) } //TODO: check if the "hash" to add is valid and present in swarm - _, err = client.downloadManifest(hash) + _, err = client.DownloadManifest(hash) if err != nil { utils.Fatalf("Hash to add is not present: %v", err) } @@ -162,7 +163,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) // Replace the hash for parent Manifests - newMRoot := manifest{} + newMRoot := swarm.Manifest{} for _, entry := range mroot.Entries { if longestPathEntry.Path == entry.Path { entry.Hash = newHash @@ -172,7 +173,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin mroot = newMRoot } else { // Add the entry in the leaf Manifest - newEntry := manifestEntry{ + newEntry := swarm.ManifestEntry{ Path: path, Hash: hash, ContentType: ctype, @@ -180,7 +181,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin mroot.Entries = append(mroot.Entries, newEntry) } - newManifestHash, err := client.uploadManifest(mroot) + newManifestHash, err := client.UploadManifest(mroot) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } @@ -192,20 +193,20 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st var ( bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = &client{api: bzzapi} - newEntry = manifestEntry{ + client = swarm.NewClient(bzzapi) + newEntry = swarm.ManifestEntry{ Path: "", Hash: "", ContentType: "", } - longestPathEntry = manifestEntry{ + longestPathEntry = swarm.ManifestEntry{ Path: "", Hash: "", ContentType: "", } ) - mroot, err := client.downloadManifest(mhash) + mroot, err := client.DownloadManifest(mhash) if err != nil { utils.Fatalf("Manifest download failed: %v", err) } @@ -236,7 +237,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) // Replace the hash for parent Manifests - newMRoot := manifest{} + newMRoot := swarm.Manifest{} for _, entry := range mroot.Entries { if longestPathEntry.Path == entry.Path { entry.Hash = newHash @@ -249,10 +250,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st if newEntry.Path != "" { // Replace the hash for leaf Manifest - newMRoot := manifest{} + newMRoot := swarm.Manifest{} for _, entry := range mroot.Entries { if newEntry.Path == entry.Path { - myEntry := manifestEntry{ + myEntry := swarm.ManifestEntry{ Path: entry.Path, Hash: hash, ContentType: ctype, @@ -265,7 +266,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st mroot = newMRoot } - newManifestHash, err := client.uploadManifest(mroot) + newManifestHash, err := client.UploadManifest(mroot) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } @@ -276,20 +277,20 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { var ( bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = &client{api: bzzapi} - entryToRemove = manifestEntry{ + client = swarm.NewClient(bzzapi) + entryToRemove = swarm.ManifestEntry{ Path: "", Hash: "", ContentType: "", } - longestPathEntry = manifestEntry{ + longestPathEntry = swarm.ManifestEntry{ Path: "", Hash: "", ContentType: "", } ) - mroot, err := client.downloadManifest(mhash) + mroot, err := client.DownloadManifest(mhash) if err != nil { utils.Fatalf("Manifest download failed: %v", err) } @@ -318,7 +319,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath) // Replace the hash for parent Manifests - newMRoot := manifest{} + newMRoot := swarm.Manifest{} for _, entry := range mroot.Entries { if longestPathEntry.Path == entry.Path { entry.Hash = newHash @@ -330,7 +331,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { if entryToRemove.Path != "" { // remove the entry in this Manifest - newMRoot := manifest{} + newMRoot := swarm.Manifest{} for _, entry := range mroot.Entries { if entryToRemove.Path != entry.Path { newMRoot.Entries = append(newMRoot.Entries, entry) @@ -339,7 +340,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { mroot = newMRoot } - newManifestHash, err := client.uploadManifest(mroot) + newManifestHash, err := client.UploadManifest(mroot) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 7b49617787..696b907d22 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -18,21 +18,15 @@ package main import ( - "bytes" "encoding/json" "fmt" - "io" - "io/ioutil" - "mime" - "net/http" "os" "os/user" "path" - "path/filepath" "strings" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/log" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" "gopkg.in/urfave/cli.v1" ) @@ -50,7 +44,7 @@ func upload(ctx *cli.Context) { var ( file = args[0] - client = &client{api: bzzapi} + client = swarm.NewClient(bzzapi) ) fi, err := os.Stat(expandPath(file)) if err != nil { @@ -63,25 +57,25 @@ func upload(ctx *cli.Context) { if !wantManifest { utils.Fatalf("Manifest is required for directory uploads") } - mhash, err := client.uploadDirectory(file, defaultPath) + mhash, err := client.UploadDirectory(file, defaultPath) if err != nil { utils.Fatalf("Failed to upload directory: %v", err) } fmt.Println(mhash) return } - entry, err := client.uploadFile(file, fi) + entry, err := client.UploadFile(file, fi) if err != nil { utils.Fatalf("Upload failed: %v", err) } - mroot := manifest{[]manifestEntry{entry}} + mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}} if !wantManifest { // Print the manifest. This is the only output to stdout. mrootJSON, _ := json.MarshalIndent(mroot, "", " ") fmt.Println(string(mrootJSON)) return } - hash, err := client.uploadManifest(mroot) + hash, err := client.UploadManifest(mroot) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } @@ -111,148 +105,3 @@ func homeDir() string { } return "" } - -// client wraps interaction with the swarm HTTP gateway. -type client struct { - api string -} - -// manifest is the JSON representation of a swarm manifest. -type manifestEntry struct { - Hash string `json:"hash,omitempty"` - ContentType string `json:"contentType,omitempty"` - Path string `json:"path,omitempty"` -} - -// manifest is the JSON representation of a swarm manifest. -type manifest struct { - Entries []manifestEntry `json:"entries,omitempty"` -} - -func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) { - mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}")))) - if err != nil { - return "", fmt.Errorf("failed to upload empty manifest") - } - if len(defaultPath) > 0 { - fi, err := os.Stat(defaultPath) - if err != nil { - return "", err - } - mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi) - if err != nil { - return "", err - } - } - prefix := filepath.ToSlash(filepath.Clean(dir)) + "/" - err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { - if err != nil || fi.IsDir() { - return err - } - if !strings.HasPrefix(path, dir) { - return fmt.Errorf("path %s outside directory %s", path, dir) - } - uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix) - mhash, err = c.uploadToManifest(mhash, uripath, path, fi) - return err - }) - return mhash, err -} - -func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) { - hash, err := c.uploadFileContent(file, fi) - m := manifestEntry{ - Hash: hash, - ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())), - } - return m, err -} - -func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) { - fd, err := os.Open(file) - if err != nil { - return "", err - } - defer fd.Close() - log.Info("Uploading swarm content", "file", file, "bytes", fi.Size()) - return c.postRaw("application/octet-stream", fi.Size(), fd) -} - -func (c *client) uploadManifest(m manifest) (string, error) { - jsm, err := json.Marshal(m) - if err != nil { - panic(err) - } - log.Info("Uploading swarm manifest") - return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm))) -} - -func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) { - fd, err := os.Open(fpath) - if err != nil { - return "", err - } - defer fd.Close() - log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path) - req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd) - if err != nil { - return "", err - } - req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name()))) - req.ContentLength = fi.Size() - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - return "", fmt.Errorf("bad status: %s", resp.Status) - } - content, err := ioutil.ReadAll(resp.Body) - return string(content), err -} - -func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) { - req, err := http.NewRequest("POST", c.api+"/bzzr:/", body) - if err != nil { - return "", err - } - req.Header.Set("content-type", mimetype) - req.ContentLength = size - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - return "", fmt.Errorf("bad status: %s", resp.Status) - } - content, err := ioutil.ReadAll(resp.Body) - return string(content), err -} - -func (c *client) downloadManifest(mhash string) (manifest, error) { - - mroot := manifest{} - req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil) - if err != nil { - return mroot, err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return mroot, err - } - defer resp.Body.Close() - - if resp.StatusCode >= 400 { - return mroot, fmt.Errorf("bad status: %s", resp.Status) - - } - content, err := ioutil.ReadAll(resp.Body) - - err = json.Unmarshal(content, &mroot) - if err != nil { - return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err) - } - return mroot, err -} diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go new file mode 100644 index 0000000000..15e44f35d7 --- /dev/null +++ b/swarm/api/client/client.go @@ -0,0 +1,298 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "mime" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/ethereum/go-ethereum/log" +) + +var ( + DefaultGateway = "http://localhost:8500" + DefaultClient = NewClient(DefaultGateway) +) + +// Manifest represents a swarm manifest. +type Manifest struct { + Entries []ManifestEntry `json:"entries,omitempty"` +} + +// ManifestEntry represents an entry in a swarm manifest. +type ManifestEntry struct { + Hash string `json:"hash,omitempty"` + ContentType string `json:"contentType,omitempty"` + Path string `json:"path,omitempty"` +} + +func NewClient(gateway string) *Client { + return &Client{ + Gateway: gateway, + } +} + +// Client wraps interaction with a swarm HTTP gateway. +type Client struct { + Gateway string +} + +func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) { + mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}")))) + if err != nil { + return "", fmt.Errorf("failed to upload empty manifest") + } + if len(defaultPath) > 0 { + fi, err := os.Stat(defaultPath) + if err != nil { + return "", err + } + mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi) + if err != nil { + return "", err + } + } + prefix := filepath.ToSlash(filepath.Clean(dir)) + "/" + err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + if err != nil || fi.IsDir() { + return err + } + if !strings.HasPrefix(path, dir) { + return fmt.Errorf("path %s outside directory %s", path, dir) + } + uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix) + mhash, err = c.uploadToManifest(mhash, uripath, path, fi) + return err + }) + return mhash, err +} + +func (c *Client) UploadFile(file string, fi os.FileInfo) (ManifestEntry, error) { + hash, err := c.uploadFileContent(file, fi) + m := ManifestEntry{ + Hash: hash, + ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())), + } + return m, err +} + +func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) { + fd, err := os.Open(file) + if err != nil { + return "", err + } + defer fd.Close() + log.Info("Uploading swarm content", "file", file, "bytes", fi.Size()) + return c.postRaw("application/octet-stream", fi.Size(), fd) +} + +func (c *Client) UploadManifest(m Manifest) (string, error) { + jsm, err := json.Marshal(m) + if err != nil { + panic(err) + } + log.Info("Uploading swarm manifest") + return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm))) +} + +func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) { + fd, err := os.Open(fpath) + if err != nil { + return "", err + } + defer fd.Close() + log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path) + req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd) + if err != nil { + return "", err + } + req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name()))) + req.ContentLength = fi.Size() + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return "", fmt.Errorf("bad status: %s", resp.Status) + } + content, err := ioutil.ReadAll(resp.Body) + return string(content), err +} + +func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) { + req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body) + if err != nil { + return "", err + } + req.Header.Set("content-type", mimetype) + req.ContentLength = size + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return "", fmt.Errorf("bad status: %s", resp.Status) + } + content, err := ioutil.ReadAll(resp.Body) + return string(content), err +} + +func (c *Client) DownloadManifest(mhash string) (Manifest, error) { + + mroot := Manifest{} + req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil) + if err != nil { + return mroot, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return mroot, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + return mroot, fmt.Errorf("bad status: %s", resp.Status) + + } + content, err := ioutil.ReadAll(resp.Body) + + err = json.Unmarshal(content, &mroot) + if err != nil { + return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err) + } + return mroot, err +} + +// ManifestFileList downloads the manifest with the given hash and generates a +// list of files and directory prefixes which have the specified prefix. +// +// For example, if the manifest represents the following directory structure: +// +// file1.txt +// file2.txt +// dir1/file3.txt +// dir1/dir2/file4.txt +// +// Then: +// +// - a prefix of "" would return [dir1/, file1.txt, file2.txt] +// - a prefix of "file" would return [file1.txt, file2.txt] +// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt] +func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) { + manifest, err := c.DownloadManifest(hash) + if err != nil { + return nil, err + } + + // handleFile handles a manifest entry which is a direct reference to a + // file (i.e. it is not a swarm manifest) + handleFile := func(entry ManifestEntry) { + // ignore the file if it doesn't have the specified prefix + if !strings.HasPrefix(entry.Path, prefix) { + return + } + // if the path after the prefix contains a directory separator, + // add a directory prefix to the entries, otherwise add the + // file + suffix := strings.TrimPrefix(entry.Path, prefix) + if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 { + entries = append(entries, ManifestEntry{ + Path: prefix + suffix[:sepIndex+1], + ContentType: "DIR", + }) + } else { + if entry.Path == "" { + entry.Path = "/" + } + entries = append(entries, entry) + } + } + + // handleManifest handles a manifest entry which is a reference to + // another swarm manifest. + handleManifest := func(entry ManifestEntry) error { + // if the manifest's path is a prefix of the specified prefix + // then just recurse into the manifest by stripping its path + // from the prefix + if strings.HasPrefix(prefix, entry.Path) { + subPrefix := strings.TrimPrefix(prefix, entry.Path) + subEntries, err := c.ManifestFileList(entry.Hash, subPrefix) + if err != nil { + return err + } + // prefix the manifest's path to the sub entries and + // add them to the returned entries + for i, subEntry := range subEntries { + subEntry.Path = entry.Path + subEntry.Path + subEntries[i] = subEntry + } + entries = append(entries, subEntries...) + return nil + } + + // if the manifest's path has the specified prefix, then if the + // path after the prefix contains a directory separator, add a + // directory prefix to the entries, otherwise recurse into the + // manifest + if strings.HasPrefix(entry.Path, prefix) { + suffix := strings.TrimPrefix(entry.Path, prefix) + sepIndex := strings.Index(suffix, "/") + if sepIndex > -1 { + entries = append(entries, ManifestEntry{ + Path: prefix + suffix[:sepIndex+1], + ContentType: "DIR", + }) + return nil + } + subEntries, err := c.ManifestFileList(entry.Hash, "") + if err != nil { + return err + } + // prefix the manifest's path to the sub entries and + // add them to the returned entries + for i, subEntry := range subEntries { + subEntry.Path = entry.Path + subEntry.Path + subEntries[i] = subEntry + } + entries = append(entries, subEntries...) + return nil + } + return nil + } + + for _, entry := range manifest.Entries { + if entry.ContentType == "application/bzz-manifest+json" { + if err := handleManifest(entry); err != nil { + return nil, err + } + } else { + handleFile(entry) + } + } + + return +} diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go new file mode 100644 index 0000000000..1354744750 --- /dev/null +++ b/swarm/api/client/client_test.go @@ -0,0 +1,105 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package client + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/swarm/testutil" +) + +func TestClientManifestFileList(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + dir, err := ioutil.TempDir("", "swarm-client-test") + if err != nil { + t.Fatal(err) + } + files := []string{ + "file1.txt", + "file2.txt", + "dir1/file3.txt", + "dir1/file4.txt", + "dir2/file5.txt", + "dir2/dir3/file6.txt", + "dir2/dir4/file7.txt", + "dir2/dir4/file8.txt", + } + for _, file := range files { + path := filepath.Join(dir, file) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatalf("error creating dir for %s: %s", path, err) + } + if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil { + t.Fatalf("error writing file %s: %s", path, err) + } + } + + client := NewClient(srv.URL) + + hash, err := client.UploadDirectory(dir, "") + if err != nil { + t.Fatalf("error uploading directory: %s", err) + } + + ls := func(prefix string) []string { + entries, err := client.ManifestFileList(hash, prefix) + if err != nil { + t.Fatal(err) + } + paths := make([]string, len(entries)) + for i, entry := range entries { + paths[i] = entry.Path + } + sort.Strings(paths) + return paths + } + + tests := map[string][]string{ + "": []string{"dir1/", "dir2/", "file1.txt", "file2.txt"}, + "file": []string{"file1.txt", "file2.txt"}, + "file1": []string{"file1.txt"}, + "file2.txt": []string{"file2.txt"}, + "file12": []string{}, + "dir": []string{"dir1/", "dir2/"}, + "dir1": []string{"dir1/"}, + "dir1/": []string{"dir1/file3.txt", "dir1/file4.txt"}, + "dir1/file": []string{"dir1/file3.txt", "dir1/file4.txt"}, + "dir1/file3.txt": []string{"dir1/file3.txt"}, + "dir1/file34": []string{}, + "dir2/": []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"}, + "dir2/file": []string{"dir2/file5.txt"}, + "dir2/dir": []string{"dir2/dir3/", "dir2/dir4/"}, + "dir2/dir3/": []string{"dir2/dir3/file6.txt"}, + "dir2/dir4/": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, + "dir2/dir4/file": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, + "dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"}, + "dir2/dir4/file78": []string{}, + } + for prefix, expected := range tests { + actual := ls(prefix) + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual) + } + } +} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index ae113750ff..44e2c203a4 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -56,9 +56,9 @@ type sequentialReader struct { lock sync.Mutex } -// Server is the basic configuration needs for the HTTP server and also +// ServerConfig is the basic configuration needed for the HTTP server and also // includes CORS settings. -type Server struct { +type ServerConfig struct { Addr string CorsString string } @@ -69,13 +69,9 @@ type Server struct { // https://github.com/atom/electron/blob/master/docs/api/protocol.md // starts up http server -func StartHttpServer(api *api.Api, server *Server) { - serveMux := http.NewServeMux() - serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - handler(w, r, api) - }) +func StartHttpServer(api *api.Api, config *ServerConfig) { var allowedOrigins []string - for _, domain := range strings.Split(server.CorsString, ",") { + for _, domain := range strings.Split(config.CorsString, ",") { allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain)) } c := cors.New(cors.Options{ @@ -84,13 +80,21 @@ func StartHttpServer(api *api.Api, server *Server) { MaxAge: 600, AllowedHeaders: []string{"*"}, }) - hdlr := c.Handler(serveMux) + hdlr := c.Handler(NewServer(api)) - go http.ListenAndServe(server.Addr, hdlr) - log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", server.Addr)) + go http.ListenAndServe(config.Addr, hdlr) + log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", config.Addr)) } -func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { +func NewServer(api *api.Api) *Server { + return &Server{api} +} + +type Server struct { + api *api.Api +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { requestURL := r.URL // This is wrong // if requestURL.Host == "" { @@ -135,7 +139,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest) return } - key, err := a.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil) + key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil) if err == nil { log.Debug(fmt.Sprintf("Content for %v stored", key.Log())) } else { @@ -160,7 +164,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { mime := r.Header.Get("Content-Type") // TODO proper root hash separation log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime)) - newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver) + newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver) if err == nil { log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey)) w.Header().Set("Content-Type", "text/plain") @@ -178,7 +182,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { } else { path = api.RegularSlashes(path) log.Debug(fmt.Sprintf("Delete '%s'.", path)) - newKey, err := a.Modify(path, "", "", nameresolver) + newKey, err := s.api.Modify(path, "", "", nameresolver) if err == nil { log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey)) w.Header().Set("Content-Type", "text/plain") @@ -199,16 +203,16 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { parsedurl, _ := api.Parse(path) if parsedurl == path { - key, err := a.Resolve(parsedurl, nameresolver) + key, err := s.api.Resolve(parsedurl, nameresolver) if err != nil { log.Error(fmt.Sprintf("%v", err)) http.Error(w, err.Error(), http.StatusBadRequest) return } - reader = a.Retrieve(key) + reader = s.api.Retrieve(key) } else { var status int - readertmp, _, status, err := a.Get(path, nameresolver) + readertmp, _, status, err := s.api.Get(path, nameresolver) if err != nil { http.Error(w, err.Error(), status) return @@ -247,7 +251,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { http.Redirect(w, r, path+"/", http.StatusFound) return } - reader, mimeType, status, err := a.Get(path, nameresolver) + reader, mimeType, status, err := s.api.Get(path, nameresolver) if err != nil { if _, ok := err.(api.ErrResolve); ok { log.Debug(fmt.Sprintf("%v", err)) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 88b49b9a5e..45a867f516 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package http +package http_test import ( "bytes" @@ -22,19 +22,16 @@ import ( "net/http" "sync" "testing" - "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestBzzrGetPath(t *testing.T) { var err error - maxproxyattempts := 3 - testmanifest := []string{ `{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`, `{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`, @@ -54,61 +51,30 @@ func TestBzzrGetPath(t *testing.T) { key := [3]storage.Key{} - dir, _ := ioutil.TempDir("", "bzz-storage-test") - - storeparams := &storage.StoreParams{ - ChunkDbPath: dir, - DbCapacity: 5000000, - CacheCapacity: 5000, - Radius: 0, - } - - localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams) - if err != nil { - t.Fatal(err) - } - chunker := storage.NewTreeChunker(storage.NewChunkerParams()) - dpa := &storage.DPA{ - Chunker: chunker, - ChunkStore: localStore, - } - dpa.Start() - defer dpa.Stop() + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() wg := &sync.WaitGroup{} for i, mf := range testmanifest { reader[i] = bytes.NewReader([]byte(mf)) - key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil) + key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil) if err != nil { t.Fatal(err) } wg.Wait() } - a := api.NewApi(dpa, nil) - - /// \todo iterate port numbers up if fail - StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""}) - // how to wait for ListenAndServe to have initialized? This is pretty cruuuude - // if we fix it we don't need maxproxyattempts anymore either - time.Sleep(1000 * time.Millisecond) - for i := 0; i <= maxproxyattempts; i++ { - _, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a") - if i == maxproxyattempts { - t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err) - } else if err != nil { - time.Sleep(100 * time.Millisecond) - continue - } - break + _, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a") + if err != nil { + t.Fatalf("Failed to connect to proxy: %v", err) } for k, v := range testrequests { var resp *http.Response var respbody []byte - url := "http://127.0.0.1:8504/bzzr:/" + url := srv.URL + "/bzzr:/" if k[:] != "" { url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain" } diff --git a/swarm/swarm.go b/swarm/swarm.go index 0ce31bcad9..add28d205d 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -195,7 +195,10 @@ func (self *Swarm) Start(net *p2p.Server) error { // start swarm http proxy server if self.config.Port != "" { addr := ":" + self.config.Port - go httpapi.StartHttpServer(self.api, &httpapi.Server{Addr: addr, CorsString: self.corsString}) + go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{ + Addr: addr, + CorsString: self.corsString, + }) } log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go new file mode 100644 index 0000000000..bf98d16eb2 --- /dev/null +++ b/swarm/testutil/http.go @@ -0,0 +1,56 @@ +package testutil + +import ( + "io/ioutil" + "net/http/httptest" + "os" + "testing" + + "github.com/ethereum/go-ethereum/swarm/api" + httpapi "github.com/ethereum/go-ethereum/swarm/api/http" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +func NewTestSwarmServer(t *testing.T) *TestSwarmServer { + dir, err := ioutil.TempDir("", "swarm-storage-test") + if err != nil { + t.Fatal(err) + } + storeparams := &storage.StoreParams{ + ChunkDbPath: dir, + DbCapacity: 5000000, + CacheCapacity: 5000, + Radius: 0, + } + localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams) + if err != nil { + os.RemoveAll(dir) + t.Fatal(err) + } + chunker := storage.NewTreeChunker(storage.NewChunkerParams()) + dpa := &storage.DPA{ + Chunker: chunker, + ChunkStore: localStore, + } + dpa.Start() + a := api.NewApi(dpa, nil) + srv := httptest.NewServer(httpapi.NewServer(a)) + return &TestSwarmServer{ + Server: srv, + Dpa: dpa, + dir: dir, + } +} + +type TestSwarmServer struct { + *httptest.Server + + Dpa *storage.DPA + dir string +} + +func (t *TestSwarmServer) Close() { + t.Server.Close() + t.Dpa.Stop() + os.RemoveAll(t.dir) +}