From 1be680fdf06b94d61987c2ffc1e65bc203ca7bc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Sat, 15 Apr 2017 20:10:19 +0200 Subject: [PATCH 01/15] Add a managed HTTP(S) transport implementation This implements a "smart subtransport" which lets us implement just the I/O portion and leave the Git Smart Protocol logic in the library. --- git.go | 8 ++ http.go | 315 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ wrapper.c | 34 ++++++ 3 files changed, 357 insertions(+) create mode 100644 http.go diff --git a/git.go b/git.go index 0925e45..077d04b 100644 --- a/git.go +++ b/git.go @@ -129,6 +129,14 @@ func init() { panic("libgit2 was not built with threading support") } + if err := RegisterManagedHttp(); err != nil { + panic(err) + } + + if err := RegisterManagedHttps(); err != nil { + panic(err) + } + // This is not something we should be doing, as we may be // stomping all over someone else's setup. The user should do // this themselves or use some binding/wrapper which does it diff --git a/http.go b/http.go new file mode 100644 index 0000000..c32fc18 --- /dev/null +++ b/http.go @@ -0,0 +1,315 @@ +package git + +/* +#include +#include + +typedef struct { + git_smart_subtransport parent; + void *ptr; +} managed_smart_subtransport; + +typedef struct { + git_smart_subtransport_stream parent; + void *ptr; +} managed_smart_subtransport_stream; + +int _go_git_transport_register(const char *scheme); +int _go_git_transport_smart(git_transport **out, git_remote *owner); +void _go_git_setup_smart_subtransport(managed_smart_subtransport *t, void *ptr); +void _go_git_setup_smart_subtransport_stream(managed_smart_subtransport_stream *t, void *ptr); +*/ +import "C" + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "net/http" + "reflect" + "runtime" + //"runtime/debug" + "unsafe" +) + +type SmartService int + +const ( + SmartServiceUploadpackLs = C.GIT_SERVICE_UPLOADPACK_LS + SmartServiceUploadpack = C.GIT_SERVICE_UPLOADPACK + SmartServiceReceivepackLs = C.GIT_SERVICE_RECEIVEPACK_LS + SmartServiceReceivepack = C.GIT_SERVICE_RECEIVEPACK +) + +type SmartSubtransport interface { + Action(url string, action SmartService) (SmartSubtransportStream, error) + Close() error + Free() +} + +type SmartSubtransportStream interface { + Read(buf []byte) (int, error) + Write(buf []byte) error + Free() +} + +func RegisterManagedHttp() error { + httpStr := C.CString("http") + defer C.free(unsafe.Pointer(httpStr)) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + ret := C._go_git_transport_register(httpStr) + if ret != 0 { + return MakeGitError(ret) + } + + return nil +} + +func RegisterManagedHttps() error { + httpsStr := C.CString("https") + defer C.free(unsafe.Pointer(httpsStr)) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + ret := C._go_git_transport_register(httpsStr) + if ret != 0 { + return MakeGitError(ret) + } + + return nil +} + +type ManagedTransport struct{} + +func (self *ManagedTransport) Action(url string, action SmartService) (SmartSubtransportStream, error) { + var req *http.Request + var err error + switch action { + case SmartServiceUploadpackLs: + req, err = http.NewRequest("GET", url+"/info/refs?service=git-upload-pack", nil) + + case SmartServiceUploadpack: + req, err = http.NewRequest("POST", url+"/git-upload-pack", nil) + if err != nil { + break + } + + req.Header["Content-Type"] = []string{"application/x-git-upload-pack-request"} + + case SmartServiceReceivepackLs: + req, err = http.NewRequest("GET", url+"/info/refs?service=git-receive-pack", nil) + + case SmartServiceReceivepack: + req, err = http.NewRequest("POST", url+"/info/refs?service=git-upload-pack", nil) + if err != nil { + break + } + + req.Header["Content-Type"] = []string{"application/x-git-receive-pack-request"} + default: + err = errors.New("unknown action") + } + + if err != nil { + return nil, err + } + + req.Header["User-Agent"] = []string{"git/2.0 (git2go)"} + return newManagedHttpStream(req), nil +} + +func (self *ManagedTransport) Close() error { + return nil +} + +func (self *ManagedTransport) Free() { +} + +type ManagedHttpStream struct { + req *http.Request + resp *http.Response + postBuffer bytes.Buffer + sentRequest bool +} + +func newManagedHttpStream(req *http.Request) *ManagedHttpStream { + return &ManagedHttpStream{ + req: req, + } +} + +func (self *ManagedHttpStream) Read(buf []byte) (int, error) { + if !self.sentRequest { + if err := self.sendRequest(); err != nil { + return 0, err + } + } + + return self.resp.Body.Read(buf) +} + +func (self *ManagedHttpStream) Write(buf []byte) error { + // We write it all into a buffer and send it off when the transport asks + // us to read. + self.postBuffer.Write(buf) + return nil +} + +func (self *ManagedHttpStream) Free() { + self.resp.Body.Close() +} + +func (self *ManagedHttpStream) sendRequest() error { + self.req.Body = ioutil.NopCloser(&self.postBuffer) + resp, err := http.DefaultClient.Do(self.req) + if err != nil { + return err + } + + self.sentRequest = true + self.resp = resp + return nil +} + +func setLibgit2Error(err error) C.int { + cstr := C.CString(err.Error()) + defer C.free(unsafe.Pointer(cstr)) + C.giterr_set_str(C.GITERR_NET, cstr) + + return -1 +} + +//export httpAction +func httpAction(out **C.git_smart_subtransport_stream, t *C.git_smart_subtransport, url *C.char, action C.git_smart_service_t) C.int { + transport, err := getSmartSubtransportInterface(t) + if err != nil { + return setLibgit2Error(err) + } + + managed, err := transport.Action(C.GoString(url), SmartService(action)) + if err != nil { + return setLibgit2Error(err) + } + + stream := C.calloc(1, C.size_t(unsafe.Sizeof(C.managed_smart_subtransport_stream{}))) + managedPtr := pointerHandles.Track(managed) + C._go_git_setup_smart_subtransport_stream(stream, managedPtr) + + *out = (*C.git_smart_subtransport_stream)(stream) + return 0 +} + +//export httpClose +func httpClose(t *C.git_smart_subtransport) C.int { + transport, err := getSmartSubtransportInterface(t) + if err != nil { + return setLibgit2Error(err) + } + + if err := transport.Close(); err != nil { + return setLibgit2Error(err) + } + + return 0 +} + +//export httpFree +func httpFree(transport *C.git_smart_subtransport) { + wrapperPtr := (*C.managed_smart_subtransport)(unsafe.Pointer(transport)) + pointerHandles.Untrack(wrapperPtr.ptr) +} + +var errNoSmartSubtransport = errors.New("passed object does not implement SmartSubtransport") + +func getSmartSubtransportInterface(_t *C.git_smart_subtransport) (SmartSubtransport, error) { + wrapperPtr := (*C.managed_smart_subtransport)(unsafe.Pointer(_t)) + + transport, ok := pointerHandles.Get(wrapperPtr.ptr).(SmartSubtransport) + if !ok { + return nil, errNoSmartSubtransport + } + + return transport, nil +} + +//export httpTransportCb +func httpTransportCb(out **C.git_transport, owner *C.git_remote, param unsafe.Pointer) C.int { + return C._go_git_transport_smart(out, owner) +} + +//export httpSmartSubtransportCb +func httpSmartSubtransportCb(out **C.git_smart_subtransport, owner *C.git_transport, param unsafe.Pointer) C.int { + if out == nil { + return -1 + } + + transport := C.calloc(1, C.size_t(unsafe.Sizeof(C.managed_smart_subtransport{}))) + managed := &ManagedTransport{} + managedPtr := pointerHandles.Track(managed) + C._go_git_setup_smart_subtransport(transport, managedPtr) + + *out = (*C.git_smart_subtransport)(transport) + return 0 +} + +var errNoSmartSubtransportStream = errors.New("passed object does not implement SmartSubtransportStream") + +func getSmartSubtransportStreamInterface(_s *C.git_smart_subtransport_stream) (SmartSubtransportStream, error) { + wrapperPtr := (*C.managed_smart_subtransport_stream)(unsafe.Pointer(_s)) + + transport, ok := pointerHandles.Get(wrapperPtr.ptr).(SmartSubtransportStream) + if !ok { + return nil, errNoSmartSubtransportStream + } + + return transport, nil +} + +//export smartSubtransportRead +func smartSubtransportRead(s *C.git_smart_subtransport_stream, data unsafe.Pointer, l C.size_t, read *C.size_t) C.int { + stream, err := getSmartSubtransportStreamInterface(s) + if err != nil { + return setLibgit2Error(err) + } + + var p []byte + header := (*reflect.SliceHeader)(unsafe.Pointer(&p)) + header.Cap = int(l) + header.Len = int(l) + header.Data = uintptr(data) + + n, err := stream.Read(p) + if err != nil { + return setLibgit2Error(err) + } + + *read = C.size_t(n) + return 0 +} + +//export smartSubtransportWrite +func smartSubtransportWrite(s *C.git_smart_subtransport_stream, data unsafe.Pointer, l C.size_t) C.int { + stream, err := getSmartSubtransportStreamInterface(s) + if err != nil { + return setLibgit2Error(err) + } + + var p []byte + header := (*reflect.SliceHeader)(unsafe.Pointer(&p)) + header.Cap = int(l) + header.Len = int(l) + header.Data = uintptr(data) + + if err := stream.Write(p); err != nil { + return setLibgit2Error(err) + } + + return 0 +} + +//export smartSubtransportFree +func smartSubtransportFree(s *C.git_smart_subtransport_stream) { +} diff --git a/wrapper.c b/wrapper.c index 11c2f32..d498284 100644 --- a/wrapper.c +++ b/wrapper.c @@ -2,6 +2,7 @@ #include #include #include +#include typedef int (*gogit_submodule_cbk)(git_submodule *sm, const char *name, void *payload); @@ -180,4 +181,37 @@ void _go_git_writestream_free(git_writestream *stream) stream->free(stream); } +int _go_git_transport_register(const char *scheme) +{ + return git_transport_register(scheme, httpTransportCb, NULL); +} + +int _go_git_transport_smart(git_transport **out, git_remote *owner) +{ + git_smart_subtransport_definition defn = { + httpSmartSubtransportCb, + 1, // RPC/Stateless + }; + + return git_transport_smart(out, owner, &defn); +} + +void _go_git_setup_smart_subtransport(managed_smart_subtransport *t, void *ptr) +{ + t->parent.action = httpAction; + t->parent.close = httpClose; + t->parent.free = httpFree; + + t->ptr = ptr; +} + +void _go_git_setup_smart_subtransport_stream(managed_smart_subtransport_stream *s, void *ptr) +{ + s->parent.read = smartSubtransportRead; + s->parent.write = smartSubtransportWrite; + s->parent.free = smartSubtransportFree; + + s->ptr = ptr; +} + /* EOF */ -- 2.45.2 From c726f932dbe2f52deca64183d3dffea6a3c86384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 17 Apr 2017 12:44:28 +0200 Subject: [PATCH 02/15] http: use an environment-provided proxy We currently cannot ask the transport for its proxy settings, so for now we'll use the one from the environment. --- http.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/http.go b/http.go index c32fc18..b790849 100644 --- a/http.go +++ b/http.go @@ -82,9 +82,17 @@ func RegisterManagedHttps() error { return nil } -type ManagedTransport struct{} +type ManagedTransport struct { + owner *C.git_transport + + client *http.Client +} func (self *ManagedTransport) Action(url string, action SmartService) (SmartSubtransportStream, error) { + if err := self.ensureClient(); err != nil { + return nil, err + } + var req *http.Request var err error switch action { @@ -128,6 +136,19 @@ func (self *ManagedTransport) Close() error { func (self *ManagedTransport) Free() { } +func (self *ManagedTransport) ensureClient() error { + if self.client != nil { + return nil + } + + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + } + self.client = &http.Client{Transport: transport} + + return nil +} + type ManagedHttpStream struct { req *http.Request resp *http.Response @@ -247,7 +268,7 @@ func httpSmartSubtransportCb(out **C.git_smart_subtransport, owner *C.git_transp } transport := C.calloc(1, C.size_t(unsafe.Sizeof(C.managed_smart_subtransport{}))) - managed := &ManagedTransport{} + managed := &ManagedTransport{owner: owner} managedPtr := pointerHandles.Track(managed) C._go_git_setup_smart_subtransport(transport, managedPtr) -- 2.45.2 From e80ae1125a7ebe371e603ba74b5d8e1e310f591b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 17 Apr 2017 18:34:52 +0200 Subject: [PATCH 03/15] fixup! Add a managed HTTP(S) transport implementation --- http.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/http.go b/http.go index b790849..c028ea0 100644 --- a/http.go +++ b/http.go @@ -24,12 +24,10 @@ import "C" import ( "bytes" "errors" - "fmt" "io/ioutil" "net/http" "reflect" "runtime" - //"runtime/debug" "unsafe" ) -- 2.45.2 From 464c24ab07b8e8694078a51eade4e40531ca05f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 17 Apr 2017 19:02:49 +0200 Subject: [PATCH 04/15] remote: add a test to see that we're calling the credentials callback This does get called when we use the libgit2 stack, but is not yet something the Go one does. --- remote_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/remote_test.go b/remote_test.go index 8b20fd2..9e14391 100644 --- a/remote_test.go +++ b/remote_test.go @@ -184,3 +184,26 @@ func TestRemotePrune(t *testing.T) { t.Fatal("Expected error getting a pruned reference") } } + +func TestRemoteCredentialsCalled(t *testing.T) { + t.Parallel() + + repo := createTestRepo(t) + defer cleanupTestRepo(t, repo) + + remote, err := repo.Remotes.CreateAnonymous("https://github.com/libgit2/non-existent") + checkFatal(t, err) + + fetchOpts := FetchOptions{ + RemoteCallbacks: RemoteCallbacks{ + CredentialsCallback: func(url, username string, allowedTypes CredType) (ErrorCode, *Cred) { + return ErrUser, nil + }, + }, + } + + err = remote.Fetch(nil, &fetchOpts, "fetch") + if !IsErrorCode(err, ErrUser) { + t.Fatal("Bad Fetch return, expected ErrUser, got", err) + } +} -- 2.45.2 From e2eda973161d638ab67c6eafb816fd8333972ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 17 Apr 2017 20:08:37 +0200 Subject: [PATCH 05/15] http: call the credentials callback when facing a 401 The test for now just tests that we can handle returning a `GIT_EUSER` if the caller decides to abort. --- http.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/http.go b/http.go index c028ea0..0f0600d 100644 --- a/http.go +++ b/http.go @@ -24,7 +24,7 @@ import "C" import ( "bytes" "errors" - "io/ioutil" + "fmt" "net/http" "reflect" "runtime" @@ -124,7 +124,7 @@ func (self *ManagedTransport) Action(url string, action SmartService) (SmartSubt } req.Header["User-Agent"] = []string{"git/2.0 (git2go)"} - return newManagedHttpStream(req), nil + return newManagedHttpStream(self, req), nil } func (self *ManagedTransport) Close() error { @@ -148,15 +148,17 @@ func (self *ManagedTransport) ensureClient() error { } type ManagedHttpStream struct { + owner *ManagedTransport req *http.Request resp *http.Response postBuffer bytes.Buffer sentRequest bool } -func newManagedHttpStream(req *http.Request) *ManagedHttpStream { +func newManagedHttpStream(owner *ManagedTransport, req *http.Request) *ManagedHttpStream { return &ManagedHttpStream{ - req: req, + owner: owner, + req: req, } } @@ -182,10 +184,51 @@ func (self *ManagedHttpStream) Free() { } func (self *ManagedHttpStream) sendRequest() error { - self.req.Body = ioutil.NopCloser(&self.postBuffer) - resp, err := http.DefaultClient.Do(self.req) - if err != nil { - return err + var resp *http.Response + var err error + var userName string + var password string + for { + req := &http.Request{ + Method: self.req.Method, + URL: self.req.URL, + Header: self.req.Header, + } + + req.SetBasicAuth(userName, password) + resp, err = http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusOK { + break + } + + if resp.StatusCode == http.StatusUnauthorized { + resp.Body.Close() + var cred *C.git_cred + ret := C.git_transport_smart_credentials(&cred, self.owner.owner, nil, C.GIT_CREDTYPE_USERPASS_PLAINTEXT) + + if ret != 0 { + return MakeGitError(ret) + } + + if cred.credtype != C.GIT_CREDTYPE_USERPASS_PLAINTEXT { + C.git_cred_free(cred) + return fmt.Errorf("Unexpected credential type %d", cred.credtype) + } + ptCred := (*C.git_cred_userpass_plaintext)(unsafe.Pointer(cred)) + userName = C.GoString(ptCred.username) + password = C.GoString(ptCred.password) + C.git_cred_free(cred) + + continue + } + + // Any other error we treat as a hard error and punt back to the caller + resp.Body.Close() + return fmt.Errorf("Unhandled HTTP error %s", resp.Status) } self.sentRequest = true @@ -198,6 +241,10 @@ func setLibgit2Error(err error) C.int { defer C.free(unsafe.Pointer(cstr)) C.giterr_set_str(C.GITERR_NET, cstr) + if gitErr, ok := err.(*GitError); ok { + return C.int(gitErr.Code) + } + return -1 } -- 2.45.2 From 1339ad1e71a5757f7fefab4f5275a6288fc1ec4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 17 Apr 2017 20:19:56 +0200 Subject: [PATCH 06/15] http: add casts because Go doesn't believe in const --- http.go | 4 ++-- wrapper.c | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/http.go b/http.go index 0f0600d..275ab91 100644 --- a/http.go +++ b/http.go @@ -335,7 +335,7 @@ func getSmartSubtransportStreamInterface(_s *C.git_smart_subtransport_stream) (S } //export smartSubtransportRead -func smartSubtransportRead(s *C.git_smart_subtransport_stream, data unsafe.Pointer, l C.size_t, read *C.size_t) C.int { +func smartSubtransportRead(s *C.git_smart_subtransport_stream, data *C.char, l C.size_t, read *C.size_t) C.int { stream, err := getSmartSubtransportStreamInterface(s) if err != nil { return setLibgit2Error(err) @@ -345,7 +345,7 @@ func smartSubtransportRead(s *C.git_smart_subtransport_stream, data unsafe.Point header := (*reflect.SliceHeader)(unsafe.Pointer(&p)) header.Cap = int(l) header.Len = int(l) - header.Data = uintptr(data) + header.Data = uintptr(unsafe.Pointer(data)) n, err := stream.Read(p) if err != nil { diff --git a/wrapper.c b/wrapper.c index d498284..aafe901 100644 --- a/wrapper.c +++ b/wrapper.c @@ -198,7 +198,9 @@ int _go_git_transport_smart(git_transport **out, git_remote *owner) void _go_git_setup_smart_subtransport(managed_smart_subtransport *t, void *ptr) { - t->parent.action = httpAction; + typedef int (*transport_action)(git_smart_subtransport_stream **out, git_smart_subtransport *transport, const char *url, git_smart_service_t action); + + t->parent.action = (transport_action)httpAction; t->parent.close = httpClose; t->parent.free = httpFree; @@ -207,8 +209,10 @@ void _go_git_setup_smart_subtransport(managed_smart_subtransport *t, void *ptr) void _go_git_setup_smart_subtransport_stream(managed_smart_subtransport_stream *s, void *ptr) { + typedef int (*transport_stream_write)(git_smart_subtransport_stream *stream, const char *buffer, size_t len); + s->parent.read = smartSubtransportRead; - s->parent.write = smartSubtransportWrite; + s->parent.write = (transport_stream_write)smartSubtransportWrite; s->parent.free = smartSubtransportFree; s->ptr = ptr; -- 2.45.2 From 2aa2ae51504f2bb97ef68d36dc47d5cbe13d8907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Sat, 20 May 2017 17:38:07 +0200 Subject: [PATCH 07/15] fixup! http: call the credentials callback when facing a 401 --- http.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http.go b/http.go index 275ab91..340433c 100644 --- a/http.go +++ b/http.go @@ -208,6 +208,9 @@ func (self *ManagedHttpStream) sendRequest() error { if resp.StatusCode == http.StatusUnauthorized { resp.Body.Close() var cred *C.git_cred + + runtime.LockOSThread() + defer runtime.UnlockOSThread() ret := C.git_transport_smart_credentials(&cred, self.owner.owner, nil, C.GIT_CREDTYPE_USERPASS_PLAINTEXT) if ret != 0 { -- 2.45.2 From dd57a6a89bc80c8bb7097ba37f2c71f79eeda677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Sat, 20 May 2017 17:40:00 +0200 Subject: [PATCH 08/15] vendor: update libgit2 to a1023a430 --- submodule.go | 2 -- vendor/libgit2 | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/submodule.go b/submodule.go index 406ed08..f4b4f15 100644 --- a/submodule.go +++ b/submodule.go @@ -15,7 +15,6 @@ import ( type SubmoduleUpdateOptions struct { *CheckoutOpts *FetchOptions - CloneCheckoutStrategy CheckoutStrategy } // Submodule @@ -369,7 +368,6 @@ func populateSubmoduleUpdateOptions(ptr *C.git_submodule_update_options, opts *S populateCheckoutOpts(&ptr.checkout_opts, opts.CheckoutOpts) populateFetchOptions(&ptr.fetch_opts, opts.FetchOptions) - ptr.clone_checkout_strategy = C.uint(opts.CloneCheckoutStrategy) return nil } diff --git a/vendor/libgit2 b/vendor/libgit2 index df4dfaa..a1023a4 160000 --- a/vendor/libgit2 +++ b/vendor/libgit2 @@ -1 +1 @@ -Subproject commit df4dfaadcf709646ebab2e57e3589952cf1ac809 +Subproject commit a1023a43027207ac7a5df7233bddfe7347bee256 -- 2.45.2 From c5e6dc298e89370d4789e6a3a7c12d3569898982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Sat, 20 May 2017 19:08:23 +0200 Subject: [PATCH 09/15] http: correct EOF from stream and do send out our buffered data --- http.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/http.go b/http.go index 340433c..0b2ef57 100644 --- a/http.go +++ b/http.go @@ -25,6 +25,8 @@ import ( "bytes" "errors" "fmt" + "io" + "io/ioutil" "net/http" "reflect" "runtime" @@ -128,6 +130,7 @@ func (self *ManagedTransport) Action(url string, action SmartService) (SmartSubt } func (self *ManagedTransport) Close() error { + self.client = nil return nil } @@ -190,9 +193,11 @@ func (self *ManagedHttpStream) sendRequest() error { var password string for { req := &http.Request{ - Method: self.req.Method, - URL: self.req.URL, - Header: self.req.Header, + Method: self.req.Method, + URL: self.req.URL, + Header: self.req.Header, + Body: ioutil.NopCloser(&self.postBuffer), + ContentLength: int64(self.postBuffer.Len()), } req.SetBasicAuth(userName, password) @@ -352,7 +357,13 @@ func smartSubtransportRead(s *C.git_smart_subtransport_stream, data *C.char, l C n, err := stream.Read(p) if err != nil { - return setLibgit2Error(err) + if err == io.EOF { + *read = C.size_t(0) + return 0 + } + + setLibgit2Error(err) + return -1 } *read = C.size_t(n) -- 2.45.2 From 50f588fcc452b18c9ae21aa446f2853c6e029210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Sat, 20 May 2017 19:09:30 +0200 Subject: [PATCH 10/15] error: don't read an error string on ErrUser --- git.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git.go b/git.go index 077d04b..f8099d1 100644 --- a/git.go +++ b/git.go @@ -279,7 +279,7 @@ func MakeGitError(errorCode C.int) error { var errMessage string var errClass ErrorClass - if errorCode != C.GIT_ITEROVER { + if errorCode != C.GIT_ITEROVER && errorCode != C.GIT_EUSER { err := C.giterr_last() if err != nil { errMessage = C.GoString(err.message) -- 2.45.2 From 22da146353a4313f376149f07c949ded293bbfc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Sun, 21 May 2017 21:47:33 +0200 Subject: [PATCH 11/15] http: set the proxy from the user-given options --- http.go | 27 ++++++++++++++++++++++++++- remote.go | 7 +++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/http.go b/http.go index 0b2ef57..9de001b 100644 --- a/http.go +++ b/http.go @@ -28,6 +28,7 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "reflect" "runtime" "unsafe" @@ -142,8 +143,32 @@ func (self *ManagedTransport) ensureClient() error { return nil } + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + var cpopts C.git_proxy_options + if ret := C.git_transport_smart_proxy_options(&cpopts, self.owner); ret < 0 { + return MakeGitError(ret) + } + + var proxyFn func(*http.Request) (*url.URL, error) + proxyOpts := proxyOptionsFromC(&cpopts) + switch proxyOpts.Type { + case ProxyTypeNone: + proxyFn = nil + case ProxyTypeAuto: + proxyFn = http.ProxyFromEnvironment + case ProxyTypeSpecified: + parsedUrl, err := url.Parse(proxyOpts.Url) + if err != nil { + return err + } + + proxyFn = http.ProxyURL(parsedUrl) + } + transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, + Proxy: proxyFn, } self.client = &http.Client{Transport: transport} diff --git a/remote.go b/remote.go index b4b1dd7..07ff291 100644 --- a/remote.go +++ b/remote.go @@ -141,6 +141,13 @@ type ProxyOptions struct { Url string } +func proxyOptionsFromC(copts *C.git_proxy_options) ProxyOptions { + return ProxyOptions{ + Type: ProxyType(copts._type), + Url: C.GoString(copts.url), + } +} + type Remote struct { ptr *C.git_remote callbacks RemoteCallbacks -- 2.45.2 From b2ec6e6bfc5708a2569b63dee33898b23d1eea3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 22 May 2017 09:25:41 +0200 Subject: [PATCH 12/15] travis: don't go recursive with the submodules --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6131e6d..d073581 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,13 @@ go: script: make test-static +# The default of --recursive chokes on the libgit2 tests +git: + submodules: false + +before_install: + - git submodule update --init + matrix: allow_failures: - go: tip -- 2.45.2 From 2d54afed6199b10d2330c98f4c283e977e48bed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Sun, 9 Jul 2017 11:41:53 +0200 Subject: [PATCH 13/15] http: add casts --- http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http.go b/http.go index 9de001b..048322f 100644 --- a/http.go +++ b/http.go @@ -295,7 +295,7 @@ func httpAction(out **C.git_smart_subtransport_stream, t *C.git_smart_subtranspo stream := C.calloc(1, C.size_t(unsafe.Sizeof(C.managed_smart_subtransport_stream{}))) managedPtr := pointerHandles.Track(managed) - C._go_git_setup_smart_subtransport_stream(stream, managedPtr) + C._go_git_setup_smart_subtransport_stream((*C.managed_smart_subtransport_stream)(stream), managedPtr) *out = (*C.git_smart_subtransport_stream)(stream) return 0 @@ -348,7 +348,7 @@ func httpSmartSubtransportCb(out **C.git_smart_subtransport, owner *C.git_transp transport := C.calloc(1, C.size_t(unsafe.Sizeof(C.managed_smart_subtransport{}))) managed := &ManagedTransport{owner: owner} managedPtr := pointerHandles.Track(managed) - C._go_git_setup_smart_subtransport(transport, managedPtr) + C._go_git_setup_smart_subtransport((*C.managed_smart_subtransport)(transport), managedPtr) *out = (*C.git_smart_subtransport)(transport) return 0 -- 2.45.2 From 6fe4f461f64d7463b9599e1a1fa60ca11c6325b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 22 May 2017 10:14:30 +0200 Subject: [PATCH 14/15] http: stream POST contents This lets us stream the data straight from the creation of the packfile, instead of having to buffer it all in memory. --- http.go | 64 ++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/http.go b/http.go index 048322f..4abbfc0 100644 --- a/http.go +++ b/http.go @@ -22,15 +22,14 @@ void _go_git_setup_smart_subtransport_stream(managed_smart_subtransport_stream * import "C" import ( - "bytes" "errors" "fmt" "io" - "io/ioutil" "net/http" "net/url" "reflect" "runtime" + "sync" "unsafe" ) @@ -127,7 +126,14 @@ func (self *ManagedTransport) Action(url string, action SmartService) (SmartSubt } req.Header["User-Agent"] = []string{"git/2.0 (git2go)"} - return newManagedHttpStream(self, req), nil + + stream := newManagedHttpStream(self, req) + if req.Method == "POST" { + stream.recvReply.Add(1) + stream.sendRequestBackground() + } + + return stream, nil } func (self *ManagedTransport) Close() error { @@ -179,31 +185,49 @@ type ManagedHttpStream struct { owner *ManagedTransport req *http.Request resp *http.Response - postBuffer bytes.Buffer + reader *io.PipeReader + writer *io.PipeWriter sentRequest bool + recvReply sync.WaitGroup + httpError error } func newManagedHttpStream(owner *ManagedTransport, req *http.Request) *ManagedHttpStream { + r, w := io.Pipe() return &ManagedHttpStream{ - owner: owner, - req: req, + owner: owner, + req: req, + reader: r, + writer: w, } } func (self *ManagedHttpStream) Read(buf []byte) (int, error) { if !self.sentRequest { + self.recvReply.Add(1) if err := self.sendRequest(); err != nil { return 0, err } } + if err := self.writer.Close(); err != nil { + return 0, err + } + + self.recvReply.Wait() + + if self.httpError != nil { + return 0, self.httpError + } + return self.resp.Body.Read(buf) } func (self *ManagedHttpStream) Write(buf []byte) error { - // We write it all into a buffer and send it off when the transport asks - // us to read. - self.postBuffer.Write(buf) + if self.httpError != nil { + return self.httpError + } + self.writer.Write(buf) return nil } @@ -211,18 +235,30 @@ func (self *ManagedHttpStream) Free() { self.resp.Body.Close() } +func (self *ManagedHttpStream) sendRequestBackground() { + go func() { + self.httpError = self.sendRequest() + }() + self.sentRequest = true +} + func (self *ManagedHttpStream) sendRequest() error { + defer self.recvReply.Done() + self.resp = nil + var resp *http.Response var err error var userName string var password string for { req := &http.Request{ - Method: self.req.Method, - URL: self.req.URL, - Header: self.req.Header, - Body: ioutil.NopCloser(&self.postBuffer), - ContentLength: int64(self.postBuffer.Len()), + Method: self.req.Method, + URL: self.req.URL, + Header: self.req.Header, + } + if req.Method == "POST" { + req.Body = self.reader + req.ContentLength = -1 } req.SetBasicAuth(userName, password) -- 2.45.2 From b63157de51a6caeffe067d3410b0a3f375b94390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Sun, 9 Jul 2017 13:49:53 +0200 Subject: [PATCH 15/15] http: propagate error codes --- http.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/http.go b/http.go index 4abbfc0..09d6e09 100644 --- a/http.go +++ b/http.go @@ -423,8 +423,7 @@ func smartSubtransportRead(s *C.git_smart_subtransport_stream, data *C.char, l C return 0 } - setLibgit2Error(err) - return -1 + return setLibgit2Error(err) } *read = C.size_t(n) -- 2.45.2