From d7fd15b1e0fda55c7237609bbdc8caddff49d032 Mon Sep 17 00:00:00 2001 From: lhchavez Date: Thu, 5 Jul 2018 03:50:10 +0000 Subject: [PATCH 1/3] Add support for mempack This allows the creation of git packfiles without needing to write them to the filesystem. --- mempack.go | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 mempack.go diff --git a/mempack.go b/mempack.go new file mode 100644 index 0000000..0b82cd4 --- /dev/null +++ b/mempack.go @@ -0,0 +1,97 @@ +package git + +/* +#include +#include + +extern int git_mempack_new(git_odb_backend **out); +extern int git_mempack_dump(git_buf *pack, git_repository *repo, git_odb_backend *backend); +extern void git_mempack_reset(git_odb_backend *backend); +extern void _go_git_odb_backend_free(git_odb_backend *backend); +*/ +import "C" + +import ( + "runtime" + "unsafe" +) + +// Mempack is a custom ODB backend that permits packing object in-memory. +type Mempack struct { + ptr *C.git_odb_backend +} + +// NewMempack creates a new mempack instance and registers it to the ODB. +func NewMempack(odb *Odb) (mempack *Mempack, err error) { + mempack = new(Mempack) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C.git_mempack_new(&mempack.ptr) + if ret < 0 { + return nil, MakeGitError(ret) + } + runtime.SetFinalizer(mempack, (*Mempack).Free) + + ret = C.git_odb_add_backend(odb.ptr, mempack.ptr, C.int(999)) + if ret < 0 { + // Since git_odb_add_alternate() takes ownership of the ODB backend, the + // only case in which we free the mempack's memory is if it fails to be + // added to the ODB. Mempack.Free() is actually just a no-op. + C._go_git_odb_backend_free(mempack.ptr) + mempack.Free() + return nil, MakeGitError(ret) + } + + return mempack, nil +} + +// Dump dumps all the queued in-memory writes to a packfile. +// +// It is the caller's responsibility to ensure that the generated packfile is +// available to the repository (e.g. by writing it to disk, or doing something +// crazy like distributing it across several copies of the repository over a +// network). +// +// Once the generated packfile is available to the repository, call +// Mempack.Reset to cleanup the memory store. +// +// Calling Mempack.Reset before the packfile has been written to disk will +// result in an inconsistent repository (the objects in the memory store won't +// be accessible). +func (mempack *Mempack) Dump(repository *Repository) ([]byte, error) { + buf := C.git_buf{} + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + var repoPtr *C.git_repository + if repository != nil { + repoPtr = repository.ptr + } + + ret := C.git_mempack_dump(&buf, repoPtr, mempack.ptr) + if ret < 0 { + return nil, MakeGitError(ret) + } + defer C.git_buf_free(&buf) + + return C.GoBytes(unsafe.Pointer(buf.ptr), C.int(buf.size)), nil +} + +// Reset resets the memory packer by clearing all the queued objects. +// +// This assumes that Mempack.Dump has been called before to store all the +// queued objects into a single packfile. +func (mempack *Mempack) Reset() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + C.git_mempack_reset(mempack.ptr) +} + +// Free frees the mempack and its resources. +func (mempack *Mempack) Free() { + runtime.SetFinalizer(mempack, nil) +} From a2c93a0bcf63c88f9324333f7f3f6a204a817617 Mon Sep 17 00:00:00 2001 From: lhchavez Date: Tue, 8 Jan 2019 02:45:04 +0000 Subject: [PATCH 2/3] Addressed review feedback and added a test --- mempack.go | 22 ++++-------------- mempack_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 mempack_test.go diff --git a/mempack.go b/mempack.go index 0b82cd4..72a3df5 100644 --- a/mempack.go +++ b/mempack.go @@ -32,15 +32,13 @@ func NewMempack(odb *Odb) (mempack *Mempack, err error) { if ret < 0 { return nil, MakeGitError(ret) } - runtime.SetFinalizer(mempack, (*Mempack).Free) ret = C.git_odb_add_backend(odb.ptr, mempack.ptr, C.int(999)) if ret < 0 { // Since git_odb_add_alternate() takes ownership of the ODB backend, the // only case in which we free the mempack's memory is if it fails to be - // added to the ODB. Mempack.Free() is actually just a no-op. + // added to the ODB. C._go_git_odb_backend_free(mempack.ptr) - mempack.Free() return nil, MakeGitError(ret) } @@ -66,16 +64,12 @@ func (mempack *Mempack) Dump(repository *Repository) ([]byte, error) { runtime.LockOSThread() defer runtime.UnlockOSThread() - var repoPtr *C.git_repository - if repository != nil { - repoPtr = repository.ptr - } - - ret := C.git_mempack_dump(&buf, repoPtr, mempack.ptr) + ret := C.git_mempack_dump(&buf, repository.ptr, mempack.ptr) + runtime.KeepAlive(repository) if ret < 0 { return nil, MakeGitError(ret) } - defer C.git_buf_free(&buf) + defer C.git_buf_dispose(&buf) return C.GoBytes(unsafe.Pointer(buf.ptr), C.int(buf.size)), nil } @@ -85,13 +79,5 @@ func (mempack *Mempack) Dump(repository *Repository) ([]byte, error) { // This assumes that Mempack.Dump has been called before to store all the // queued objects into a single packfile. func (mempack *Mempack) Reset() { - runtime.LockOSThread() - defer runtime.UnlockOSThread() - C.git_mempack_reset(mempack.ptr) } - -// Free frees the mempack and its resources. -func (mempack *Mempack) Free() { - runtime.SetFinalizer(mempack, nil) -} diff --git a/mempack_test.go b/mempack_test.go new file mode 100644 index 0000000..3e31dcf --- /dev/null +++ b/mempack_test.go @@ -0,0 +1,60 @@ +package git + +import ( + "bytes" + "testing" +) + +func TestMempack(t *testing.T) { + t.Parallel() + + odb, err := NewOdb() + checkFatal(t, err) + + repo, err := NewRepositoryWrapOdb(odb) + checkFatal(t, err) + + mempack, err := NewMempack(odb) + checkFatal(t, err) + + id, err := odb.Write([]byte("hello, world!"), ObjectBlob) + checkFatal(t, err) + + expectedId, err := NewOid("30f51a3fba5274d53522d0f19748456974647b4f") + checkFatal(t, err) + if !expectedId.Equal(id) { + t.Errorf("mismatched id. expected %v, got %v", expectedId.String(), id.String()) + } + + // The object should be available from the odb. + { + obj, err := odb.Read(expectedId) + checkFatal(t, err) + defer obj.Free() + } + + data, err := mempack.Dump(repo) + checkFatal(t, err) + + expectedData := []byte{ + 0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x9d, 0x08, 0x82, 0x3b, 0xd8, 0xa8, 0xea, 0xb5, 0x10, 0xad, 0x6a, + 0xc7, 0x5c, 0x82, 0x3c, 0xfd, 0x3e, 0xd3, 0x1e, + } + if !bytes.Equal(expectedData, data) { + t.Errorf("mismatched mempack data. expected %v, got %v", expectedData, data) + } + + mempack.Reset() + + // After the reset, the object should now be unavailable. + { + obj, err := odb.Read(expectedId) + if err == nil { + t.Errorf("object %s unexpectedly found", obj.Id().String()) + obj.Free() + } else if !IsErrorCode(err, ErrNotFound) { + t.Errorf("unexpected error %v", err) + } + } +} From f4ea2a561da9868efc670467061129ac6e53825d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Tue, 15 Jan 2019 20:28:36 +0100 Subject: [PATCH 3/3] Keep odb alive when adding mempack --- mempack.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mempack.go b/mempack.go index 72a3df5..337e67d 100644 --- a/mempack.go +++ b/mempack.go @@ -34,6 +34,7 @@ func NewMempack(odb *Odb) (mempack *Mempack, err error) { } ret = C.git_odb_add_backend(odb.ptr, mempack.ptr, C.int(999)) + runtime.KeepAlive(odb) if ret < 0 { // Since git_odb_add_alternate() takes ownership of the ODB backend, the // only case in which we free the mempack's memory is if it fails to be