diff --git a/indexer.go b/indexer.go new file mode 100644 index 0000000..d1b9372 --- /dev/null +++ b/indexer.go @@ -0,0 +1,99 @@ +package git + +/* +#include + +extern const git_oid * git_indexer_hash(const git_indexer *idx); +extern int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats); +extern int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats); +extern int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload); +extern void git_indexer_free(git_indexer *idx); +*/ +import "C" +import ( + "reflect" + "runtime" + "unsafe" +) + +// Indexer can post-process packfiles and create an .idx file for efficient +// lookup. +type Indexer struct { + ptr *C.git_indexer + stats C.git_transfer_progress + callbacks RemoteCallbacks + callbacksHandle unsafe.Pointer +} + +// NewIndexer creates a new indexer instance. +func NewIndexer(packfilePath string, odb *Odb, callback TransferProgressCallback) (indexer *Indexer, err error) { + indexer = new(Indexer) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + var odbPtr *C.git_odb = nil + if odb != nil { + odbPtr = odb.ptr + } + + indexer.callbacks.TransferProgressCallback = callback + indexer.callbacksHandle = pointerHandles.Track(&indexer.callbacks) + + cstr := C.CString(packfilePath) + defer C.free(unsafe.Pointer(cstr)) + + ret := C._go_git_indexer_new(&indexer.ptr, cstr, 0, odbPtr, indexer.callbacksHandle) + runtime.KeepAlive(odb) + if ret < 0 { + pointerHandles.Untrack(indexer.callbacksHandle) + return nil, MakeGitError(ret) + } + + runtime.SetFinalizer(indexer, (*Indexer).Free) + return indexer, nil +} + +// Write adds data to the indexer. +func (indexer *Indexer) Write(data []byte) (int, error) { + header := (*reflect.SliceHeader)(unsafe.Pointer(&data)) + ptr := unsafe.Pointer(header.Data) + size := C.size_t(header.Len) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C.git_indexer_append(indexer.ptr, ptr, size, &indexer.stats) + runtime.KeepAlive(indexer) + if ret < 0 { + return 0, MakeGitError(ret) + } + + return len(data), nil +} + +// Commit finalizes the pack and index. It resolves any pending deltas and +// writes out the index file. +// +// It also returns the packfile's hash. A packfile's name is derived from the +// sorted hashing of all object names. +func (indexer *Indexer) Commit() (*Oid, error) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C.git_indexer_commit(indexer.ptr, &indexer.stats) + if ret < 0 { + return nil, MakeGitError(ret) + } + + id := newOidFromC(C.git_indexer_hash(indexer.ptr)) + runtime.KeepAlive(indexer) + return id, nil +} + +// Free frees the indexer and its resources. +func (indexer *Indexer) Free() { + pointerHandles.Untrack(indexer.callbacksHandle) + runtime.SetFinalizer(indexer, nil) + C.git_indexer_free(indexer.ptr) +} diff --git a/indexer_test.go b/indexer_test.go new file mode 100644 index 0000000..1b65c95 --- /dev/null +++ b/indexer_test.go @@ -0,0 +1,93 @@ +package git + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "testing" +) + +var ( + // This is a packfile with three objects. The second is a delta which + // depends on the third, which is also a delta. + outOfOrderPack = []byte{ + 0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, + 0x32, 0x78, 0x9c, 0x63, 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x08, 0x76, + 0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95, 0x10, + 0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63, 0x62, + 0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x75, 0x01, + 0xd7, 0x71, 0x36, 0x66, 0xf4, 0xde, 0x82, 0x27, 0x76, 0xc7, 0x62, 0x2c, + 0x10, 0xf1, 0xb0, 0x7d, 0xe2, 0x80, 0xdc, 0x78, 0x9c, 0x63, 0x62, 0x62, + 0x62, 0xb7, 0x03, 0x00, 0x00, 0x69, 0x00, 0x4c, 0xde, 0x7d, 0xaa, 0xe4, + 0x19, 0x87, 0x58, 0x80, 0x61, 0x09, 0x9a, 0x33, 0xca, 0x7a, 0x31, 0x92, + 0x6f, 0xae, 0x66, 0x75, + } +) + +func TestIndexerOutOfOrder(t *testing.T) { + t.Parallel() + + tmpPath, err := ioutil.TempDir("", "git2go") + checkFatal(t, err) + defer os.RemoveAll(tmpPath) + + var finalStats TransferProgress + idx, err := NewIndexer(tmpPath, nil, func(stats TransferProgress) ErrorCode { + finalStats = stats + return ErrOk + }) + checkFatal(t, err) + defer idx.Free() + + _, err = idx.Write(outOfOrderPack) + checkFatal(t, err) + oid, err := idx.Commit() + checkFatal(t, err) + + // The packfile contains the hash as the last 20 bytes. + expectedOid := NewOidFromBytes(outOfOrderPack[len(outOfOrderPack)-20:]) + if !expectedOid.Equal(oid) { + t.Errorf("mismatched packfile hash, expected %v, got %v", expectedOid, oid) + } + if finalStats.TotalObjects != 3 { + t.Errorf("mismatched transferred objects, expected 3, got %v", finalStats.TotalObjects) + } + if finalStats.ReceivedObjects != 3 { + t.Errorf("mismatched received objects, expected 3, got %v", finalStats.ReceivedObjects) + } + if finalStats.IndexedObjects != 3 { + t.Errorf("mismatched indexed objects, expected 3, got %v", finalStats.IndexedObjects) + } + + odb, err := NewOdb() + checkFatal(t, err) + defer odb.Free() + + backend, err := NewOdbBackendOnePack(path.Join(tmpPath, fmt.Sprintf("pack-%s.idx", oid.String()))) + checkFatal(t, err) + // Transfer the ownership of the backend to the odb, no freeing needed. + err = odb.AddBackend(backend, 1) + checkFatal(t, err) + + packfileObjects := 0 + err = odb.ForEach(func(id *Oid) error { + packfileObjects += 1 + return nil + }) + checkFatal(t, err) + if packfileObjects != 3 { + t.Errorf("mismatched packfile objects, expected 3, got %v", packfileObjects) + } + + // Inspect one of the well-known objects in the packfile. + obj, err := odb.Read(NewOidFromBytes([]byte{ + 0x19, 0x10, 0x28, 0x15, 0x66, 0x3d, 0x23, 0xf8, 0xb7, 0x5a, 0x47, 0xe7, + 0xa0, 0x19, 0x65, 0xdc, 0xdc, 0x96, 0x46, 0x8c, + })) + checkFatal(t, err) + defer obj.Free() + if "foo" != string(obj.Data()) { + t.Errorf("mismatched packfile object contents, expected foo, got %q", string(obj.Data())) + } +} diff --git a/odb.go b/odb.go index 6489653..994ff2a 100644 --- a/odb.go +++ b/odb.go @@ -3,8 +3,13 @@ package git /* #include +extern int git_odb_backend_one_pack(git_odb_backend **out, const char *index_file); extern int _go_git_odb_foreach(git_odb *db, void *payload); extern void _go_git_odb_backend_free(git_odb_backend *backend); +extern int _go_git_odb_write_pack(git_odb_writepack **out, git_odb *db, void *progress_payload); +extern int _go_git_odb_writepack_append(git_odb_writepack *writepack, const void *, size_t, git_transfer_progress *); +extern int _go_git_odb_writepack_commit(git_odb_writepack *writepack, git_transfer_progress *); +extern void _go_git_odb_writepack_free(git_odb_writepack *writepack); */ import "C" import ( @@ -42,8 +47,20 @@ func NewOdbBackendFromC(ptr unsafe.Pointer) (backend *OdbBackend) { return backend } -func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) { +func (v *Odb) AddAlternate(backend *OdbBackend, priority int) (err error) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + ret := C.git_odb_add_alternate(v.ptr, backend.ptr, C.int(priority)) + runtime.KeepAlive(v) + if ret < 0 { + backend.Free() + return MakeGitError(ret) + } + return nil +} + +func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -56,6 +73,21 @@ func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) { return nil } +func NewOdbBackendOnePack(packfileIndexPath string) (backend *OdbBackend, err error) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + cstr := C.CString(packfileIndexPath) + defer C.free(unsafe.Pointer(cstr)) + + var odbOnePack *C.git_odb_backend = nil + ret := C.git_odb_backend_one_pack(&odbOnePack, cstr) + if ret < 0 { + return nil, MakeGitError(ret) + } + return NewOdbBackendFromC(unsafe.Pointer(odbOnePack)), nil +} + func (v *Odb) ReadHeader(oid *Oid) (uint64, ObjectType, error) { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -231,6 +263,31 @@ func (v *Odb) NewWriteStream(size int64, otype ObjectType) (*OdbWriteStream, err return stream, nil } +// NewWritePack opens a stream for writing a pack file to the ODB. If the ODB +// layer understands pack files, then the given packfile will likely be +// streamed directly to disk (and a corresponding index created). If the ODB +// layer does not understand pack files, the objects will be stored in whatever +// format the ODB layer uses. +func (v *Odb) NewWritePack(callback TransferProgressCallback) (*OdbWritepack, error) { + writepack := new(OdbWritepack) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + writepack.callbacks.TransferProgressCallback = callback + writepack.callbacksHandle = pointerHandles.Track(&writepack.callbacks) + + ret := C._go_git_odb_write_pack(&writepack.ptr, v.ptr, writepack.callbacksHandle) + runtime.KeepAlive(v) + if ret < 0 { + pointerHandles.Untrack(writepack.callbacksHandle) + return nil, MakeGitError(ret) + } + + runtime.SetFinalizer(writepack, (*OdbWritepack).Free) + return writepack, nil +} + func (v *OdbBackend) Free() { C._go_git_odb_backend_free(v.ptr) } @@ -360,3 +417,47 @@ func (stream *OdbWriteStream) Free() { runtime.SetFinalizer(stream, nil) C.git_odb_stream_free(stream.ptr) } + +// OdbWritepack is a stream to write a packfile to the ODB. +type OdbWritepack struct { + ptr *C.git_odb_writepack + stats C.git_transfer_progress + callbacks RemoteCallbacks + callbacksHandle unsafe.Pointer +} + +func (writepack *OdbWritepack) Write(data []byte) (int, error) { + header := (*reflect.SliceHeader)(unsafe.Pointer(&data)) + ptr := unsafe.Pointer(header.Data) + size := C.size_t(header.Len) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C._go_git_odb_writepack_append(writepack.ptr, ptr, size, &writepack.stats) + runtime.KeepAlive(writepack) + if ret < 0 { + return 0, MakeGitError(ret) + } + + return len(data), nil +} + +func (writepack *OdbWritepack) Commit() error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C._go_git_odb_writepack_commit(writepack.ptr, &writepack.stats) + runtime.KeepAlive(writepack) + if ret < 0 { + return MakeGitError(ret) + } + + return nil +} + +func (writepack *OdbWritepack) Free() { + pointerHandles.Untrack(writepack.callbacksHandle) + runtime.SetFinalizer(writepack, nil) + C._go_git_odb_writepack_free(writepack.ptr) +} diff --git a/odb_test.go b/odb_test.go index 46acdba..e44c927 100644 --- a/odb_test.go +++ b/odb_test.go @@ -152,3 +152,37 @@ func TestOdbForeach(t *testing.T) { t.Fatalf("Odb.ForEach() did not return the expected error, got %v", err) } } + +func TestOdbWritepack(t *testing.T) { + t.Parallel() + repo := createTestRepo(t) + defer cleanupTestRepo(t, repo) + + _, _ = seedTestRepo(t, repo) + + odb, err := repo.Odb() + checkFatal(t, err) + + var finalStats TransferProgress + writepack, err := odb.NewWritePack(func(stats TransferProgress) ErrorCode { + finalStats = stats + return ErrOk + }) + checkFatal(t, err) + defer writepack.Free() + + _, err = writepack.Write(outOfOrderPack) + checkFatal(t, err) + err = writepack.Commit() + checkFatal(t, err) + + if finalStats.TotalObjects != 3 { + t.Errorf("mismatched transferred objects, expected 3, got %v", finalStats.TotalObjects) + } + if finalStats.ReceivedObjects != 3 { + t.Errorf("mismatched received objects, expected 3, got %v", finalStats.ReceivedObjects) + } + if finalStats.IndexedObjects != 3 { + t.Errorf("mismatched indexed objects, expected 3, got %v", finalStats.IndexedObjects) + } +} diff --git a/wrapper.c b/wrapper.c index 3656773..c4a5ff0 100644 --- a/wrapper.c +++ b/wrapper.c @@ -185,4 +185,32 @@ git_credtype_t _go_git_cred_credtype(git_cred *cred) { return cred->credtype; } +int _go_git_odb_write_pack(git_odb_writepack **out, git_odb *db, void *progress_payload) +{ + return git_odb_write_pack(out, db, (git_transfer_progress_cb)transferProgressCallback, progress_payload); +} + +int _go_git_odb_writepack_append(git_odb_writepack *writepack, const void *data, size_t size, git_transfer_progress *stats) +{ + return writepack->append(writepack, data, size, stats); +} + +int _go_git_odb_writepack_commit(git_odb_writepack *writepack, git_transfer_progress *stats) +{ + return writepack->commit(writepack, stats); +} + +void _go_git_odb_writepack_free(git_odb_writepack *writepack) +{ + writepack->free(writepack); +} + +int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload) +{ + git_indexer_options indexer_options = GIT_INDEXER_OPTIONS_INIT; + indexer_options.progress_cb = (git_transfer_progress_cb)transferProgressCallback; + indexer_options.progress_cb_payload = progress_cb_payload; + return git_indexer_new(out, path, mode, odb, &indexer_options); +} + /* EOF */