From a2680c63c2ef4aded7c9e590b433255161c37b46 Mon Sep 17 00:00:00 2001 From: lhchavez Date: Thu, 22 Dec 2016 06:46:13 -0800 Subject: [PATCH 1/5] Add support for indexers and alternate odb packfiles This allows for implementations of git servers written in Go. --- indexer.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ odb.go | 32 ++++++++++++++++--- wrapper.c | 5 +++ 3 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 indexer.go diff --git a/indexer.go b/indexer.go new file mode 100644 index 0000000..80f34fe --- /dev/null +++ b/indexer.go @@ -0,0 +1,94 @@ +package git + +/* +#include + +extern const git_oid * git_indexer_hash(const git_indexer *idx); +extern int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats); +extern int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats); +extern int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload); +extern void git_indexer_free(git_indexer *idx); +*/ +import "C" +import ( + "reflect" + "runtime" + "unsafe" +) + +//export indexerTransferProgress +func indexerTransferProgress(_stats *C.git_transfer_progress, ptr unsafe.Pointer) C.int { + callback, ok := pointerHandles.Get(ptr).(TransferProgressCallback) + if !ok { + return 0 + } + return C.int(callback(newTransferProgressFromC(_stats))) +} + +type Indexer struct { + ptr *C.git_indexer + stats C.git_transfer_progress + callback unsafe.Pointer +} + +func NewIndexer(packfilePath string, odb *Odb, callback TransferProgressCallback) (indexer *Indexer, err error) { + indexer = new(Indexer) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + var odbPtr *C.git_odb = nil + if odb != nil { + odbPtr = odb.ptr + } + + if callback != nil { + indexer.callback = pointerHandles.Track(callback) + } + + ret := C._go_git_indexer_new(&indexer.ptr, C.CString(packfilePath), 0, odbPtr, indexer.callback) + if ret < 0 { + if indexer.callback != nil { + pointerHandles.Untrack(indexer.callback) + } + return nil, MakeGitError(ret) + } + + runtime.SetFinalizer(indexer, (*Indexer).Free) + return indexer, nil +} + +func (indexer *Indexer) Write(data []byte) (int, error) { + header := (*reflect.SliceHeader)(unsafe.Pointer(&data)) + ptr := unsafe.Pointer(header.Data) + size := C.size_t(header.Len) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C.git_indexer_append(indexer.ptr, ptr, size, &indexer.stats) + if ret < 0 { + return 0, MakeGitError(ret) + } + + return len(data), nil +} + +func (indexer *Indexer) Commit() (*Oid, error) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C.git_indexer_commit(indexer.ptr, &indexer.stats) + if ret < 0 { + return nil, MakeGitError(ret) + } + + return newOidFromC(C.git_indexer_hash(indexer.ptr)), nil +} + +func (indexer *Indexer) Free() { + if indexer.callback != nil { + pointerHandles.Untrack(indexer.callback) + } + C.git_indexer_free(indexer.ptr) +} diff --git a/odb.go b/odb.go index 28bfad6..248cb55 100644 --- a/odb.go +++ b/odb.go @@ -3,6 +3,7 @@ package git /* #include +extern int git_odb_backend_one_pack(git_odb_backend **out, const char *index_file); extern int _go_git_odb_foreach(git_odb *db, void *payload); extern void _go_git_odb_backend_free(git_odb_backend *backend); */ @@ -41,8 +42,19 @@ func NewOdbBackendFromC(ptr *C.git_odb_backend) (backend *OdbBackend) { return backend } -func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) { +func (v *Odb) AddAlternate(backend *OdbBackend, priority int) (err error) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + ret := C.git_odb_add_alternate(v.ptr, backend.ptr, C.int(priority)) + if ret < 0 { + backend.Free() + return MakeGitError(ret) + } + return nil +} + +func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -54,12 +66,24 @@ func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) { return nil } +func NewOdbBackendOnePack(packfileIndexPath string) (backend *OdbBackend, err error) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + var odbOnePack *C.git_odb_backend = nil + ret := C.git_odb_backend_one_pack(&odbOnePack, C.CString(packfileIndexPath)) + if ret < 0 { + return nil, MakeGitError(ret) + } + return NewOdbBackendFromC(odbOnePack), nil +} + func (v *Odb) ReadHeader(oid *Oid) (uint64, ObjectType, error) { runtime.LockOSThread() defer runtime.UnlockOSThread() - + var sz C.size_t - var cotype C.git_otype + var cotype C.git_otype ret := C.git_odb_read_header(&sz, &cotype, v.ptr, oid.toC()) if ret < 0 { @@ -68,7 +92,7 @@ func (v *Odb) ReadHeader(oid *Oid) (uint64, ObjectType, error) { return uint64(sz), ObjectType(cotype), nil } - + func (v *Odb) Exists(oid *Oid) bool { ret := C.git_odb_exists(v.ptr, oid.toC()) return ret != 0 diff --git a/wrapper.c b/wrapper.c index 7407611..d0ca4b6 100644 --- a/wrapper.c +++ b/wrapper.c @@ -174,4 +174,9 @@ void _go_git_writestream_free(git_writestream *stream) stream->free(stream); } +int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload) +{ + return git_indexer_new(out, path, mode, odb, (git_transfer_progress_cb)&indexerTransferProgress, progress_cb_payload); +} + /* EOF */ -- 2.45.2 From 0ef25268fa8f64949d68c3eac355c3ce455857e2 Mon Sep 17 00:00:00 2001 From: lhchavez Date: Fri, 23 Dec 2016 19:46:55 -0800 Subject: [PATCH 2/5] Also add support for git_odb_writepack --- indexer.go | 32 +++++++---------------- odb.go | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ wrapper.c | 22 +++++++++++++++- 3 files changed, 108 insertions(+), 23 deletions(-) diff --git a/indexer.go b/indexer.go index 80f34fe..6653628 100644 --- a/indexer.go +++ b/indexer.go @@ -16,19 +16,11 @@ import ( "unsafe" ) -//export indexerTransferProgress -func indexerTransferProgress(_stats *C.git_transfer_progress, ptr unsafe.Pointer) C.int { - callback, ok := pointerHandles.Get(ptr).(TransferProgressCallback) - if !ok { - return 0 - } - return C.int(callback(newTransferProgressFromC(_stats))) -} - type Indexer struct { - ptr *C.git_indexer - stats C.git_transfer_progress - callback unsafe.Pointer + ptr *C.git_indexer + stats C.git_transfer_progress + callbacks RemoteCallbacks + callbacksHandle unsafe.Pointer } func NewIndexer(packfilePath string, odb *Odb, callback TransferProgressCallback) (indexer *Indexer, err error) { @@ -42,15 +34,12 @@ func NewIndexer(packfilePath string, odb *Odb, callback TransferProgressCallback odbPtr = odb.ptr } - if callback != nil { - indexer.callback = pointerHandles.Track(callback) - } + indexer.callbacks.TransferProgressCallback = callback + indexer.callbacksHandle = pointerHandles.Track(&indexer.callbacks) - ret := C._go_git_indexer_new(&indexer.ptr, C.CString(packfilePath), 0, odbPtr, indexer.callback) + ret := C._go_git_indexer_new(&indexer.ptr, C.CString(packfilePath), 0, odbPtr, indexer.callbacksHandle) if ret < 0 { - if indexer.callback != nil { - pointerHandles.Untrack(indexer.callback) - } + pointerHandles.Untrack(indexer.callbacksHandle) return nil, MakeGitError(ret) } @@ -87,8 +76,7 @@ func (indexer *Indexer) Commit() (*Oid, error) { } func (indexer *Indexer) Free() { - if indexer.callback != nil { - pointerHandles.Untrack(indexer.callback) - } + pointerHandles.Untrack(indexer.callbacksHandle) + runtime.SetFinalizer(indexer, nil) C.git_indexer_free(indexer.ptr) } diff --git a/odb.go b/odb.go index 248cb55..74a09be 100644 --- a/odb.go +++ b/odb.go @@ -6,6 +6,10 @@ package git extern int git_odb_backend_one_pack(git_odb_backend **out, const char *index_file); extern int _go_git_odb_foreach(git_odb *db, void *payload); extern void _go_git_odb_backend_free(git_odb_backend *backend); +extern int _go_git_odb_write_pack(git_odb_writepack **out, git_odb *db, void *progress_payload); +extern int _go_git_odb_writepack_append(git_odb_writepack *writepack, const void *, size_t, git_transfer_progress *); +extern int _go_git_odb_writepack_commit(git_odb_writepack *writepack, git_transfer_progress *); +extern void _go_git_odb_writepack_free(git_odb_writepack *writepack); */ import "C" import ( @@ -229,6 +233,30 @@ func (v *Odb) NewWriteStream(size int64, otype ObjectType) (*OdbWriteStream, err return stream, nil } +// NewWritePack opens a stream for writing a pack file to the ODB. If the ODB +// layer understands pack files, then the given packfile will likely be +// streamed directly to disk (and a corresponding index created). If the ODB +// layer does not understand pack files, the objects will be stored in whatever +// format the ODB layer uses. +func (v *Odb) NewWritePack(callback TransferProgressCallback) (*OdbWritepack, error) { + writepack := new(OdbWritepack) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + writepack.callbacks.TransferProgressCallback = callback + writepack.callbacksHandle = pointerHandles.Track(&writepack.callbacks) + + ret := C._go_git_odb_write_pack(&writepack.ptr, v.ptr, writepack.callbacksHandle) + if ret < 0 { + pointerHandles.Untrack(writepack.callbacksHandle) + return nil, MakeGitError(ret) + } + + runtime.SetFinalizer(writepack, (*OdbWritepack).Free) + return writepack, nil +} + func (v *OdbBackend) Free() { C._go_git_odb_backend_free(v.ptr) } @@ -342,3 +370,52 @@ func (stream *OdbWriteStream) Free() { runtime.SetFinalizer(stream, nil) C.git_odb_stream_free(stream.ptr) } + +func odbWritepackTransferProgress(_stats *C.git_transfer_progress, ptr unsafe.Pointer) C.int { + callback, ok := pointerHandles.Get(ptr).(TransferProgressCallback) + if !ok { + return 0 + } + return C.int(callback(newTransferProgressFromC(_stats))) +} + +type OdbWritepack struct { + ptr *C.git_odb_writepack + stats C.git_transfer_progress + callbacks RemoteCallbacks + callbacksHandle unsafe.Pointer +} + +func (writepack *OdbWritepack) Write(data []byte) (int, error) { + header := (*reflect.SliceHeader)(unsafe.Pointer(&data)) + ptr := unsafe.Pointer(header.Data) + size := C.size_t(header.Len) + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C._go_git_odb_writepack_append(writepack.ptr, ptr, size, &writepack.stats) + if ret < 0 { + return 0, MakeGitError(ret) + } + + return len(data), nil +} + +func (writepack *OdbWritepack) Commit() error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + ret := C._go_git_odb_writepack_commit(writepack.ptr, &writepack.stats) + if ret < 0 { + return MakeGitError(ret) + } + + return nil +} + +func (writepack *OdbWritepack) Free() { + pointerHandles.Untrack(writepack.callbacksHandle) + runtime.SetFinalizer(writepack, nil) + C._go_git_odb_writepack_free(writepack.ptr) +} diff --git a/wrapper.c b/wrapper.c index d0ca4b6..dd35f88 100644 --- a/wrapper.c +++ b/wrapper.c @@ -174,9 +174,29 @@ void _go_git_writestream_free(git_writestream *stream) stream->free(stream); } +int _go_git_odb_write_pack(git_odb_writepack **out, git_odb *db, void *progress_payload) +{ + return git_odb_write_pack(out, db, (git_transfer_progress_cb)transferProgressCallback, progress_payload); +} + +int _go_git_odb_writepack_append(git_odb_writepack *writepack, const void *data, size_t size, git_transfer_progress *stats) +{ + return writepack->append(writepack, data, size, stats); +} + +int _go_git_odb_writepack_commit(git_odb_writepack *writepack, git_transfer_progress *stats) +{ + return writepack->commit(writepack, stats); +} + +void _go_git_odb_writepack_free(git_odb_writepack *writepack) +{ + writepack->free(writepack); +} + int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload) { - return git_indexer_new(out, path, mode, odb, (git_transfer_progress_cb)&indexerTransferProgress, progress_cb_payload); + return git_indexer_new(out, path, mode, odb, (git_transfer_progress_cb)transferProgressCallback, progress_cb_payload); } /* EOF */ -- 2.45.2 From 04038d337843a4c096e80cc3a23f9a4896c8861d Mon Sep 17 00:00:00 2001 From: lhchavez Date: Fri, 23 Dec 2016 19:53:33 -0800 Subject: [PATCH 3/5] Add documentation --- indexer.go | 10 ++++++++++ odb.go | 1 + 2 files changed, 11 insertions(+) diff --git a/indexer.go b/indexer.go index 6653628..de486e3 100644 --- a/indexer.go +++ b/indexer.go @@ -16,6 +16,8 @@ import ( "unsafe" ) +// Indexer can post-process packfiles and create an .idx file for efficient +// lookup. type Indexer struct { ptr *C.git_indexer stats C.git_transfer_progress @@ -23,6 +25,7 @@ type Indexer struct { callbacksHandle unsafe.Pointer } +// NewIndexer creates a new indexer instance. func NewIndexer(packfilePath string, odb *Odb, callback TransferProgressCallback) (indexer *Indexer, err error) { indexer = new(Indexer) @@ -47,6 +50,7 @@ func NewIndexer(packfilePath string, odb *Odb, callback TransferProgressCallback return indexer, nil } +// Write adds data to the indexer. func (indexer *Indexer) Write(data []byte) (int, error) { header := (*reflect.SliceHeader)(unsafe.Pointer(&data)) ptr := unsafe.Pointer(header.Data) @@ -63,6 +67,11 @@ func (indexer *Indexer) Write(data []byte) (int, error) { return len(data), nil } +// Commit finalizes the pack and index. It resolves any pending deltas and +// writes out the index file. +// +// It also returns the packfile's hash. A packfile's name is derived from the +// sorted hashing of all object names. func (indexer *Indexer) Commit() (*Oid, error) { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -75,6 +84,7 @@ func (indexer *Indexer) Commit() (*Oid, error) { return newOidFromC(C.git_indexer_hash(indexer.ptr)), nil } +// Free frees the indexer and its resources. func (indexer *Indexer) Free() { pointerHandles.Untrack(indexer.callbacksHandle) runtime.SetFinalizer(indexer, nil) diff --git a/odb.go b/odb.go index 74a09be..c68ec44 100644 --- a/odb.go +++ b/odb.go @@ -379,6 +379,7 @@ func odbWritepackTransferProgress(_stats *C.git_transfer_progress, ptr unsafe.Po return C.int(callback(newTransferProgressFromC(_stats))) } +// OdbWritepack is a stream to write a packfile to the ODB. type OdbWritepack struct { ptr *C.git_odb_writepack stats C.git_transfer_progress -- 2.45.2 From f783fb1d275b6749558735e458f0bdc239df9da4 Mon Sep 17 00:00:00 2001 From: lhchavez Date: Fri, 23 Dec 2016 19:55:17 -0800 Subject: [PATCH 4/5] Remove an unused function --- odb.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/odb.go b/odb.go index c68ec44..f6584ce 100644 --- a/odb.go +++ b/odb.go @@ -371,14 +371,6 @@ func (stream *OdbWriteStream) Free() { C.git_odb_stream_free(stream.ptr) } -func odbWritepackTransferProgress(_stats *C.git_transfer_progress, ptr unsafe.Pointer) C.int { - callback, ok := pointerHandles.Get(ptr).(TransferProgressCallback) - if !ok { - return 0 - } - return C.int(callback(newTransferProgressFromC(_stats))) -} - // OdbWritepack is a stream to write a packfile to the ODB. type OdbWritepack struct { ptr *C.git_odb_writepack -- 2.45.2 From 5d0a4c752a74258a5f42e40fccd2908ac4e336b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 9 Jan 2017 22:13:18 +0000 Subject: [PATCH 5/5] Bump vendored libgit2 to ee89941fa --- vendor/libgit2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/libgit2 b/vendor/libgit2 index ae5838f..ee89941 160000 --- a/vendor/libgit2 +++ b/vendor/libgit2 @@ -1 +1 @@ -Subproject commit ae5838f118a4819e608990a815bf8fc482be5772 +Subproject commit ee89941fa2b861332defb5e3a8ce12c23c496ed7 -- 2.45.2