Add support for indexers and alternate odb packfiles
This allows for implementations of git servers written in Go.
This commit is contained in:
lhchavez
parent
37f732a833
commit
05bc5e36ff
|
|
@ -0,0 +1,99 @@
|
|||
package git
|
||||
|
||||
/*
|
||||
#include <git2.h>
|
||||
|
||||
extern const git_oid * git_indexer_hash(const git_indexer *idx);
|
||||
extern int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats);
|
||||
extern int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats);
|
||||
extern int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload);
|
||||
extern void git_indexer_free(git_indexer *idx);
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"reflect"
|
||||
"runtime"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Indexer can post-process packfiles and create an .idx file for efficient
|
||||
// lookup.
|
||||
type Indexer struct {
|
||||
ptr *C.git_indexer
|
||||
stats C.git_transfer_progress
|
||||
callbacks RemoteCallbacks
|
||||
callbacksHandle unsafe.Pointer
|
||||
}
|
||||
|
||||
// NewIndexer creates a new indexer instance.
|
||||
func NewIndexer(packfilePath string, odb *Odb, callback TransferProgressCallback) (indexer *Indexer, err error) {
|
||||
indexer = new(Indexer)
|
||||
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
var odbPtr *C.git_odb = nil
|
||||
if odb != nil {
|
||||
odbPtr = odb.ptr
|
||||
}
|
||||
|
||||
indexer.callbacks.TransferProgressCallback = callback
|
||||
indexer.callbacksHandle = pointerHandles.Track(&indexer.callbacks)
|
||||
|
||||
cstr := C.CString(packfilePath)
|
||||
defer C.free(unsafe.Pointer(cstr))
|
||||
|
||||
ret := C._go_git_indexer_new(&indexer.ptr, cstr, 0, odbPtr, indexer.callbacksHandle)
|
||||
runtime.KeepAlive(odb)
|
||||
if ret < 0 {
|
||||
pointerHandles.Untrack(indexer.callbacksHandle)
|
||||
return nil, MakeGitError(ret)
|
||||
}
|
||||
|
||||
runtime.SetFinalizer(indexer, (*Indexer).Free)
|
||||
return indexer, nil
|
||||
}
|
||||
|
||||
// Write adds data to the indexer.
|
||||
func (indexer *Indexer) Write(data []byte) (int, error) {
|
||||
header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
|
||||
ptr := unsafe.Pointer(header.Data)
|
||||
size := C.size_t(header.Len)
|
||||
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
ret := C.git_indexer_append(indexer.ptr, ptr, size, &indexer.stats)
|
||||
runtime.KeepAlive(indexer)
|
||||
if ret < 0 {
|
||||
return 0, MakeGitError(ret)
|
||||
}
|
||||
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
// Commit finalizes the pack and index. It resolves any pending deltas and
|
||||
// writes out the index file.
|
||||
//
|
||||
// It also returns the packfile's hash. A packfile's name is derived from the
|
||||
// sorted hashing of all object names.
|
||||
func (indexer *Indexer) Commit() (*Oid, error) {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
ret := C.git_indexer_commit(indexer.ptr, &indexer.stats)
|
||||
if ret < 0 {
|
||||
return nil, MakeGitError(ret)
|
||||
}
|
||||
|
||||
id := newOidFromC(C.git_indexer_hash(indexer.ptr))
|
||||
runtime.KeepAlive(indexer)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Free frees the indexer and its resources.
|
||||
func (indexer *Indexer) Free() {
|
||||
pointerHandles.Untrack(indexer.callbacksHandle)
|
||||
runtime.SetFinalizer(indexer, nil)
|
||||
C.git_indexer_free(indexer.ptr)
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
package git
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var (
|
||||
// This is a packfile with three objects. The second is a delta which
|
||||
// depends on the third, which is also a delta.
|
||||
outOfOrderPack = []byte{
|
||||
0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
|
||||
0x32, 0x78, 0x9c, 0x63, 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x08, 0x76,
|
||||
0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95, 0x10,
|
||||
0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63, 0x62,
|
||||
0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x75, 0x01,
|
||||
0xd7, 0x71, 0x36, 0x66, 0xf4, 0xde, 0x82, 0x27, 0x76, 0xc7, 0x62, 0x2c,
|
||||
0x10, 0xf1, 0xb0, 0x7d, 0xe2, 0x80, 0xdc, 0x78, 0x9c, 0x63, 0x62, 0x62,
|
||||
0x62, 0xb7, 0x03, 0x00, 0x00, 0x69, 0x00, 0x4c, 0xde, 0x7d, 0xaa, 0xe4,
|
||||
0x19, 0x87, 0x58, 0x80, 0x61, 0x09, 0x9a, 0x33, 0xca, 0x7a, 0x31, 0x92,
|
||||
0x6f, 0xae, 0x66, 0x75,
|
||||
}
|
||||
)
|
||||
|
||||
func TestIndexerOutOfOrder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpPath, err := ioutil.TempDir("", "git2go")
|
||||
checkFatal(t, err)
|
||||
defer os.RemoveAll(tmpPath)
|
||||
|
||||
var finalStats TransferProgress
|
||||
idx, err := NewIndexer(tmpPath, nil, func(stats TransferProgress) ErrorCode {
|
||||
finalStats = stats
|
||||
return ErrOk
|
||||
})
|
||||
checkFatal(t, err)
|
||||
defer idx.Free()
|
||||
|
||||
_, err = idx.Write(outOfOrderPack)
|
||||
checkFatal(t, err)
|
||||
oid, err := idx.Commit()
|
||||
checkFatal(t, err)
|
||||
|
||||
// The packfile contains the hash as the last 20 bytes.
|
||||
expectedOid := NewOidFromBytes(outOfOrderPack[len(outOfOrderPack)-20:])
|
||||
if !expectedOid.Equal(oid) {
|
||||
t.Errorf("mismatched packfile hash, expected %v, got %v", expectedOid, oid)
|
||||
}
|
||||
if finalStats.TotalObjects != 3 {
|
||||
t.Errorf("mismatched transferred objects, expected 3, got %v", finalStats.TotalObjects)
|
||||
}
|
||||
if finalStats.ReceivedObjects != 3 {
|
||||
t.Errorf("mismatched received objects, expected 3, got %v", finalStats.ReceivedObjects)
|
||||
}
|
||||
if finalStats.IndexedObjects != 3 {
|
||||
t.Errorf("mismatched indexed objects, expected 3, got %v", finalStats.IndexedObjects)
|
||||
}
|
||||
|
||||
odb, err := NewOdb()
|
||||
checkFatal(t, err)
|
||||
defer odb.Free()
|
||||
|
||||
backend, err := NewOdbBackendOnePack(path.Join(tmpPath, fmt.Sprintf("pack-%s.idx", oid.String())))
|
||||
checkFatal(t, err)
|
||||
// Transfer the ownership of the backend to the odb, no freeing needed.
|
||||
err = odb.AddBackend(backend, 1)
|
||||
checkFatal(t, err)
|
||||
|
||||
packfileObjects := 0
|
||||
err = odb.ForEach(func(id *Oid) error {
|
||||
packfileObjects += 1
|
||||
return nil
|
||||
})
|
||||
checkFatal(t, err)
|
||||
if packfileObjects != 3 {
|
||||
t.Errorf("mismatched packfile objects, expected 3, got %v", packfileObjects)
|
||||
}
|
||||
|
||||
// Inspect one of the well-known objects in the packfile.
|
||||
obj, err := odb.Read(NewOidFromBytes([]byte{
|
||||
0x19, 0x10, 0x28, 0x15, 0x66, 0x3d, 0x23, 0xf8, 0xb7, 0x5a, 0x47, 0xe7,
|
||||
0xa0, 0x19, 0x65, 0xdc, 0xdc, 0x96, 0x46, 0x8c,
|
||||
}))
|
||||
checkFatal(t, err)
|
||||
defer obj.Free()
|
||||
if "foo" != string(obj.Data()) {
|
||||
t.Errorf("mismatched packfile object contents, expected foo, got %q", string(obj.Data()))
|
||||
}
|
||||
}
|
||||
103
odb.go
103
odb.go
|
|
@ -3,8 +3,13 @@ package git
|
|||
/*
|
||||
#include <git2.h>
|
||||
|
||||
extern int git_odb_backend_one_pack(git_odb_backend **out, const char *index_file);
|
||||
extern int _go_git_odb_foreach(git_odb *db, void *payload);
|
||||
extern void _go_git_odb_backend_free(git_odb_backend *backend);
|
||||
extern int _go_git_odb_write_pack(git_odb_writepack **out, git_odb *db, void *progress_payload);
|
||||
extern int _go_git_odb_writepack_append(git_odb_writepack *writepack, const void *, size_t, git_transfer_progress *);
|
||||
extern int _go_git_odb_writepack_commit(git_odb_writepack *writepack, git_transfer_progress *);
|
||||
extern void _go_git_odb_writepack_free(git_odb_writepack *writepack);
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
|
|
@ -42,8 +47,20 @@ func NewOdbBackendFromC(ptr unsafe.Pointer) (backend *OdbBackend) {
|
|||
return backend
|
||||
}
|
||||
|
||||
func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {
|
||||
func (v *Odb) AddAlternate(backend *OdbBackend, priority int) (err error) {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
ret := C.git_odb_add_alternate(v.ptr, backend.ptr, C.int(priority))
|
||||
runtime.KeepAlive(v)
|
||||
if ret < 0 {
|
||||
backend.Free()
|
||||
return MakeGitError(ret)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
|
|
@ -56,6 +73,21 @@ func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {
|
|||
return nil
|
||||
}
|
||||
|
||||
func NewOdbBackendOnePack(packfileIndexPath string) (backend *OdbBackend, err error) {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
cstr := C.CString(packfileIndexPath)
|
||||
defer C.free(unsafe.Pointer(cstr))
|
||||
|
||||
var odbOnePack *C.git_odb_backend = nil
|
||||
ret := C.git_odb_backend_one_pack(&odbOnePack, cstr)
|
||||
if ret < 0 {
|
||||
return nil, MakeGitError(ret)
|
||||
}
|
||||
return NewOdbBackendFromC(unsafe.Pointer(odbOnePack)), nil
|
||||
}
|
||||
|
||||
func (v *Odb) ReadHeader(oid *Oid) (uint64, ObjectType, error) {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
|
@ -231,6 +263,31 @@ func (v *Odb) NewWriteStream(size int64, otype ObjectType) (*OdbWriteStream, err
|
|||
return stream, nil
|
||||
}
|
||||
|
||||
// NewWritePack opens a stream for writing a pack file to the ODB. If the ODB
|
||||
// layer understands pack files, then the given packfile will likely be
|
||||
// streamed directly to disk (and a corresponding index created). If the ODB
|
||||
// layer does not understand pack files, the objects will be stored in whatever
|
||||
// format the ODB layer uses.
|
||||
func (v *Odb) NewWritePack(callback TransferProgressCallback) (*OdbWritepack, error) {
|
||||
writepack := new(OdbWritepack)
|
||||
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
writepack.callbacks.TransferProgressCallback = callback
|
||||
writepack.callbacksHandle = pointerHandles.Track(&writepack.callbacks)
|
||||
|
||||
ret := C._go_git_odb_write_pack(&writepack.ptr, v.ptr, writepack.callbacksHandle)
|
||||
runtime.KeepAlive(v)
|
||||
if ret < 0 {
|
||||
pointerHandles.Untrack(writepack.callbacksHandle)
|
||||
return nil, MakeGitError(ret)
|
||||
}
|
||||
|
||||
runtime.SetFinalizer(writepack, (*OdbWritepack).Free)
|
||||
return writepack, nil
|
||||
}
|
||||
|
||||
func (v *OdbBackend) Free() {
|
||||
C._go_git_odb_backend_free(v.ptr)
|
||||
}
|
||||
|
|
@ -360,3 +417,47 @@ func (stream *OdbWriteStream) Free() {
|
|||
runtime.SetFinalizer(stream, nil)
|
||||
C.git_odb_stream_free(stream.ptr)
|
||||
}
|
||||
|
||||
// OdbWritepack is a stream to write a packfile to the ODB.
|
||||
type OdbWritepack struct {
|
||||
ptr *C.git_odb_writepack
|
||||
stats C.git_transfer_progress
|
||||
callbacks RemoteCallbacks
|
||||
callbacksHandle unsafe.Pointer
|
||||
}
|
||||
|
||||
func (writepack *OdbWritepack) Write(data []byte) (int, error) {
|
||||
header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
|
||||
ptr := unsafe.Pointer(header.Data)
|
||||
size := C.size_t(header.Len)
|
||||
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
ret := C._go_git_odb_writepack_append(writepack.ptr, ptr, size, &writepack.stats)
|
||||
runtime.KeepAlive(writepack)
|
||||
if ret < 0 {
|
||||
return 0, MakeGitError(ret)
|
||||
}
|
||||
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
func (writepack *OdbWritepack) Commit() error {
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
|
||||
ret := C._go_git_odb_writepack_commit(writepack.ptr, &writepack.stats)
|
||||
runtime.KeepAlive(writepack)
|
||||
if ret < 0 {
|
||||
return MakeGitError(ret)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (writepack *OdbWritepack) Free() {
|
||||
pointerHandles.Untrack(writepack.callbacksHandle)
|
||||
runtime.SetFinalizer(writepack, nil)
|
||||
C._go_git_odb_writepack_free(writepack.ptr)
|
||||
}
|
||||
|
|
|
|||
34
odb_test.go
34
odb_test.go
|
|
@ -152,3 +152,37 @@ func TestOdbForeach(t *testing.T) {
|
|||
t.Fatalf("Odb.ForEach() did not return the expected error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOdbWritepack(t *testing.T) {
|
||||
t.Parallel()
|
||||
repo := createTestRepo(t)
|
||||
defer cleanupTestRepo(t, repo)
|
||||
|
||||
_, _ = seedTestRepo(t, repo)
|
||||
|
||||
odb, err := repo.Odb()
|
||||
checkFatal(t, err)
|
||||
|
||||
var finalStats TransferProgress
|
||||
writepack, err := odb.NewWritePack(func(stats TransferProgress) ErrorCode {
|
||||
finalStats = stats
|
||||
return ErrOk
|
||||
})
|
||||
checkFatal(t, err)
|
||||
defer writepack.Free()
|
||||
|
||||
_, err = writepack.Write(outOfOrderPack)
|
||||
checkFatal(t, err)
|
||||
err = writepack.Commit()
|
||||
checkFatal(t, err)
|
||||
|
||||
if finalStats.TotalObjects != 3 {
|
||||
t.Errorf("mismatched transferred objects, expected 3, got %v", finalStats.TotalObjects)
|
||||
}
|
||||
if finalStats.ReceivedObjects != 3 {
|
||||
t.Errorf("mismatched received objects, expected 3, got %v", finalStats.ReceivedObjects)
|
||||
}
|
||||
if finalStats.IndexedObjects != 3 {
|
||||
t.Errorf("mismatched indexed objects, expected 3, got %v", finalStats.IndexedObjects)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
28
wrapper.c
28
wrapper.c
|
|
@ -185,4 +185,32 @@ git_credtype_t _go_git_cred_credtype(git_cred *cred) {
|
|||
return cred->credtype;
|
||||
}
|
||||
|
||||
int _go_git_odb_write_pack(git_odb_writepack **out, git_odb *db, void *progress_payload)
|
||||
{
|
||||
return git_odb_write_pack(out, db, (git_transfer_progress_cb)transferProgressCallback, progress_payload);
|
||||
}
|
||||
|
||||
int _go_git_odb_writepack_append(git_odb_writepack *writepack, const void *data, size_t size, git_transfer_progress *stats)
|
||||
{
|
||||
return writepack->append(writepack, data, size, stats);
|
||||
}
|
||||
|
||||
int _go_git_odb_writepack_commit(git_odb_writepack *writepack, git_transfer_progress *stats)
|
||||
{
|
||||
return writepack->commit(writepack, stats);
|
||||
}
|
||||
|
||||
void _go_git_odb_writepack_free(git_odb_writepack *writepack)
|
||||
{
|
||||
writepack->free(writepack);
|
||||
}
|
||||
|
||||
int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload)
|
||||
{
|
||||
git_indexer_options indexer_options = GIT_INDEXER_OPTIONS_INIT;
|
||||
indexer_options.progress_cb = (git_transfer_progress_cb)transferProgressCallback;
|
||||
indexer_options.progress_cb_payload = progress_cb_payload;
|
||||
return git_indexer_new(out, path, mode, odb, &indexer_options);
|
||||
}
|
||||
|
||||
/* EOF */
|
||||
|
|
|
|||
Loading…
Reference in New Issue