diff --git a/handles.go b/handles.go
index f5d30f0..d27d3c3 100644
--- a/handles.go
+++ b/handles.go
@@ -1,5 +1,9 @@
 package git
 
+/*
+#include <stdlib.h>
+*/
+import "C"
 import (
 	"fmt"
 	"sync"
@@ -9,75 +13,45 @@ import (
 type HandleList struct {
 	sync.RWMutex
 	// stores the Go pointers
-	handles []interface{}
-	// Indicates which indices are in use.
-	set map[int]bool
+	handles map[unsafe.Pointer]interface{}
 }
 
 func NewHandleList() *HandleList {
 	return &HandleList{
-		handles: make([]interface{}, 5),
-		set:     make(map[int]bool),
+		handles: make(map[unsafe.Pointer]interface{}),
 	}
 }
 
-// findUnusedSlot finds the smallest-index empty space in our
-// list. You must only run this function while holding a write lock.
-func (v *HandleList) findUnusedSlot() int {
-	for i := 1; i < len(v.handles); i++ {
-		if !v.set[i] {
-			return i
-		}
-	}
-
-	// reaching here means we've run out of entries so append and
-	// return the new index, which is equal to the old length.
-	slot := len(v.handles)
-	v.handles = append(v.handles, nil)
-
-	return slot
-}
-
 // Track adds the given pointer to the list of pointers to track and
 // returns a pointer value which can be passed to C as an opaque
 // pointer.
 func (v *HandleList) Track(pointer interface{}) unsafe.Pointer {
+	handle := C.malloc(1)
+
 	v.Lock()
-
-	slot := v.findUnusedSlot()
-	v.handles[slot] = pointer
-	v.set[slot] = true
-
+	v.handles[handle] = pointer
 	v.Unlock()
 
-	return unsafe.Pointer(uintptr(slot))
+	return handle
 }
 
 // Untrack stops tracking the pointer given by the handle
 func (v *HandleList) Untrack(handle unsafe.Pointer) {
-	slot := int(uintptr(handle))
-
 	v.Lock()
-
-	v.handles[slot] = nil
-	delete(v.set, slot)
-
+	delete(v.handles, handle)
+	C.free(handle)
 	v.Unlock()
 }
 
 // Get retrieves the pointer from the given handle
 func (v *HandleList) Get(handle unsafe.Pointer) interface{} {
-	slot := int(uintptr(handle))
-
 	v.RLock()
+	defer v.RUnlock()
 
-	if !v.set[slot] {
+	ptr, ok := v.handles[handle]
+	if !ok {
 		panic(fmt.Sprintf("invalid pointer handle: %p", handle))
 	}
 
-	ptr := v.handles[slot]
-
-	v.RUnlock()
-
 	return ptr
 }