1
1
mirror of https://github.com/go-gitea/gitea synced 2025-07-22 18:28:37 +00:00

Add pure SSH LFS support (#31516)

Fixes #17554
/claim #17554

Docs PR https://gitea.com/gitea/docs/pulls/49

To test, run pushes like: `GIT_TRACE=1` git push. The trace output
should mention "pure SSH connection".
This commit is contained in:
ConcurrentCrab
2024-09-27 19:57:37 +05:30
committed by GitHub
parent fdb1df9eca
commit 8a9fd7f771
13 changed files with 945 additions and 53 deletions

View File

@@ -0,0 +1,301 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package backend
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
// Version is the git-lfs-transfer protocol version number.
const Version = "1"
// Capabilities is a list of Git LFS capabilities supported by this package.
var Capabilities = []string{
"version=" + Version,
"locking",
}
var _ transfer.Backend = &GiteaBackend{}
// GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API
type GiteaBackend struct {
ctx context.Context
server *url.URL
op string
token string
itoken string
logger transfer.Logger
}
func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (transfer.Backend, error) {
// runServ guarantees repo will be in form [owner]/[name].git
server, err := url.Parse(setting.LocalURL)
if err != nil {
return nil, err
}
server = server.JoinPath("api/internal/repo", repo, "info/lfs")
return &GiteaBackend{ctx: ctx, server: server, op: op, token: token, itoken: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil
}
// Batch implements transfer.Backend
func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args transfer.Args) ([]transfer.BatchItem, error) {
reqBody := lfs.BatchRequest{Operation: g.op}
if transfer, ok := args[argTransfer]; ok {
reqBody.Transfers = []string{transfer}
}
if ref, ok := args[argRefname]; ok {
reqBody.Ref = &lfs.Reference{Name: ref}
}
reqBody.Objects = make([]lfs.Pointer, len(pointers))
for i := range pointers {
reqBody.Objects[i].Oid = pointers[i].Oid
reqBody.Objects[i].Size = pointers[i].Size
}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
g.logger.Log("json marshal error", err)
return nil, err
}
url := g.server.JoinPath("objects/batch").String()
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
return nil, err
}
if resp.StatusCode != http.StatusOK {
g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode))
return nil, statusCodeToErr(resp.StatusCode)
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
g.logger.Log("http read error", err)
return nil, err
}
var respBody lfs.BatchResponse
err = json.Unmarshal(respBytes, &respBody)
if err != nil {
g.logger.Log("json umarshal error", err)
return nil, err
}
// rebuild slice, we can't rely on order in resp being the same as req
pointers = pointers[:0]
opNum := opMap[g.op]
for _, obj := range respBody.Objects {
pointer := transfer.Pointer{Oid: obj.Pointer.Oid, Size: obj.Pointer.Size}
item := transfer.BatchItem{Pointer: pointer, Args: map[string]string{}}
switch opNum {
case opDownload:
if action, ok := obj.Actions[actionDownload]; ok {
item.Present = true
idMap := obj.Actions
idMapBytes, err := json.Marshal(idMap)
if err != nil {
g.logger.Log("json marshal error", err)
return nil, err
}
idMapStr := base64.StdEncoding.EncodeToString(idMapBytes)
item.Args[argID] = idMapStr
if authHeader, ok := action.Header[headerAuthorisation]; ok {
authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader))
item.Args[argToken] = authHeaderB64
}
if action.ExpiresAt != nil {
item.Args[argExpiresAt] = action.ExpiresAt.String()
}
} else {
// must be an error, but the SSH protocol can't propagate individual errors
g.logger.Log("object not found", obj.Pointer.Oid, obj.Pointer.Size)
item.Present = false
}
case opUpload:
if action, ok := obj.Actions[actionUpload]; ok {
item.Present = false
idMap := obj.Actions
idMapBytes, err := json.Marshal(idMap)
if err != nil {
g.logger.Log("json marshal error", err)
return nil, err
}
idMapStr := base64.StdEncoding.EncodeToString(idMapBytes)
item.Args[argID] = idMapStr
if authHeader, ok := action.Header[headerAuthorisation]; ok {
authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader))
item.Args[argToken] = authHeaderB64
}
if action.ExpiresAt != nil {
item.Args[argExpiresAt] = action.ExpiresAt.String()
}
} else {
item.Present = true
}
}
pointers = append(pointers, item)
}
return pointers, nil
}
// Download implements transfer.Backend. The returned reader must be closed by the
// caller.
func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, int64, error) {
idMapStr, exists := args[argID]
if !exists {
return nil, 0, ErrMissingID
}
idMapBytes, err := base64.StdEncoding.DecodeString(idMapStr)
if err != nil {
g.logger.Log("base64 decode error", err)
return nil, 0, transfer.ErrCorruptData
}
idMap := map[string]*lfs.Link{}
err = json.Unmarshal(idMapBytes, &idMap)
if err != nil {
g.logger.Log("json unmarshal error", err)
return nil, 0, transfer.ErrCorruptData
}
action, exists := idMap[actionDownload]
if !exists {
g.logger.Log("argument id incorrect")
return nil, 0, transfer.ErrCorruptData
}
url := action.Href
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeOctetStream,
}
req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil)
resp, err := req.Response()
if err != nil {
return nil, 0, err
}
if resp.StatusCode != http.StatusOK {
return nil, 0, statusCodeToErr(resp.StatusCode)
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, 0, err
}
respSize := int64(len(respBytes))
respBuf := io.NopCloser(bytes.NewBuffer(respBytes))
return respBuf, respSize, nil
}
// StartUpload implements transfer.Backend.
func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer.Args) error {
idMapStr, exists := args[argID]
if !exists {
return ErrMissingID
}
idMapBytes, err := base64.StdEncoding.DecodeString(idMapStr)
if err != nil {
g.logger.Log("base64 decode error", err)
return transfer.ErrCorruptData
}
idMap := map[string]*lfs.Link{}
err = json.Unmarshal(idMapBytes, &idMap)
if err != nil {
g.logger.Log("json unmarshal error", err)
return transfer.ErrCorruptData
}
action, exists := idMap[actionUpload]
if !exists {
g.logger.Log("argument id incorrect")
return transfer.ErrCorruptData
}
url := action.Href
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerContentType: mimeOctetStream,
headerContentLength: strconv.FormatInt(size, 10),
}
reqBytes, err := io.ReadAll(r)
if err != nil {
return err
}
req := newInternalRequest(g.ctx, url, http.MethodPut, headers, reqBytes)
resp, err := req.Response()
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return statusCodeToErr(resp.StatusCode)
}
return nil
}
// Verify implements transfer.Backend.
func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (transfer.Status, error) {
reqBody := lfs.Pointer{Oid: oid, Size: size}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
return transfer.NewStatus(transfer.StatusInternalServerError), err
}
idMapStr, exists := args[argID]
if !exists {
return transfer.NewStatus(transfer.StatusBadRequest, "missing argument: id"), ErrMissingID
}
idMapBytes, err := base64.StdEncoding.DecodeString(idMapStr)
if err != nil {
g.logger.Log("base64 decode error", err)
return transfer.NewStatus(transfer.StatusBadRequest, "corrupt argument: id"), transfer.ErrCorruptData
}
idMap := map[string]*lfs.Link{}
err = json.Unmarshal(idMapBytes, &idMap)
if err != nil {
g.logger.Log("json unmarshal error", err)
return transfer.NewStatus(transfer.StatusBadRequest, "corrupt argument: id"), transfer.ErrCorruptData
}
action, exists := idMap[actionVerify]
if !exists {
// the server sent no verify action
return transfer.SuccessStatus(), nil
}
url := action.Href
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
return transfer.NewStatus(transfer.StatusInternalServerError), err
}
if resp.StatusCode != http.StatusOK {
return transfer.NewStatus(uint32(resp.StatusCode), http.StatusText(resp.StatusCode)), statusCodeToErr(resp.StatusCode)
}
return transfer.SuccessStatus(), nil
}
// LockBackend implements transfer.Backend.
func (g *GiteaBackend) LockBackend(_ transfer.Args) transfer.LockBackend {
return newGiteaLockBackend(g)
}

View File

@@ -0,0 +1,296 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package backend
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"time"
"code.gitea.io/gitea/modules/json"
lfslock "code.gitea.io/gitea/modules/structs"
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
var _ transfer.LockBackend = &giteaLockBackend{}
type giteaLockBackend struct {
ctx context.Context
g *GiteaBackend
server *url.URL
token string
itoken string
logger transfer.Logger
}
func newGiteaLockBackend(g *GiteaBackend) transfer.LockBackend {
server := g.server.JoinPath("locks")
return &giteaLockBackend{ctx: g.ctx, g: g, server: server, token: g.token, itoken: g.itoken, logger: g.logger}
}
// Create implements transfer.LockBackend
func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) {
reqBody := lfslock.LFSLockRequest{Path: path}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
g.logger.Log("json marshal error", err)
return nil, err
}
url := g.server.String()
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
return nil, err
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
g.logger.Log("http read error", err)
return nil, err
}
if resp.StatusCode != http.StatusCreated {
g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode))
return nil, statusCodeToErr(resp.StatusCode)
}
var respBody lfslock.LFSLockResponse
err = json.Unmarshal(respBytes, &respBody)
if err != nil {
g.logger.Log("json umarshal error", err)
return nil, err
}
if respBody.Lock == nil {
g.logger.Log("api returned nil lock")
return nil, fmt.Errorf("api returned nil lock")
}
respLock := respBody.Lock
owner := userUnknown
if respLock.Owner != nil {
owner = respLock.Owner.Name
}
lock := newGiteaLock(g, respLock.ID, respLock.Path, respLock.LockedAt, owner)
return lock, nil
}
// Unlock implements transfer.LockBackend
func (g *giteaLockBackend) Unlock(lock transfer.Lock) error {
reqBody := lfslock.LFSLockDeleteRequest{}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
g.logger.Log("json marshal error", err)
return err
}
url := g.server.JoinPath(lock.ID(), "unlock").String()
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode))
return statusCodeToErr(resp.StatusCode)
}
// no need to read response
return nil
}
// FromPath implements transfer.LockBackend
func (g *giteaLockBackend) FromPath(path string) (transfer.Lock, error) {
v := url.Values{
argPath: []string{path},
}
respLocks, _, err := g.queryLocks(v)
if err != nil {
return nil, err
}
if len(respLocks) == 0 {
return nil, transfer.ErrNotFound
}
return respLocks[0], nil
}
// FromID implements transfer.LockBackend
func (g *giteaLockBackend) FromID(id string) (transfer.Lock, error) {
v := url.Values{
argID: []string{id},
}
respLocks, _, err := g.queryLocks(v)
if err != nil {
return nil, err
}
if len(respLocks) == 0 {
return nil, transfer.ErrNotFound
}
return respLocks[0], nil
}
// Range implements transfer.LockBackend
func (g *giteaLockBackend) Range(cursor string, limit int, iter func(transfer.Lock) error) (string, error) {
v := url.Values{
argLimit: []string{strconv.FormatInt(int64(limit), 10)},
}
if cursor != "" {
v[argCursor] = []string{cursor}
}
respLocks, cursor, err := g.queryLocks(v)
if err != nil {
return "", err
}
for _, lock := range respLocks {
err := iter(lock)
if err != nil {
return "", err
}
}
return cursor, nil
}
func (g *giteaLockBackend) queryLocks(v url.Values) ([]transfer.Lock, string, error) {
urlq := g.server.JoinPath() // get a copy
urlq.RawQuery = v.Encode()
url := urlq.String()
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
return nil, "", err
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
g.logger.Log("http read error", err)
return nil, "", err
}
if resp.StatusCode != http.StatusOK {
g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode))
return nil, "", statusCodeToErr(resp.StatusCode)
}
var respBody lfslock.LFSLockList
err = json.Unmarshal(respBytes, &respBody)
if err != nil {
g.logger.Log("json umarshal error", err)
return nil, "", err
}
respLocks := make([]transfer.Lock, 0, len(respBody.Locks))
for _, respLock := range respBody.Locks {
owner := userUnknown
if respLock.Owner != nil {
owner = respLock.Owner.Name
}
lock := newGiteaLock(g, respLock.ID, respLock.Path, respLock.LockedAt, owner)
respLocks = append(respLocks, lock)
}
return respLocks, respBody.Next, nil
}
var _ transfer.Lock = &giteaLock{}
type giteaLock struct {
g *giteaLockBackend
id string
path string
lockedAt time.Time
owner string
}
func newGiteaLock(g *giteaLockBackend, id, path string, lockedAt time.Time, owner string) transfer.Lock {
return &giteaLock{g: g, id: id, path: path, lockedAt: lockedAt, owner: owner}
}
// Unlock implements transfer.Lock
func (g *giteaLock) Unlock() error {
return g.g.Unlock(g)
}
// ID implements transfer.Lock
func (g *giteaLock) ID() string {
return g.id
}
// Path implements transfer.Lock
func (g *giteaLock) Path() string {
return g.path
}
// FormattedTimestamp implements transfer.Lock
func (g *giteaLock) FormattedTimestamp() string {
return g.lockedAt.UTC().Format(time.RFC3339)
}
// OwnerName implements transfer.Lock
func (g *giteaLock) OwnerName() string {
return g.owner
}
func (g *giteaLock) CurrentUser() (string, error) {
return userSelf, nil
}
// AsLockSpec implements transfer.Lock
func (g *giteaLock) AsLockSpec(ownerID bool) ([]string, error) {
msgs := []string{
fmt.Sprintf("lock %s", g.ID()),
fmt.Sprintf("path %s %s", g.ID(), g.Path()),
fmt.Sprintf("locked-at %s %s", g.ID(), g.FormattedTimestamp()),
fmt.Sprintf("ownername %s %s", g.ID(), g.OwnerName()),
}
if ownerID {
user, err := g.CurrentUser()
if err != nil {
return nil, fmt.Errorf("error getting current user: %w", err)
}
who := "theirs"
if user == g.OwnerName() {
who = "ours"
}
msgs = append(msgs, fmt.Sprintf("owner %s %s", g.ID(), who))
}
return msgs, nil
}
// AsArguments implements transfer.Lock
func (g *giteaLock) AsArguments() []string {
return []string{
fmt.Sprintf("id=%s", g.ID()),
fmt.Sprintf("path=%s", g.Path()),
fmt.Sprintf("locked-at=%s", g.FormattedTimestamp()),
fmt.Sprintf("ownername=%s", g.OwnerName()),
}
}

View File

@@ -0,0 +1,141 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package backend
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"time"
"code.gitea.io/gitea/modules/httplib"
"code.gitea.io/gitea/modules/proxyprotocol"
"code.gitea.io/gitea/modules/setting"
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
// HTTP headers
const (
headerAccept = "Accept"
headerAuthorisation = "Authorization"
headerAuthX = "X-Auth"
headerContentType = "Content-Type"
headerContentLength = "Content-Length"
)
// MIME types
const (
mimeGitLFS = "application/vnd.git-lfs+json"
mimeOctetStream = "application/octet-stream"
)
// SSH protocol action keys
const (
actionDownload = "download"
actionUpload = "upload"
actionVerify = "verify"
)
// SSH protocol argument keys
const (
argCursor = "cursor"
argExpiresAt = "expires-at"
argID = "id"
argLimit = "limit"
argPath = "path"
argRefname = "refname"
argToken = "token"
argTransfer = "transfer"
)
// Default username constants
const (
userSelf = "(self)"
userUnknown = "(unknown)"
)
// Operations enum
const (
opNone = iota
opDownload
opUpload
)
var opMap = map[string]int{
"download": opDownload,
"upload": opUpload,
}
var ErrMissingID = fmt.Errorf("%w: missing id arg", transfer.ErrMissingData)
func statusCodeToErr(code int) error {
switch code {
case http.StatusBadRequest:
return transfer.ErrParseError
case http.StatusConflict:
return transfer.ErrConflict
case http.StatusForbidden:
return transfer.ErrForbidden
case http.StatusNotFound:
return transfer.ErrNotFound
case http.StatusUnauthorized:
return transfer.ErrUnauthorized
default:
return fmt.Errorf("server returned status %v: %v", code, http.StatusText(code))
}
}
func newInternalRequest(ctx context.Context, url, method string, headers map[string]string, body []byte) *httplib.Request {
req := httplib.NewRequest(url, method).
SetContext(ctx).
SetTimeout(10*time.Second, 60*time.Second).
SetTLSClientConfig(&tls.Config{
InsecureSkipVerify: true,
})
if setting.Protocol == setting.HTTPUnix {
req.SetTransport(&http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
var d net.Dialer
conn, err := d.DialContext(ctx, "unix", setting.HTTPAddr)
if err != nil {
return conn, err
}
if setting.LocalUseProxyProtocol {
if err = proxyprotocol.WriteLocalHeader(conn); err != nil {
_ = conn.Close()
return nil, err
}
}
return conn, err
},
})
} else if setting.LocalUseProxyProtocol {
req.SetTransport(&http.Transport{
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
var d net.Dialer
conn, err := d.DialContext(ctx, network, address)
if err != nil {
return conn, err
}
if err = proxyprotocol.WriteLocalHeader(conn); err != nil {
_ = conn.Close()
return nil, err
}
return conn, err
},
})
}
for k, v := range headers {
req.Header(k, v)
}
req.Body(body)
return req
}

View File

@@ -0,0 +1,21 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package lfstransfer
import (
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
var _ transfer.Logger = (*GiteaLogger)(nil)
// noop logger for passing into transfer
type GiteaLogger struct{}
func newLogger() transfer.Logger {
return &GiteaLogger{}
}
// Log implements transfer.Logger
func (g *GiteaLogger) Log(msg string, itms ...any) {
}

View File

@@ -0,0 +1,42 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package lfstransfer
import (
"context"
"fmt"
"os"
"code.gitea.io/gitea/modules/lfstransfer/backend"
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
func Main(ctx context.Context, repo, verb, token string) error {
logger := newLogger()
pktline := transfer.NewPktline(os.Stdin, os.Stdout, logger)
giteaBackend, err := backend.New(ctx, repo, verb, token, logger)
if err != nil {
return err
}
for _, cap := range backend.Capabilities {
if err := pktline.WritePacketText(cap); err != nil {
logger.Log("error sending capability due to error:", err)
}
}
if err := pktline.WriteFlush(); err != nil {
logger.Log("error flushing capabilities:", err)
}
p := transfer.NewProcessor(pktline, giteaBackend, logger)
defer logger.Log("done processing commands")
switch verb {
case "upload":
return p.ProcessCommands(transfer.UploadOperation)
case "download":
return p.ProcessCommands(transfer.DownloadOperation)
default:
return fmt.Errorf("unknown operation %q", verb)
}
}

View File

@@ -13,6 +13,7 @@ import (
// LFS represents the configuration for Git LFS
var LFS = struct {
StartServer bool `ini:"LFS_START_SERVER"`
AllowPureSSH bool `ini:"LFS_ALLOW_PURE_SSH"`
JWTSecretBytes []byte `ini:"-"`
HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`