move gotd fork into repo. (#111)

- update to latest telegram layer
- remove some references to fields in tg.Entities that don't exist in
the schema
- originally added here:
https://github.com/beeper/td/commit/820929062a2ba0104397bc01235ab58a9cff780e
  - referenced here
-
https://github.com/mautrix/telegramgo/commit/124f0967ed195b5a380c9bd02e170ada9710dde3
-
https://github.com/mautrix/telegramgo/commit/4205047aab2e0639217148b5d125bfaab668bd8e

pkg/gotd/tgtest/services/config/config.go

@@ -0,0 +1,72 @@
+// Package config contains config service implementation for tgtest server.
+package config
+
+import (
+	"context"
+
+	"go.mau.fi/mautrix-telegram/pkg/gotd/bin"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tg"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tgtest"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tgtest/services"
+)
+
+// Service is a Telegram config service.
+type Service struct {
+	cfg    *tg.Config
+	cdnCfg *tg.CDNConfig
+}
+
+// NewService creates new Service.
+func NewService(cfg *tg.Config, cdnCfg *tg.CDNConfig) *Service {
+	return &Service{cfg: cfg, cdnCfg: cdnCfg}
+}
+
+func (c *Service) HelpGetCDNConfig(ctx context.Context, req *tg.HelpGetCDNConfigRequest) (*tg.CDNConfig, error) {
+	cfg := c.cdnCfg
+	return cfg, nil
+}
+
+func (c *Service) HelpGetConfig(ctx context.Context, dc int, req *tg.HelpGetConfigRequest) (*tg.Config, error) {
+	cfg := *c.cfg
+	cfg.ThisDC = dc
+	return &cfg, nil
+}
+
+// OnMessage implements tgtest.Handler.
+func (c *Service) OnMessage(server *tgtest.Server, req *tgtest.Request) error {
+	id, err := req.Buf.PeekID()
+	if err != nil {
+		return err
+	}
+
+	var (
+		decode bin.Decoder
+		result bin.Encoder
+	)
+	switch id {
+	case tg.HelpGetCDNConfigRequestTypeID:
+		cfg := c.cdnCfg
+
+		decode = &tg.HelpGetCDNConfigRequest{}
+		result = cfg
+	case tg.HelpGetConfigRequestTypeID:
+		cfg := *c.cfg
+		cfg.ThisDC = req.DC
+
+		decode = &tg.HelpGetConfigRequest{}
+		result = &cfg
+	default:
+		return services.ErrMethodNotImplemented
+	}
+
+	if err := decode.Decode(req.Buf); err != nil {
+		return err
+	}
+	return server.SendResult(req, result)
+}
+
+// Register registers service handlers.
+func (c *Service) Register(dispatcher *tgtest.Dispatcher) {
+	dispatcher.HandleFunc(tg.HelpGetCDNConfigRequestTypeID, c.OnMessage)
+	dispatcher.HandleFunc(tg.HelpGetConfigRequestTypeID, c.OnMessage)
+}

pkg/gotd/tgtest/services/doc.go

@@ -0,0 +1,2 @@
+// Package services contains some Telegram services implemented for testing.
+package services

pkg/gotd/tgtest/services/err.go

@@ -0,0 +1,16 @@
+package services
+
+import (
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tgerr"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tgtest"
+)
+
+var (
+	// ErrMethodNotImplemented denotes that method is not implemented.
+	ErrMethodNotImplemented error = tgerr.New(400, "INPUT_METHOD_INVALID")
+
+	// NotImplemented is a simple handler which returns ErrMethodNotImplemented.
+	NotImplemented tgtest.HandlerFunc = func(server *tgtest.Server, req *tgtest.Request) error {
+		return ErrMethodNotImplemented
+	}
+)

pkg/gotd/tgtest/services/file/config.go

@@ -0,0 +1,24 @@
+package file
+
+type Config struct {
+	// Storage to store files.
+	// InMemory will be used.
+	Storage Storage
+	// HashPartSize is a size of part to use in tg.FileHash.
+	HashPartSize int
+	// HashRangeSize is size of range to return in upload.getFileHashes.
+	HashRangeSize int
+}
+
+func (c *Config) setDefaults() {
+	if c.Storage == nil {
+		c.Storage = NewInMemory()
+	}
+	// Telegram usually uses this values.
+	if c.HashPartSize == 0 {
+		c.HashPartSize = 131072
+	}
+	if c.HashRangeSize == 0 {
+		c.HashRangeSize = 10
+	}
+}

pkg/gotd/tgtest/services/file/download.go

@@ -0,0 +1,136 @@
+package file
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-faster/errors"
+
+	"go.mau.fi/mautrix-telegram/pkg/gotd/crypto"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tg"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tgerr"
+)
+
+func getLocation(loc tg.InputFileLocationClass) (string, error) {
+	v, ok := loc.(interface {
+		GetLocalID() int
+		GetVolumeID() int64
+	})
+	if !ok {
+		return "", tgerr.New(400, tg.ErrFileIDInvalid)
+	}
+
+	return fmt.Sprintf("%d_%d", v.GetLocalID(), v.GetVolumeID()), nil
+}
+
+func (m *Service) openLocation(loc tg.InputFileLocationClass) (File, error) {
+	name, err := getLocation(loc)
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := m.storage.Open(name)
+	if err != nil {
+		return nil, tgerr.New(400, tg.ErrFileIDInvalid)
+	}
+
+	return f, nil
+}
+
+func (m *Service) getPart(loc tg.InputFileLocationClass, offset int64, limit int) ([]byte, error) {
+	f, err := m.openLocation(loc)
+	if err != nil {
+		return nil, err
+	}
+
+	r := make([]byte, limit)
+	n, err := f.ReadAt(r, offset)
+	if err != nil {
+		return nil, errors.Wrap(err, "read from storage")
+	}
+
+	return r[:n], nil
+}
+
+func (m *Service) UploadGetFile(ctx context.Context, request *tg.UploadGetFileRequest) (tg.UploadFileClass, error) {
+	data, err := m.getPart(request.Location, request.Offset, request.Limit)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tg.UploadFile{
+		Type:  &tg.StorageFilePartial{},
+		Mtime: 0,
+		Bytes: data,
+	}, nil
+}
+
+func countHashes(data []byte, offset int64, partSize int) []tg.FileHash {
+	actions := data
+	batchSize := partSize
+	batches := make([][]byte, 0, (len(actions)+batchSize-1)/batchSize)
+
+	for batchSize < len(actions) {
+		actions, batches = actions[batchSize:], append(batches, actions[0:batchSize:batchSize])
+	}
+	batches = append(batches, actions)
+
+	currentRange := make([]tg.FileHash, 0, 10)
+	for _, batch := range batches {
+		currentRange = append(currentRange, tg.FileHash{
+			Offset: offset,
+			Limit:  partSize,
+			Hash:   crypto.SHA256(batch),
+		})
+		offset += int64(len(batch))
+	}
+	return currentRange
+}
+
+func divAndCeil(a, b int) int {
+	r := a / b
+	if a%b != 0 {
+		r++
+	}
+
+	return r
+}
+
+// computeBatch computes hash range number for given offset.
+func computeBatch(offset int64, rangeSize, partSize int) int {
+	// Compute number of parts in partSize from offset.
+	parts := divAndCeil(int(offset+1), partSize)
+	// Compute number of hash ranges in rangeSize.
+	batches := divAndCeil(parts, rangeSize)
+
+	return batches
+}
+
+func (m *Service) UploadGetFileHashes(
+	ctx context.Context,
+	request *tg.UploadGetFileHashesRequest,
+) ([]tg.FileHash, error) {
+	f, err := m.openLocation(request.Location)
+	if err != nil {
+		return nil, err
+	}
+
+	if request.Offset >= int64(f.Size()) {
+		return nil, nil
+	}
+	partSize := m.hashPartSize
+	rangeSize := m.hashRangeSize
+	batch := computeBatch(request.Offset, rangeSize, partSize)
+
+	low := (batch - 1) * rangeSize * partSize
+	high := batch * rangeSize * partSize
+
+	r := make([]byte, high-low)
+	n, err := f.ReadAt(r, int64(low))
+	if err != nil {
+		return nil, err
+	}
+	r = r[:n]
+
+	return countHashes(r, int64(low), partSize), nil
+}

pkg/gotd/tgtest/services/file/file.go

@@ -0,0 +1,97 @@
+// Package file contains file service implementation for tgtest server.
+package file
+
+import (
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tg"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tgtest"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tgtest/services"
+)
+
+// Service is a Telegram file service.
+type Service struct {
+	storage Storage
+	// Size of part to use in tg.FileHash.
+	hashPartSize int
+	// Size of range to return in upload.getFileHashes.
+	hashRangeSize int
+}
+
+// NewService creates new file Service.
+func NewService(cfg Config) *Service {
+	cfg.setDefaults()
+
+	return &Service{
+		storage:       cfg.Storage,
+		hashPartSize:  cfg.HashPartSize,
+		hashRangeSize: cfg.HashRangeSize,
+	}
+}
+
+// OnMessage implements tgtest.Handler.
+func (m *Service) OnMessage(server *tgtest.Server, req *tgtest.Request) error {
+	id, err := req.Buf.PeekID()
+	if err != nil {
+		return err
+	}
+
+	switch id {
+	case tg.UploadGetFileRequestTypeID:
+		fileReq := tg.UploadGetFileRequest{}
+		if err := fileReq.Decode(req.Buf); err != nil {
+			return err
+		}
+
+		r, err := m.UploadGetFile(req.RequestCtx, &fileReq)
+		if err != nil {
+			return err
+		}
+
+		return server.SendResult(req, r)
+	case tg.UploadGetFileHashesRequestTypeID:
+		fileReq := tg.UploadGetFileHashesRequest{}
+		if err := fileReq.Decode(req.Buf); err != nil {
+			return err
+		}
+
+		r, err := m.UploadGetFileHashes(req.RequestCtx, &fileReq)
+		if err != nil {
+			return err
+		}
+
+		return server.SendResult(req, &tg.FileHashVector{Elems: r})
+	case tg.UploadSaveFilePartRequestTypeID:
+		fileReq := tg.UploadSaveFilePartRequest{}
+		if err := fileReq.Decode(req.Buf); err != nil {
+			return err
+		}
+
+		r, err := m.UploadSaveFilePart(req.RequestCtx, &fileReq)
+		if err != nil {
+			return err
+		}
+
+		return server.SendBool(req, r)
+	case tg.UploadSaveBigFilePartRequestTypeID:
+		fileReq := tg.UploadSaveBigFilePartRequest{}
+		if err := fileReq.Decode(req.Buf); err != nil {
+			return err
+		}
+
+		r, err := m.UploadSaveBigFilePart(req.RequestCtx, &fileReq)
+		if err != nil {
+			return err
+		}
+
+		return server.SendBool(req, r)
+	default:
+		return services.ErrMethodNotImplemented
+	}
+}
+
+// Register registers service handlers.
+func (m *Service) Register(dispatcher *tgtest.Dispatcher) {
+	dispatcher.HandleFunc(tg.UploadGetFileRequestTypeID, m.OnMessage)
+	dispatcher.HandleFunc(tg.UploadGetFileHashesRequestTypeID, m.OnMessage)
+	dispatcher.HandleFunc(tg.UploadSaveFilePartRequestTypeID, m.OnMessage)
+	dispatcher.HandleFunc(tg.UploadSaveBigFilePartRequestTypeID, m.OnMessage)
+}

pkg/gotd/tgtest/services/file/storage.go

@@ -0,0 +1,72 @@
+package file
+
+import (
+	"io"
+	"sync"
+	"sync/atomic"
+
+	"go.mau.fi/mautrix-telegram/pkg/gotd/syncio"
+)
+
+// File represents Telegram file.
+type File interface {
+	io.ReaderAt
+	io.WriterAt
+	io.Closer
+	PartSize() int
+	SetPartSize(v int)
+	Size() int
+}
+
+// Storage is an abstraction for Telegram file storage.
+type Storage interface {
+	Open(name string) (File, error)
+}
+
+type memFile struct {
+	syncio.BufWriterAt
+	partSize int32
+	_        [4]byte
+}
+
+func (m *memFile) Size() int {
+	return m.Len()
+}
+
+func (m *memFile) Close() error {
+	return nil
+}
+
+func (m *memFile) PartSize() int {
+	return int(atomic.LoadInt32(&m.partSize))
+}
+
+func (m *memFile) SetPartSize(v int) {
+	atomic.StoreInt32(&m.partSize, int32(v))
+}
+
+// InMemory is an inmemory implementation of file storage.
+type InMemory struct {
+	files    map[string]*memFile
+	filesMux sync.Mutex
+}
+
+// NewInMemory creates new InMemory.
+func NewInMemory() *InMemory {
+	return &InMemory{
+		files: map[string]*memFile{},
+	}
+}
+
+// Open implement Storage.
+func (i *InMemory) Open(name string) (File, error) {
+	i.filesMux.Lock()
+	defer i.filesMux.Unlock()
+	file, ok := i.files[name]
+	if !ok {
+		file = &memFile{}
+		i.files[name] = file
+	}
+
+	return file, nil
+}

pkg/gotd/tgtest/services/file/upload.go

@@ -0,0 +1,115 @@
+package file
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-faster/errors"
+	"go.uber.org/multierr"
+
+	"go.mau.fi/mautrix-telegram/pkg/gotd/constant"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tg"
+	"go.mau.fi/mautrix-telegram/pkg/gotd/tgerr"
+)
+
+// https://core.telegram.org/api/files#uploading-files
+const (
+	// Each part should have a sequence number, file_part, with a value ranging from 0 to 3,999.
+	uploadPartsLimit = constant.UploadMaxParts
+
+	// `part_size % 1024 = 0` (divisible by 1KB)
+	uploadPaddingPartSize = constant.UploadPadding
+	// `524288 % part_size = 0` (512KB must be evenly divisible by part_size)
+	uploadMaximumPartSize = constant.UploadMaxPartSize
+)
+
+type upload interface {
+	// GetFileID returns random file identifier created by the client.
+	GetFileID() int64
+	// GetFilePart returns numerical order of a part.
+	GetFilePart() int
+	// GetBytes returns binary data, content of a part.
+	GetBytes() []byte
+}
+
+func validatePartSize(got, stored int) *tgerr.Error {
+	switch {
+	case got == 0:
+		return tgerr.New(400, tg.ErrFilePartEmpty)
+	case got > uploadMaximumPartSize:
+		return tgerr.New(400, tg.ErrFilePartTooBig)
+	}
+
+	if stored == 0 {
+		return nil
+	}
+
+	switch {
+	case got != stored:
+		return tgerr.New(400, tg.ErrFilePartSizeChanged)
+	case uploadMaximumPartSize%got != 0,
+		got%uploadPaddingPartSize != 0:
+		return tgerr.New(400, tg.ErrFilePartSizeInvalid)
+	default:
+		return nil
+	}
+}
+
+func (m *Service) write(ctx context.Context, request upload) (err error) {
+	// TODO(tdakkota): Better way to handle user id. For now we haven't auth service to pair
+	//  user ID and authkey
+	id, ok := ctx.Value("user_id").(int)
+	if !ok {
+		id = 10
+	}
+
+	file, err := m.storage.Open(fmt.Sprintf("%d_%d", id, request.GetFileID()))
+	if err != nil {
+		return errors.Wrap(err, "open file")
+	}
+	defer func() {
+		multierr.AppendInto(&err, file.Close())
+	}()
+
+	part := request.GetFilePart()
+	if part < 0 || part > uploadPartsLimit {
+		return tgerr.New(400, tg.ErrFilePartInvalid)
+	}
+	data := request.GetBytes()
+	partSize := file.PartSize()
+	if err := validatePartSize(len(data), partSize); err != nil {
+		return err
+	}
+	if partSize == 0 {
+		partSize = len(data)
+		file.SetPartSize(partSize)
+	}
+
+	offset := int64(partSize * part)
+	if _, err := file.WriteAt(data, offset); err != nil {
+		return errors.Errorf("write at %d-%d", offset, offset+int64(len(data)))
+	}
+
+	return nil
+}
+
+func (m *Service) UploadSaveFilePart(ctx context.Context, request *tg.UploadSaveFilePartRequest) (bool, error) {
+	if err := m.write(ctx, request); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+func (m *Service) UploadSaveBigFilePart(ctx context.Context, request *tg.UploadSaveBigFilePartRequest) (bool, error) {
+	part := request.FileTotalParts
+	if part < 0 || part > uploadPartsLimit {
+		return false, tgerr.New(400, tg.ErrFilePartsInvalid)
+	}
+
+	if err := m.write(ctx, request); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}