diff --git a/backend/go.mod b/backend/go.mod index edd8d02..26cc19d 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -14,6 +14,24 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.17 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect + github.com/aws/smithy-go v1.25.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect diff --git a/backend/go.sum b/backend/go.sum index b5da560..2bbb250 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,5 +1,41 @@ github.com/a-h/templ v0.3.1020 h1:ypAT/L5ySWEnZ6Zft/5yfoWXYYkhFNvEFOeeqecg4tw= github.com/a-h/templ v0.3.1020/go.mod h1:A2DlK61v+K+NRoGnhmYbNYVmtYHcFO5/AisMvBdDxTM= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/backend/internal/db/queries/files.sql b/backend/internal/db/queries/files.sql new file mode 100644 index 0000000..9207ef5 --- /dev/null +++ b/backend/internal/db/queries/files.sql @@ -0,0 +1,18 @@ +-- name: InsertTabloFile :one +INSERT INTO tablo_files (tablo_id, s3_key, filename, content_type, size_bytes) +VALUES ($1, $2, $3, $4, $5) +RETURNING id, tablo_id, s3_key, filename, content_type, size_bytes, created_at; + +-- name: ListFilesByTablo :many +SELECT id, tablo_id, s3_key, filename, content_type, size_bytes, created_at +FROM tablo_files +WHERE tablo_id = $1 +ORDER BY created_at DESC; + +-- name: GetTabloFileByID :one +SELECT id, tablo_id, s3_key, filename, content_type, size_bytes, created_at +FROM tablo_files +WHERE id = $1 AND tablo_id = $2; + +-- name: DeleteTabloFile :exec +DELETE FROM tablo_files WHERE id = $1 AND tablo_id = $2; diff --git a/backend/internal/files/store.go b/backend/internal/files/store.go new file mode 100644 index 0000000..b370715 --- /dev/null +++ b/backend/internal/files/store.go @@ -0,0 +1,122 @@ +package files + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// FileStorer is the interface satisfied by Store and used for test injection. +type FileStorer interface { + Upload(ctx context.Context, key string, file io.Reader) (contentType string, bytesWritten int64, err error) + Delete(ctx context.Context, key string) error + PresignDownload(ctx context.Context, key string) (string, error) +} + +// Store is a thin wrapper around an S3-compatible client. +type Store struct { + client *s3.Client + bucket string +} + +// byteCountReader wraps an io.Reader and counts the number of bytes read. +type byteCountReader struct { + r io.Reader + count int64 +} + +func (b *byteCountReader) Read(p []byte) (int, error) { + n, err := b.r.Read(p) + b.count += int64(n) + return n, err +} + +// NewStore constructs a Store pointed at an S3-compatible endpoint. +// +// endpoint: e.g. "http://localhost:9000" (MinIO) or "https://.r2.cloudflarestorage.com" (R2) +// usePathStyle: true for MinIO (required per Pitfall 1), false for R2 +func NewStore(ctx context.Context, endpoint, bucket, region, accessKey, secretKey string, usePathStyle bool) (*Store, error) { + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion(region), + config.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""), + ), + ) + if err != nil { + return nil, err + } + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = usePathStyle // true for MinIO; false or omit for R2 + }) + + return &Store{client: client, bucket: bucket}, nil +} + +// Upload streams file to S3, sniffing content-type from the first 512 bytes. +// It implements the sniff-and-stream pattern (RESEARCH Pattern 2): +// - Reads first 512 bytes via io.ReadFull (io.ErrUnexpectedEOF is non-fatal for files < 512 bytes) +// - Calls http.DetectContentType on the sniffed bytes (D-05) +// - Reconstructs the full body via io.MultiReader(sniffBuf, file) (Pitfall 8 avoidance) +// - Wraps body in byteCountReader to reliably track bytes written (Pitfall 8 — header.Size unreliable) +func (s *Store) Upload(ctx context.Context, key string, file io.Reader) (contentType string, bytesWritten int64, err error) { + // Sniff content-type from first 512 bytes. + var sniffBuf [512]byte + n, readErr := io.ReadFull(file, sniffBuf[:]) + // Accept io.ErrUnexpectedEOF — normal for files < 512 bytes (Pitfall 3). + if readErr != nil && !errors.Is(readErr, io.ErrUnexpectedEOF) { + return "", 0, readErr + } + contentType = http.DetectContentType(sniffBuf[:n]) + + // Reconstruct full body: sniffed bytes + remaining reader. + body := io.MultiReader(bytes.NewReader(sniffBuf[:n]), file) + + // Wrap in a counting reader to track actual bytes written (Pitfall 8). + counter := &byteCountReader{r: body} + + _, putErr := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + Body: counter, + ContentType: aws.String(contentType), + }) + if putErr != nil { + return contentType, counter.count, putErr + } + + return contentType, counter.count, nil +} + +// Delete removes an object from S3. +func (s *Store) Delete(ctx context.Context, key string) error { + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + return err +} + +// PresignDownload returns a time-limited presigned GET URL (5-minute TTL). +func (s *Store) PresignDownload(ctx context.Context, key string) (string, error) { + presignClient := s3.NewPresignClient(s.client) + req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }, func(o *s3.PresignOptions) { + o.Expires = 5 * time.Minute + }) + if err != nil { + return "", err + } + return req.URL, nil +} diff --git a/backend/migrations/0005_files.sql b/backend/migrations/0005_files.sql new file mode 100644 index 0000000..e43c7d3 --- /dev/null +++ b/backend/migrations/0005_files.sql @@ -0,0 +1,20 @@ +-- migrations/0005_files.sql +-- Phase 5: Files (tablo file attachments) + +-- +goose Up + +CREATE TABLE tablo_files ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + tablo_id uuid NOT NULL REFERENCES tablos(id) ON DELETE CASCADE, + s3_key text NOT NULL, + filename text NOT NULL, + content_type text NOT NULL DEFAULT 'application/octet-stream', + size_bytes bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- Composite index: list files for a tablo ordered newest-first (D-06: files immutable). +CREATE INDEX tablo_files_tablo_id_idx ON tablo_files(tablo_id, created_at DESC); + +-- +goose Down +DROP TABLE IF EXISTS tablo_files;