feat(05-01): add aws-sdk-go-v2 modules, 0005_files migration, sqlc queries, and files.Store
- Add four aws-sdk-go-v2 modules: core, config, credentials, service/s3 - Write 0005_files.sql migration (tablo_files table with ON DELETE CASCADE) - Write internal/db/queries/files.sql with InsertTabloFile, ListFilesByTablo, GetTabloFileByID, DeleteTabloFile - Implement internal/files/store.go: FileStorer interface, Store struct, NewStore (UsePathStyle for MinIO), Upload (sniff+stream+bytecount), Delete, PresignDownload - sqlc generate produces files.sql.go + TabloFile model (gitignored, regeneratable)
This commit is contained in:
Arthur Belleville
parent
5ce8b70f69
commit
e0d72747e0
5 changed files with 214 additions and 0 deletions
|
|
@ -14,6 +14,24 @@ require (
|
|||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||
github.com/aws/smithy-go v1.25.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
|
|
|
|||
|
|
@ -1,5 +1,41 @@
|
|||
github.com/a-h/templ v0.3.1020 h1:ypAT/L5ySWEnZ6Zft/5yfoWXYYkhFNvEFOeeqecg4tw=
|
||||
github.com/a-h/templ v0.3.1020/go.mod h1:A2DlK61v+K+NRoGnhmYbNYVmtYHcFO5/AisMvBdDxTM=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
|
|||
18
backend/internal/db/queries/files.sql
Normal file
18
backend/internal/db/queries/files.sql
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
-- name: InsertTabloFile :one
|
||||
INSERT INTO tablo_files (tablo_id, s3_key, filename, content_type, size_bytes)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING id, tablo_id, s3_key, filename, content_type, size_bytes, created_at;
|
||||
|
||||
-- name: ListFilesByTablo :many
|
||||
SELECT id, tablo_id, s3_key, filename, content_type, size_bytes, created_at
|
||||
FROM tablo_files
|
||||
WHERE tablo_id = $1
|
||||
ORDER BY created_at DESC;
|
||||
|
||||
-- name: GetTabloFileByID :one
|
||||
SELECT id, tablo_id, s3_key, filename, content_type, size_bytes, created_at
|
||||
FROM tablo_files
|
||||
WHERE id = $1 AND tablo_id = $2;
|
||||
|
||||
-- name: DeleteTabloFile :exec
|
||||
DELETE FROM tablo_files WHERE id = $1 AND tablo_id = $2;
|
||||
122
backend/internal/files/store.go
Normal file
122
backend/internal/files/store.go
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
// FileStorer is the interface satisfied by Store and used for test injection.
|
||||
type FileStorer interface {
|
||||
Upload(ctx context.Context, key string, file io.Reader) (contentType string, bytesWritten int64, err error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
PresignDownload(ctx context.Context, key string) (string, error)
|
||||
}
|
||||
|
||||
// Store is a thin wrapper around an S3-compatible client.
|
||||
type Store struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
// byteCountReader wraps an io.Reader and counts the number of bytes read.
|
||||
type byteCountReader struct {
|
||||
r io.Reader
|
||||
count int64
|
||||
}
|
||||
|
||||
func (b *byteCountReader) Read(p []byte) (int, error) {
|
||||
n, err := b.r.Read(p)
|
||||
b.count += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// NewStore constructs a Store pointed at an S3-compatible endpoint.
|
||||
//
|
||||
// endpoint: e.g. "http://localhost:9000" (MinIO) or "https://<account>.r2.cloudflarestorage.com" (R2)
|
||||
// usePathStyle: true for MinIO (required per Pitfall 1), false for R2
|
||||
func NewStore(ctx context.Context, endpoint, bucket, region, accessKey, secretKey string, usePathStyle bool) (*Store, error) {
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
o.UsePathStyle = usePathStyle // true for MinIO; false or omit for R2
|
||||
})
|
||||
|
||||
return &Store{client: client, bucket: bucket}, nil
|
||||
}
|
||||
|
||||
// Upload streams file to S3, sniffing content-type from the first 512 bytes.
|
||||
// It implements the sniff-and-stream pattern (RESEARCH Pattern 2):
|
||||
// - Reads first 512 bytes via io.ReadFull (io.ErrUnexpectedEOF is non-fatal for files < 512 bytes)
|
||||
// - Calls http.DetectContentType on the sniffed bytes (D-05)
|
||||
// - Reconstructs the full body via io.MultiReader(sniffBuf, file) (Pitfall 8 avoidance)
|
||||
// - Wraps body in byteCountReader to reliably track bytes written (Pitfall 8 — header.Size unreliable)
|
||||
func (s *Store) Upload(ctx context.Context, key string, file io.Reader) (contentType string, bytesWritten int64, err error) {
|
||||
// Sniff content-type from first 512 bytes.
|
||||
var sniffBuf [512]byte
|
||||
n, readErr := io.ReadFull(file, sniffBuf[:])
|
||||
// Accept io.ErrUnexpectedEOF — normal for files < 512 bytes (Pitfall 3).
|
||||
if readErr != nil && !errors.Is(readErr, io.ErrUnexpectedEOF) {
|
||||
return "", 0, readErr
|
||||
}
|
||||
contentType = http.DetectContentType(sniffBuf[:n])
|
||||
|
||||
// Reconstruct full body: sniffed bytes + remaining reader.
|
||||
body := io.MultiReader(bytes.NewReader(sniffBuf[:n]), file)
|
||||
|
||||
// Wrap in a counting reader to track actual bytes written (Pitfall 8).
|
||||
counter := &byteCountReader{r: body}
|
||||
|
||||
_, putErr := s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: counter,
|
||||
ContentType: aws.String(contentType),
|
||||
})
|
||||
if putErr != nil {
|
||||
return contentType, counter.count, putErr
|
||||
}
|
||||
|
||||
return contentType, counter.count, nil
|
||||
}
|
||||
|
||||
// Delete removes an object from S3.
|
||||
func (s *Store) Delete(ctx context.Context, key string) error {
|
||||
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// PresignDownload returns a time-limited presigned GET URL (5-minute TTL).
|
||||
func (s *Store) PresignDownload(ctx context.Context, key string) (string, error) {
|
||||
presignClient := s3.NewPresignClient(s.client)
|
||||
req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
}, func(o *s3.PresignOptions) {
|
||||
o.Expires = 5 * time.Minute
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return req.URL, nil
|
||||
}
|
||||
20
backend/migrations/0005_files.sql
Normal file
20
backend/migrations/0005_files.sql
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
-- migrations/0005_files.sql
|
||||
-- Phase 5: Files (tablo file attachments)
|
||||
|
||||
-- +goose Up
|
||||
|
||||
CREATE TABLE tablo_files (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tablo_id uuid NOT NULL REFERENCES tablos(id) ON DELETE CASCADE,
|
||||
s3_key text NOT NULL,
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL DEFAULT 'application/octet-stream',
|
||||
size_bytes bigint NOT NULL DEFAULT 0,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Composite index: list files for a tablo ordered newest-first (D-06: files immutable).
|
||||
CREATE INDEX tablo_files_tablo_id_idx ON tablo_files(tablo_id, created_at DESC);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS tablo_files;
|
||||
Loading…
Reference in a new issue