Skip to content

Commit 6b120b8

Browse files
sguiheuxsguiheux
authored
feat(cdn): add route to be able to resync a backend with database (#5829)
1 parent c39725e commit 6b120b8

File tree

13 files changed

+301
-0
lines changed

13 files changed

+301
-0
lines changed

‎engine/cdn/cdn_router.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,6 @@ func (s *Service) initRouter(ctx context.Context) {
5252
r.Handle("/admin/database/encryption/{entity}", nil, r.GET(s.getAdminDatabaseEncryptedTuplesByEntity))
5353
r.Handle("/admin/database/encryption/{entity}/roll/{pk}", nil, r.POST(s.postAdminDatabaseRollEncryptedEntityByPrimaryKey))
5454

55+
r.Handle("/admin/backend/{id}/resync/{type}", nil, r.POST(s.postAdminResyncBackendWithDatabaseHandler))
56+
5557
}

‎engine/cdn/storage/cds/cds.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/ovh/cds/engine/gorpmapper"
1414
"github.com/ovh/cds/sdk"
1515
"github.com/ovh/cds/sdk/cdsclient"
16+
"github.com/rockbears/log"
1617
)
1718

1819
type CDS struct {
@@ -130,3 +131,7 @@ func (c *CDS) Read(_ sdk.CDNItemUnit, r io.Reader, w io.Writer) error {
130131
func (c *CDS) Write(_ sdk.CDNItemUnit, _ io.Reader, _ io.Writer) error {
131132
return nil
132133
}
134+
135+
func (c *CDS) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
136+
log.Error(ctx, "Resynchronization with database not implemented for CDS storage unit")
137+
}

‎engine/cdn/storage/dao.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,19 @@ func HasItemUnitsByUnitAndHashLocator(db gorp.SqlExecutor, unitID string, hashLo
174174
return len(ids) > 0, sdk.WithStack(err)
175175
}
176176

177+
func HashItemUnitByApiRefHash(db gorp.SqlExecutor, apiRefHash string, unitID string) (bool, error) {
178+
query := `
179+
SELECT count(sui.id) FROM storage_unit_item sui
180+
JOIN item on item.id = sui.item_id
181+
WHERE item.api_ref_hash = $1 AND unit_id = $2
182+
`
183+
nb, err := db.SelectInt(query, apiRefHash, unitID)
184+
if err != nil {
185+
return false, sdk.WithStack(err)
186+
}
187+
return nb > 0, nil
188+
}
189+
177190
func LoadItemUnitByID(ctx context.Context, m *gorpmapper.Mapper, db gorp.SqlExecutor, id string, opts ...gorpmapper.GetOptionFunc) (*sdk.CDNItemUnit, error) {
178191
query := gorpmapper.NewQuery("SELECT * FROM storage_unit_item WHERE id = $1 AND to_delete = false").Args(id)
179192
return getItemUnit(ctx, m, db, query, opts...)

‎engine/cdn/storage/local/buffer.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ import (
44
"context"
55
"fmt"
66
"os"
7+
"path/filepath"
8+
9+
"github.com/go-gorp/gorp"
10+
"github.com/rockbears/log"
711

812
"github.com/ovh/cds/engine/cdn/storage"
913
"github.com/ovh/cds/engine/cdn/storage/encryption"
@@ -58,3 +62,38 @@ func (b *Buffer) Size(_ sdk.CDNItemUnit) (int64, error) {
5862
func (b *Buffer) BufferType() storage.CDNBufferType {
5963
return b.bufferType
6064
}
65+
66+
func (b *Buffer) ResyncWithDatabase(ctx context.Context, db gorp.SqlExecutor, t sdk.CDNItemType, dryRun bool) {
67+
root := fmt.Sprintf("%s/%s", b.config.Path, string(t))
68+
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
69+
if info == nil {
70+
return nil
71+
}
72+
if info.IsDir() {
73+
log.Warn(ctx, "local-buffer: found directory inside %s: %s", string(t), path)
74+
return nil
75+
}
76+
_, fileName := filepath.Split(path)
77+
has, err := storage.HashItemUnitByApiRefHash(db, fileName, b.ID())
78+
if err != nil {
79+
log.Error(ctx, "local-buffer: unable to check if unit item exist for api ref hash %s: %v", fileName, err)
80+
return nil
81+
}
82+
if has {
83+
return nil
84+
}
85+
if !dryRun {
86+
if err := os.Remove(path); err != nil {
87+
log.Error(ctx, "local-buffer: unable to remove file %s: %v", path, err)
88+
return nil
89+
}
90+
log.Info(ctx, "local-buffer: file %s has been deleted", fileName)
91+
} else {
92+
log.Info(ctx, "local-buffer: file %s should be deleted", fileName)
93+
}
94+
return nil
95+
}); err != nil {
96+
log.Error(ctx, "local-buffer: error during walk operation: %v", err)
97+
}
98+
99+
}

‎engine/cdn/storage/local/local.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,3 +193,7 @@ func (s *AbstractLocal) Remove(ctx context.Context, i sdk.CDNItemUnit) error {
193193
}
194194
return nil
195195
}
196+
197+
func (s *Local) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
198+
log.Error(ctx, "Resynchronization with database not implemented for local storage unit")
199+
}

‎engine/cdn/storage/nfs/nfs.go

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,3 +332,47 @@ func (n *Buffer) ls(v *gonfs.Target, path string) ([]*gonfs.EntryPlus, error) {
332332
}
333333
return dirs, nil
334334
}
335+
336+
func (n *Buffer) ResyncWithDatabase(ctx context.Context, db gorp.SqlExecutor, t sdk.CDNItemType, dryRun bool) {
337+
dial, target, err := n.Connect()
338+
if err != nil {
339+
log.Error(ctx, "nfs-buffer: unable to connect to NFS: %v", err)
340+
return
341+
}
342+
defer dial.Close() // nolint
343+
defer target.Close() //
344+
345+
entries, err := n.ls(target, string(t))
346+
if err != nil {
347+
log.Error(ctx, "nfs-buffer: unable to list directory %s", string(t))
348+
return
349+
}
350+
for _, e := range entries {
351+
if e.IsDir() {
352+
log.Warn(ctx, "nfs-buffer: found directory inside %s: %s", string(t), e)
353+
continue
354+
}
355+
if e.FileName == "" {
356+
log.Warn(ctx, "nfs-buffer: missing file name")
357+
continue
358+
}
359+
has, err := storage.HashItemUnitByApiRefHash(db, e.FileName, n.ID())
360+
if err != nil {
361+
log.Error(ctx, "nfs-buffer: unable to check if unit item exist for api ref hash %s: %v", e.FileName, err)
362+
continue
363+
}
364+
if has {
365+
continue
366+
}
367+
if !dryRun {
368+
if err := target.Remove(string(t) + "/" + e.FileName); err != nil {
369+
log.Error(ctx, "nfs-buffer: unable to remove file %s: %v", string(t)+"/"+e.FileName, err)
370+
continue
371+
}
372+
log.Info(ctx, "nfs-buffer: file %s has been deleted", e.FileName)
373+
} else {
374+
log.Info(ctx, "nfs-buffer: file %s should be deleted", e.FileName)
375+
}
376+
}
377+
return
378+
}

‎engine/cdn/storage/redis/redis.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"strconv"
88

99
"github.com/go-gorp/gorp"
10+
"github.com/rockbears/log"
1011

1112
"github.com/ovh/cds/engine/cache"
1213
"github.com/ovh/cds/engine/cdn/redis"
@@ -142,3 +143,7 @@ func (s *Redis) Status(_ context.Context) []sdk.MonitoringStatusLine {
142143
func (s *Redis) Remove(_ context.Context, i sdk.CDNItemUnit) error {
143144
return sdk.WithStack(s.store.Delete(cache.Key(keyBuffer, i.ItemID)))
144145
}
146+
147+
func (s *Redis) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
148+
log.Error(ctx, "Resynchronization with database not implemented for redis buffer unit")
149+
}

‎engine/cdn/storage/s3/s3.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,3 +203,7 @@ func (s *S3) Remove(ctx context.Context, i sdk.CDNItemUnit) error {
203203
})
204204
return sdk.WithStack(err)
205205
}
206+
207+
func (s *S3) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
208+
log.Error(ctx, "Resynchronization with database not implemented for s3 storage unit")
209+
}

‎engine/cdn/storage/swift/swift.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,3 +139,7 @@ func (s *Swift) Remove(ctx context.Context, i sdk.CDNItemUnit) error {
139139
}
140140
return nil
141141
}
142+
143+
func (s *Swift) ResyncWithDatabase(ctx context.Context, _ gorp.SqlExecutor, _ sdk.CDNItemType, _ bool) {
144+
log.Error(ctx, "Resynchronization with database not implemented for swift storage unit")
145+
}

‎engine/cdn/storage/types.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ type Unit interface {
9898
Read(i sdk.CDNItemUnit, r io.Reader, w io.Writer) error
9999
NewReader(ctx context.Context, i sdk.CDNItemUnit) (io.ReadCloser, error)
100100
GetDriverName() string
101+
ResyncWithDatabase(ctx context.Context, db gorp.SqlExecutor, t sdk.CDNItemType, dryRun bool)
101102
}
102103

103104
type BufferUnit interface {

0 commit comments

Comments
 (0)