Skip to content

Commit a045b21

Browse files
alex-aizmanalex-aizman
committed
[config change] new content checksum
* still xxhash but a different implementation * internally, there's now a strict delineation: * OneOfOne xxhash - for system metadata * cespare/xxhash (denoted as "xxhash2" in cluster config) - for user data Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent 984854b commit a045b21

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+256
-235
lines changed

‎ais/bucketmeta_internal_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ var _ = Describe("BMD marshal and unmarshal", func() {
3333
// Set path for proxy (it uses ConfigDir)
3434
config := cmn.GCO.BeginUpdate()
3535
config.ConfigDir = mpath
36-
config.Cksum.Type = cos.ChecksumXXHash
36+
config.Cksum.Type = cos.ChecksumOneXxh
3737
config.Space = cmn.SpaceConf{
3838
LowWM: 75, HighWM: 90, OOS: 95,
3939
}

‎ais/emd_internal_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ var _ = Describe("EtlMD marshal and unmarshal", func() {
5252
// Set path for proxy (it uses ConfigDir)
5353
config := cmn.GCO.BeginUpdate()
5454
config.ConfigDir = mpath
55-
config.Cksum.Type = cos.ChecksumXXHash
55+
config.Cksum.Type = cos.ChecksumOneXxh
5656
config.Space = cmn.SpaceConf{
5757
LowWM: 75, HighWM: 90, OOS: 95,
5858
}

‎ais/metasync_internal_test.go

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ func newPrimary() *proxy {
8787
config.Timeout.MaxKeepalive = cos.Duration(4 * time.Second)
8888
config.Client.Timeout = cos.Duration(10 * time.Second)
8989
config.Client.TimeoutLong = cos.Duration(10 * time.Second)
90-
config.Cksum.Type = cos.ChecksumXXHash
90+
config.Cksum.Type = cos.ChecksumOneXxh
9191
cmn.GCO.CommitUpdate(config)
9292
cmn.GCO.SetInitialGconfPath("/tmp/ais-tests/ais.config")
9393

@@ -123,7 +123,7 @@ func newSecondary(name string) *proxy {
123123
config.Keepalive.Proxy.Interval = cos.Duration(3 * time.Second)
124124
config.Timeout.CplaneOperation = cos.Duration(2 * time.Second)
125125
config.Timeout.MaxKeepalive = cos.Duration(4 * time.Second)
126-
config.Cksum.Type = cos.ChecksumXXHash
126+
config.Cksum.Type = cos.ChecksumOneXxh
127127
cmn.GCO.CommitUpdate(config)
128128

129129
o := newBMDOwnerPrx(cmn.GCO.Get())
@@ -174,22 +174,22 @@ func TestMetasyncDeepCopy(t *testing.T) {
174174
bmd := newBucketMD()
175175
bmd.add(meta.NewBck("bucket1", apc.AIS, cmn.NsGlobal), &cmn.Bprops{
176176
Cksum: cmn.CksumConf{
177-
Type: cos.ChecksumXXHash,
177+
Type: cos.ChecksumOneXxh,
178178
},
179179
})
180180
bmd.add(meta.NewBck("bucket2", apc.AIS, cmn.NsGlobal), &cmn.Bprops{
181181
Cksum: cmn.CksumConf{
182-
Type: cos.ChecksumXXHash,
182+
Type: cos.ChecksumOneXxh,
183183
},
184184
})
185185
bmd.add(meta.NewBck("bucket3", apc.AWS, cmn.NsGlobal), &cmn.Bprops{
186186
Cksum: cmn.CksumConf{
187-
Type: cos.ChecksumXXHash,
187+
Type: cos.ChecksumOneXxh,
188188
},
189189
})
190190
bmd.add(meta.NewBck("bucket4", apc.AWS, cmn.NsGlobal), &cmn.Bprops{
191191
Cksum: cmn.CksumConf{
192-
Type: cos.ChecksumXXHash,
192+
Type: cos.ChecksumOneXxh,
193193
},
194194
})
195195

@@ -640,12 +640,12 @@ func TestMetasyncData(t *testing.T) {
640640
// sync bucketmd, fail target and retry
641641
bmd.add(meta.NewBck("bucket1", apc.AIS, cmn.NsGlobal), &cmn.Bprops{
642642
Cksum: cmn.CksumConf{
643-
Type: cos.ChecksumXXHash,
643+
Type: cos.ChecksumOneXxh,
644644
},
645645
})
646646
bmd.add(meta.NewBck("bucket2", apc.AIS, cmn.NsGlobal), &cmn.Bprops{
647647
Cksum: cmn.CksumConf{
648-
Type: cos.ChecksumXXHash,
648+
Type: cos.ChecksumOneXxh,
649649
},
650650
})
651651
primary.owner.bmd.putPersist(bmd, nil)
@@ -664,7 +664,7 @@ func TestMetasyncData(t *testing.T) {
664664
// after rejecting a few sync requests
665665
bmd = bmd.clone()
666666
bprops := &cmn.Bprops{
667-
Cksum: cmn.CksumConf{Type: cos.ChecksumXXHash},
667+
Cksum: cmn.CksumConf{Type: cos.ChecksumOneXxh},
668668
LRU: cmn.GCO.Get().LRU,
669669
}
670670
bmd.add(meta.NewBck("bucket3", apc.AIS, cmn.NsGlobal), bprops)

‎ais/multiproxy_internal_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func newDiscoverServerPrimary() *proxy {
5656
config.Timeout.MaxKeepalive = cos.Duration(4 * time.Second)
5757
config.Client.Timeout = cos.Duration(10 * time.Second)
5858
config.Client.TimeoutLong = cos.Duration(10 * time.Second)
59-
config.Cksum.Type = cos.ChecksumXXHash
59+
config.Cksum.Type = cos.ChecksumCesXxh
6060
cmn.GCO.CommitUpdate(config)
6161

6262
p.owner.smap = newSmapOwner(config)

‎ais/test/bucket_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3130,7 +3130,7 @@ func TestAllChecksums(t *testing.T) {
31303130
checksums := cos.SupportedChecksums()
31313131
for _, mirrored := range []bool{false, true} {
31323132
for _, cksumType := range checksums {
3133-
if testing.Short() && cksumType != cos.ChecksumNone && cksumType != cos.ChecksumXXHash {
3133+
if testing.Short() && cksumType != cos.ChecksumNone && cksumType != cos.ChecksumOneXxh && cksumType != cos.ChecksumCesXxh {
31343134
continue
31353135
}
31363136
tag := cksumType
@@ -3146,7 +3146,7 @@ func TestAllChecksums(t *testing.T) {
31463146
}
31473147

31483148
for _, cksumType := range checksums {
3149-
if testing.Short() && cksumType != cos.ChecksumNone && cksumType != cos.ChecksumXXHash {
3149+
if testing.Short() && cksumType != cos.ChecksumNone && cksumType != cos.ChecksumOneXxh && cksumType != cos.ChecksumCesXxh {
31503150
continue
31513151
}
31523152
tag := cksumType + "/EC"
@@ -3171,13 +3171,13 @@ func testWarmValidation(t *testing.T, cksumType string, mirrored, eced bool) {
31713171
t: t,
31723172
num: 1000,
31733173
numGetsEachFile: 1,
3174-
fileSize: uint64(cos.KiB + rand.Int64N(cos.KiB*10)),
3174+
fileSize: cos.MiB/2 + rand.Uint64N(cos.KiB*10),
31753175
}
31763176
numCorrupted = rand.IntN(m.num/100) + 2
31773177
)
31783178
if testing.Short() {
31793179
m.num = 40
3180-
m.fileSize = cos.KiB
3180+
m.fileSize = cos.KiB + rand.Uint64N(cos.KiB)
31813181
numCorrupted = 13
31823182
}
31833183

‎ais/test/cp_multiobj_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func TestCopyMultiObjSimple(t *testing.T) {
3131
const (
3232
copyCnt = 20
3333
objSize = 128
34-
cksumType = cos.ChecksumXXHash
34+
cksumType = cos.ChecksumCesXxh
3535
)
3636
var (
3737
objCnt = 2345

‎ais/test/etl_tar2tf_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ func TestETLTar2TFS3(t *testing.T) {
7171
tools.CreateBucket(t, proxyURL, bck, nil, true /*cleanup*/)
7272

7373
// PUT TAR to the cluster
74-
f, err := readers.NewExistingFile(tarPath, cos.ChecksumXXHash)
74+
f, err := readers.NewExistingFile(tarPath, cos.ChecksumCesXxh)
7575
tassert.CheckFatal(t, err)
7676
putArgs := api.PutArgs{
7777
BaseParams: baseParams,
@@ -149,7 +149,7 @@ func TestETLTar2TFRanges(t *testing.T) {
149149
tools.CreateBucket(t, proxyURL, bck, nil, true /*cleanup*/)
150150

151151
// PUT TAR to the cluster
152-
f, err := readers.NewExistingFile(tarPath, cos.ChecksumXXHash)
152+
f, err := readers.NewExistingFile(tarPath, cos.ChecksumCesXxh)
153153
tassert.CheckFatal(t, err)
154154
putArgs := api.PutArgs{
155155
BaseParams: baseParams,

‎ais/test/etl_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ import (
3939
"github.com/NVIDIA/aistore/tools/trand"
4040
"github.com/NVIDIA/aistore/xact"
4141
"github.com/NVIDIA/go-tfdata/tfdata/core"
42-
"github.com/OneOfOne/xxhash"
42+
onexxh "github.com/OneOfOne/xxhash"
4343
)
4444

4545
const (
@@ -456,7 +456,7 @@ func TestETLInlineObjWithArgs(t *testing.T) {
456456
// Read the object and calculate the hash using the same hash algorithm as the ETL (same random seed).
457457
// The results should match because the hash is calculated in the same way.
458458
data, _ := io.ReadAll(r)
459-
hash := xxhash.Checksum64S(data, seed)
459+
hash := onexxh.Checksum64S(data, seed)
460460
hashHex := fmt.Sprintf("%016x", hash)
461461

462462
return bytes.NewReader([]byte(hashHex))

‎ais/test/maintain_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ func TestMaintenanceDecommissionRebalance(t *testing.T) {
208208
tools.CreateBucket(t, proxyURL, bck, nil, true /*cleanup*/)
209209
for i := range objCount {
210210
objName := fmt.Sprintf("%sobj%04d", objPath, i)
211-
r, _ := readers.NewRand(int64(fileSize), cos.ChecksumXXHash)
211+
r, _ := readers.NewRand(int64(fileSize), cos.ChecksumCesXxh)
212212
_, err := api.PutObject(&api.PutArgs{
213213
BaseParams: baseParams,
214214
Bck: bck,

‎ais/test/object_test.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -809,7 +809,7 @@ func TestChecksumValidateOnWarmGetForRemoteBucket(t *testing.T) {
809809
_ = mock.NewTarget(mock.NewBaseBownerMock(
810810
meta.NewBck(
811811
m.bck.Name, m.bck.Provider, cmn.NsGlobal,
812-
&cmn.Bprops{Cksum: cmn.CksumConf{Type: cos.ChecksumXXHash}, Extra: p.Extra, BID: 0xa73b9f11},
812+
&cmn.Bprops{Cksum: cmn.CksumConf{Type: cos.ChecksumCesXxh}, Extra: p.Extra, BID: 0xa73b9f11},
813813
),
814814
))
815815

@@ -860,7 +860,7 @@ func TestChecksumValidateOnWarmGetForRemoteBucket(t *testing.T) {
860860
oldFileInfo, _ = os.Stat(fqn)
861861

862862
tlog.Logf("Changing file xattr[%s]: %s\n", objName, fqn)
863-
err = tools.SetXattrCksum(fqn, m.bck, cos.NewCksum(cos.ChecksumXXHash, "01234"))
863+
err = tools.SetXattrCksum(fqn, m.bck, cos.NewCksum(cos.ChecksumCesXxh, "01234"))
864864
tassert.CheckError(t, err)
865865
validateGETUponFileChangeForChecksumValidation(t, proxyURL, objName, fqn, oldFileInfo)
866866

@@ -879,7 +879,7 @@ func TestChecksumValidateOnWarmGetForRemoteBucket(t *testing.T) {
879879
}
880880

881881
tlog.Logf("Changing file xattr[%s]: %s\n", objName, fqn)
882-
err = tools.SetXattrCksum(fqn, m.bck, cos.NewCksum(cos.ChecksumXXHash, "01234abcde"))
882+
err = tools.SetXattrCksum(fqn, m.bck, cos.NewCksum(cos.ChecksumCesXxh, "01234abcde"))
883883
tassert.CheckError(t, err)
884884

885885
_, err = api.GetObject(baseParams, m.bck, objName, nil)
@@ -913,7 +913,7 @@ func TestValidateOnWarmGetRemoteBucket(t *testing.T) {
913913
tMock := mock.NewTarget(mock.NewBaseBownerMock(
914914
meta.NewBck(
915915
m.bck.Name, m.bck.Provider, cmn.NsGlobal,
916-
&cmn.Bprops{Cksum: cmn.CksumConf{Type: cos.ChecksumXXHash}, Extra: p.Extra, BID: 0xa73b9f11},
916+
&cmn.Bprops{Cksum: cmn.CksumConf{Type: cos.ChecksumCesXxh}, Extra: p.Extra, BID: 0xa73b9f11},
917917
),
918918
))
919919

@@ -941,7 +941,7 @@ func TestValidateOnWarmGetRemoteBucket(t *testing.T) {
941941
propsToSet := &cmn.BpropsToSet{
942942
Versioning: &cmn.VersionConfToSet{ValidateWarmGet: apc.Ptr(true)},
943943
Cksum: &cmn.CksumConfToSet{
944-
Type: apc.Ptr(cos.ChecksumXXHash),
944+
Type: apc.Ptr(cos.ChecksumCesXxh),
945945
ValidateWarmGet: apc.Ptr(true),
946946
},
947947
}
@@ -980,7 +980,7 @@ func TestValidateOnWarmGetRemoteBucket(t *testing.T) {
980980
modify func(*testing.T, *core.LOM)
981981
}{
982982
"checksum": {modify: func(_ *testing.T, lom *core.LOM) {
983-
lom.SetCksum(cos.NewCksum(cos.ChecksumXXHash, "01234"))
983+
lom.SetCksum(cos.NewCksum(cos.ChecksumCesXxh, "01234"))
984984
}},
985985
"local_content": {modify: func(t *testing.T, lom *core.LOM) {
986986
err := os.WriteFile(lom.FQN, []byte("modified"), cos.PermRWR)
@@ -1158,7 +1158,7 @@ func TestChecksumValidateOnWarmGetForBucket(t *testing.T) {
11581158
_ = mock.NewTarget(mock.NewBaseBownerMock(
11591159
meta.NewBck(
11601160
m.bck.Name, apc.AIS, cmn.NsGlobal,
1161-
&cmn.Bprops{Cksum: cmn.CksumConf{Type: cos.ChecksumXXHash}, BID: 1},
1161+
&cmn.Bprops{Cksum: cmn.CksumConf{Type: cos.ChecksumCesXxh}, BID: 1},
11621162
),
11631163
meta.CloneBck(&m.bck),
11641164
))
@@ -1199,7 +1199,7 @@ func TestChecksumValidateOnWarmGetForBucket(t *testing.T) {
11991199
objName = m.objNames[1]
12001200
fqn = findObjOnDisk(m.bck, objName)
12011201
tlog.Logf("Changing file xattr[%s]: %s\n", objName, fqn)
1202-
err = tools.SetXattrCksum(fqn, m.bck, cos.NewCksum(cos.ChecksumXXHash, "01234abcde"))
1202+
err = tools.SetXattrCksum(fqn, m.bck, cos.NewCksum(cos.ChecksumCesXxh, "01234abcde"))
12031203
tassert.CheckError(t, err)
12041204
executeTwoGETsForChecksumValidation(proxyURL, m.bck, objName, t)
12051205

@@ -1218,7 +1218,7 @@ func TestChecksumValidateOnWarmGetForBucket(t *testing.T) {
12181218
}
12191219

12201220
tlog.Logf("Changing file xattr[%s]: %s\n", objName, fqn)
1221-
err = tools.SetXattrCksum(fqn, m.bck, cos.NewCksum(cos.ChecksumXXHash, "01234abcde"))
1221+
err = tools.SetXattrCksum(fqn, m.bck, cos.NewCksum(cos.ChecksumCesXxh, "01234abcde"))
12221222
tassert.CheckError(t, err)
12231223
_, err = api.GetObject(baseParams, m.bck, objName, nil)
12241224
tassert.CheckError(t, err)
@@ -1501,7 +1501,7 @@ func Test_checksum(t *testing.T) {
15011501

15021502
propsToSet := &cmn.BpropsToSet{
15031503
Cksum: &cmn.CksumConfToSet{
1504-
Type: apc.Ptr(cos.ChecksumXXHash),
1504+
Type: apc.Ptr(cos.ChecksumCesXxh),
15051505
ValidateColdGet: apc.Ptr(true),
15061506
},
15071507
}

0 commit comments

Comments
 (0)