Reviewing files that changed from the base of the PR and between 2caa6e1 and 6550930.

• docs/grpc/docs.md (1 hunks)
• docs/redoc/master/openapi.json (2 hunks)
• lib/api/src/grpc/conversions.rs (3 hunks)
• lib/api/src/grpc/proto/points.proto (1 hunks)
• lib/api/src/grpc/qdrant.rs (1 hunks)
• lib/api/src/rest/models.rs (1 hunks)
• lib/collection/src/collection/payload_index_schema.rs (3 hunks)
• lib/collection/src/collection/state_management.rs (2 hunks)
• lib/collection/src/collection_manager/collection_updater.rs (3 hunks)
• lib/collection/src/collection_manager/holders/proxy_segment.rs (9 hunks)
• lib/collection/src/collection_manager/holders/segment_holder.rs (1 hunks)
• lib/collection/src/collection_manager/optimizers/indexing_optimizer.rs (1 hunks)
• lib/collection/src/collection_manager/optimizers/segment_optimizer.rs (6 hunks)
• lib/collection/src/collection_manager/optimizers/vacuum_optimizer.rs (1 hunks)
• lib/collection/src/collection_manager/segments_updater.rs (5 hunks)
• lib/collection/src/operations/verification/mod.rs (2 hunks)
• lib/collection/src/shards/local_shard/shard_ops.rs (1 hunks)
• lib/collection/src/tests/points_dedup.rs (1 hunks)
• lib/collection/tests/integration/collection_test.rs (4 hunks)
• lib/common/common/src/counter/hardware_accumulator.rs (6 hunks)
• lib/common/common/src/counter/hardware_counter.rs (8 hunks)
• lib/common/common/src/counter/hardware_data.rs (1 hunks)
• lib/common/common/src/counter/referenced_counter.rs (1 hunks)
• lib/segment/benches/boolean_filtering.rs (1 hunks)
• lib/segment/benches/range_filtering.rs (1 hunks)
• lib/segment/benches/segment_info.rs (1 hunks)
• lib/segment/benches/sparse_index_search.rs (1 hunks)
• lib/segment/src/entry/entry_point.rs (2 hunks)
• lib/segment/src/fixtures/payload_context_fixture.rs (1 hunks)
• lib/segment/src/index/field_index/bool_index/mmap_bool_index.rs (6 hunks)
• lib/segment/src/index/field_index/bool_index/mod.rs (7 hunks)
• lib/segment/src/index/field_index/bool_index/simple_bool_index.rs (3 hunks)
• lib/segment/src/index/field_index/field_index_base.rs (6 hunks)
• lib/segment/src/index/field_index/full_text_index/immutable_inverted_index.rs (1 hunks)
• lib/segment/src/index/field_index/full_text_index/inverted_index.rs (2 hunks)
• lib/segment/src/index/field_index/full_text_index/mmap_inverted_index/mod.rs (1 hunks)
• lib/segment/src/index/field_index/full_text_index/mmap_text_index.rs (4 hunks)
• lib/segment/src/index/field_index/full_text_index/mutable_inverted_index.rs (2 hunks)
• lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs (6 hunks)
• lib/segment/src/index/field_index/full_text_index/tests/mod.rs (1 hunks)
• lib/segment/src/index/field_index/full_text_index/text_index.rs (2 hunks)
• lib/segment/src/index/field_index/geo_index/mod.rs (13 hunks)
• lib/segment/src/index/field_index/geo_index/mutable_geo_index.rs (5 hunks)
• lib/segment/src/index/field_index/map_index/immutable_map_index.rs (1 hunks)
• lib/segment/src/index/field_index/map_index/mod.rs (11 hunks)
• lib/segment/src/index/field_index/map_index/mutable_map_index.rs (3 hunks)
• lib/segment/src/index/field_index/null_index/mmap_null_index.rs (4 hunks)
• lib/segment/src/index/field_index/numeric_index/mod.rs (8 hunks)
• lib/segment/src/index/field_index/numeric_index/mutable_numeric_index.rs (1 hunks)
• lib/segment/src/index/field_index/numeric_index/tests.rs (6 hunks)
• lib/segment/src/index/payload_index_base.rs (2 hunks)
• lib/segment/src/index/plain_payload_index.rs (1 hunks)
• lib/segment/src/index/query_optimization/rescore_formula/value_retriever.rs (2 hunks)
• lib/segment/src/index/struct_payload_index.rs (9 hunks)
• lib/segment/src/segment/entry.rs (3 hunks)
• lib/segment/src/segment/segment_ops.rs (1 hunks)
• lib/segment/src/segment/tests.rs (2 hunks)
• lib/segment/src/segment_constructor/segment_builder.rs (1 hunks)
• lib/segment/src/vector_storage/dense/dynamic_mmap_flags.rs (2 hunks)
• lib/segment/tests/integration/batch_search_test.rs (1 hunks)
• lib/segment/tests/integration/byte_storage_hnsw_test.rs (1 hunks)
• lib/segment/tests/integration/byte_storage_quantization_test.rs (1 hunks)
• lib/segment/tests/integration/disbalanced_vectors_test.rs (1 hunks)
• lib/segment/tests/integration/exact_search_test.rs (1 hunks)
• lib/segment/tests/integration/filtrable_hnsw_test.rs (1 hunks)
• lib/segment/tests/integration/gpu_hnsw_test.rs (2 hunks)
• lib/segment/tests/integration/hnsw_discover_test.rs (1 hunks)
• lib/segment/tests/integration/hnsw_quantized_search_test.rs (1 hunks)
• lib/segment/tests/integration/multivector_filtrable_hnsw_test.rs (1 hunks)
• lib/segment/tests/integration/multivector_hnsw_test.rs (1 hunks)
• lib/segment/tests/integration/multivector_quantization_test.rs (1 hunks)
• lib/segment/tests/integration/nested_filtering_test.rs (1 hunks)
• lib/segment/tests/integration/payload_index_test.rs (15 hunks)
• lib/segment/tests/integration/segment_builder_test.rs (1 hunks)
• lib/segment/tests/integration/segment_on_disk_snapshot.rs (2 hunks)
• lib/segment/tests/integration/sparse_vector_index_search_tests.rs (1 hunks)
• lib/storage/src/content_manager/consensus_manager.rs (4 hunks)
• lib/storage/src/content_manager/mod.rs (2 hunks)
• lib/storage/src/content_manager/snapshots/recover.rs (2 hunks)
• lib/storage/src/content_manager/toc/collection_container.rs (2 hunks)
• lib/storage/src/content_manager/toc/collection_meta_ops.rs (4 hunks)
• lib/storage/src/content_manager/toc/mod.rs (1 hunks)
• lib/storage/src/content_manager/toc/request_hw_counter.rs (2 hunks)
• lib/storage/src/dispatcher.rs (3 hunks)
• lib/storage/tests/integration/alias_tests.rs (4 hunks)
• src/actix/api/collections_api.rs (5 hunks)
• src/actix/api/update_api.rs (1 hunks)
• src/common/collections.rs (14 hunks)
• src/common/metrics.rs (2 hunks)
• src/common/update.rs (2 hunks)
• src/consensus.rs (2 hunks)
• src/migrations/single_to_cluster.rs (3 hunks)
• src/tonic/api/collections_api.rs (2 hunks)

• GitHub Check: test-snapshot-operations-s3-minio
• GitHub Check: test-shard-snapshot-api-s3-minio
• GitHub Check: test-low-resources
• GitHub Check: test-consistency
• GitHub Check: test-consensus
• GitHub Check: Basic TLS/HTTPS tests
• GitHub Check: test (macos-latest)
• GitHub Check: test
• GitHub Check: test (windows-latest)
• GitHub Check: test-consensus-compose
• GitHub Check: test
• GitHub Check: test
• GitHub Check: test (ubuntu-latest)

docs/redoc/master/openapi.json (1)

12668-12672: New payload index I/O write tracking field added correctly

The addition of the payload_index_io_write field to the HardwareUsage structure follows the proper schema pattern and maintains consistency with existing fields. This new field will enable tracking of I/O write operations specifically for the payload index as mentioned in the PR objectives.

lib/api/src/rest/models.rs (1)

60-60: Addition of payload_index_io_write field complements existing I/O metrics.

This new field logically extends the current hardware usage tracking by adding a dedicated counter for payload index write operations. It pairs with the existing payload_index_io_read field and follows the established naming pattern in the struct.

lib/api/src/grpc/qdrant.rs (1)

6521-6524: Field addition for payload index writes monitoring looks good

The addition of the payload_index_io_write field and subsequent reordering of the tag numbers follows the established pattern and enhances the hardware usage metrics. Since the payload_index_io_read field was introduced in PR #5951 and wasn't in production use yet (as noted in the retrieved learning), these changes maintain compatibility.

Just ensure that any client code is updated to handle the new field and the reordered tag numbers for vector_io_read and vector_io_write.

lib/storage/src/content_manager/toc/request_hw_counter.rs (2)

41-41: Good addition of payload_index_io_write field.

The addition of the payload_index_io_write field is consistent with the existing pattern of hardware usage tracking and properly matches the pattern with the corresponding read metric that was added in PR #5951.

---

57-57: Good addition to gRPC API representation.

The payload_index_io_write field is consistently added to both REST and gRPC API representations with proper type casting to u64 for the gRPC version.

lib/storage/src/content_manager/toc/mod.rs (1)

684-684: Good implementation of payload_index_io_write in metrics aggregation.

The addition of the payload_index_io_write field to the HardwareUsage struct in the all_hw_metrics method ensures that write operations for payload indexes are properly tracked and reported in aggregated metrics.

lib/collection/src/collection_manager/optimizers/indexing_optimizer.rs (1)

515-516: Addition of hardware counter for tracking payload index I/O writes

Adding the HardwareCounterCell instance here and passing it to process_field_index_operation provides a way to measure payload index I/O writes during the test, consistent with the PR's objective to implement measurement for the "create field-index" endpoint.

Also applies to: 524-524

lib/collection/src/shards/local_shard/shard_ops.rs (1)

91-91: Using clone() for hardware measurements accumulator

Good change to use clone() on the hardware measurement accumulator. This ensures the original accumulator remains unchanged while the operation gets its own copy to work with, which prevents any potential issues with shared mutable state.

lib/storage/tests/integration/alias_tests.rs (3)

127-127: Added hardware measurement for collection creation

Adding the hardware measurement accumulator to track performance metrics during collection creation is consistent with the PR's objective to enhance hardware usage measurements.

---

143-143: Added hardware measurement for alias creation

Adding the hardware measurement accumulator to track performance metrics during alias operations is consistent with the overall approach of enhancing hardware usage measurements across the codebase.

---

169-169: Added hardware measurement for complex alias operations

Adding the hardware measurement accumulator to track performance metrics during complex alias operations (create, delete, rename) completes the consistent pattern of enhancing hardware usage measurements for all collection meta operations.

lib/segment/tests/integration/segment_on_disk_snapshot.rs (1)

83-83: Added hardware counter to field index creation calls

Passing the hardware counter to create_field_index calls enables tracking of payload index I/O writes during the test. This change is consistent with the PR's objective to implement measurement for field index creation operations.

Also applies to: 99-99

lib/collection/src/tests/points_dedup.rs (1)

101-101: Added hardware metrics tracking for payload index creation

The addition of HwMeasurementAcc::new() parameter to the create_payload_index method call enables tracking of hardware metrics during payload index creation operations. This change aligns with the PR objective of implementing measurement for payload index operations.

lib/segment/tests/integration/byte_storage_hnsw_test.rs (3)

190-191: Added hardware counter for performance measurement

Creating a dedicated hardware counter instance before the indexing operations allows for proper tracking of hardware resource usage.

---

195-199: Updated method call to include hardware counter

The set_indexed call on the segment_float payload index has been updated to include the hardware counter parameter, enabling performance metrics collection during payload index operations.

---

204-208: Updated method call to include hardware counter

Similarly, the set_indexed call on the segment_byte payload index now includes the hardware counter parameter, ensuring consistent tracking of performance metrics across different segment types.

lib/storage/src/content_manager/mod.rs (2)

2-2: Added import for hardware measurement accumulator

This import supports the new parameter in the CollectionContainer trait method.

---

215-215:

✅ Verification successful

Modified trait method signature to include hardware metrics

The perform_collection_meta_op method in the CollectionContainer trait now accepts a hardware measurement accumulator. This is a significant interface change that will require updating all implementers of this trait to pass hardware measurement data.

---

🏁 Script executed:

#!/bin/bash
# Search for all implementations of the CollectionContainer trait to verify they've been updated
rg "impl.*CollectionContainer for" --type rust -A 10 | grep "perform_collection_meta_op"

Length of output: 399

---

Interface update verified: All implementations now correctly include the hardware metric accumulator parameter.

• The search confirmed that the perform_collection_meta_op method updates propagate to implementations in both lib/storage/src/content_manager/consensus_manager.rs and lib/storage/src/content_manager/toc/collection_container.rs, with the hardware accumulator (hw_acc) being passed appropriately.
• The snippet in lib/storage/src/content_manager/mod.rs accurately reflects the new parameter:

          hw_acc: HwMeasurementAcc,

• No outstanding issues remain regarding this interface change.

lib/segment/benches/boolean_filtering.rs (1)

111-115: Updated benchmark to include hardware counter in indexing

The set_indexed method call in the benchmark has been updated to include the hardware counter parameter, ensuring that the benchmarks can properly measure hardware resource usage during payload indexing operations.

lib/common/common/src/counter/hardware_data.rs (1)

10-10: New hardware metric field added for tracking payload index write operations.

The addition of payload_index_io_write field complements the existing payload_index_io_read field, creating a complete pair for tracking both read and write operations related to payload indexing. This is consistent with the other IO metric pairs in the struct.

lib/segment/benches/sparse_index_search.rs (1)

185-185: Updated set_indexed method call to include hardware counter.

The change adds the hardware counter as an argument to track IO write operations when creating payload field indices. This is consistent with the PR objective to measure payload index IO writes. The hw_counter is already properly initialized at line 89, so this change simply ensures it's utilized for accurate metrics tracking.

src/common/collections.rs (14)

23-23: Added hardware accumulator import.

This import is necessary to support the new hardware measurement functionality being added throughout the PR.

---

284-284: Added hardware measurement for cluster operations.

The addition of HwMeasurementAcc::disposable() to the MoveShard operation ensures method signature compatibility while indicating that this specific operation type doesn't need measurement. This is consistent with the overall approach for enhancing hardware metric tracking in the codebase.

---

321-322: Added hardware measurement for replica shard operations.

Similar to other cluster operations, the ReplicateShard operation now includes the hardware measurement accumulator to maintain API consistency.

---

353-354: Added hardware measurement for abort transfer operations.

The addition of the disposable hardware counter maintains API consistency across all cluster operations.

---

381-382: Added hardware measurement for drop replica operations.

Consistent application of the hardware measurement pattern to all collection meta operations.

---

455-456: Added hardware measurement for sharding key creation.

This follows the pattern established for all cluster operations, ensuring consistent API usage.

---

494-495: Added hardware measurement for drop sharding key operations.

Maintains consistency with the hardware measurement pattern applied to all other cluster operations.

---

539-540: Added hardware measurement for restart transfer operations.

Follows the established pattern for all cluster operations.

---

661-662: Added hardware measurement for start resharding operations.

Continues the consistent pattern of applying hardware measurement to all cluster operations.

---

688-689: Added hardware measurement for abort resharding operations.

Maintains consistency in the hardware measurement approach across cluster operations.

---

709-710: Added hardware measurement for finish resharding operations.

Follows the established pattern for all cluster operations.

---

761-762: Added hardware measurement for finish migrating points operations.

Maintains consistency in the hardware measurement approach.

---

791-792: Added hardware measurement for commit read hash ring operations.

Follows the established pattern for all cluster operations.

---

821-822: Added hardware measurement for commit write hash ring operations.

Completes the consistent application of hardware measurement to all cluster operations.

lib/collection/src/operations/verification/mod.rs (2)

305-305: Added hardware accumulator import for testing.

This import supports the newly added parameter in the create_payload_index method call in the test function.

---

520-520: Added hardware measurement to create_payload_index call in tests.

This change updates the test to properly include the new hardware measurement parameter when creating a payload index, ensuring tests correctly reflect the updated API. Using HwMeasurementAcc::new() is appropriate for this context since this is an actual operation that should be measured, unlike the cluster operations that use disposable().

lib/segment/tests/integration/filtrable_hnsw_test.rs (1)

146-151: Hardware counter now included in set_indexed call

The set_indexed method call has been updated to include the hardware counter reference as its third parameter, which aligns with the PR's objective of measuring payload index operations consistently.

lib/api/src/grpc/conversions.rs (2)

2731-2731: Added new payload_index_io_write field to HardwareUsage

The field has been added to track I/O write operations for the payload index, consistent with the PR's objective to measure payload index I/O writes.

---

2745-2745: Properly propagating payload_index_io_write field to HardwareData

The field is correctly included in the parameter list and properly mapped from the HardwareUsage struct to the HardwareData struct with appropriate type conversion from u64 to usize.

Also applies to: 2755-2755

src/consensus.rs (2)

417-417: New import for hardware measurement tracking.

Adding the HwMeasurementAcc import to support hardware metrics tracking in consensus operations.

---

1543-1543: Hardware measurement added to collection operations.

Adding HwMeasurementAcc::new() to the submit_collection_meta_op call allows tracking hardware usage metrics during collection creation operations. This aligns with the PR's goal of measuring payload index I/O writes.

lib/segment/tests/integration/multivector_filtrable_hnsw_test.rs (1)

173-177: Hardware counter added to set_indexed method.

The set_indexed method signature has been updated to include the hardware counter parameter, allowing tracking of resource usage during payload index creation.

lib/collection/src/collection/state_management.rs (2)

3-4: New import for hardware measurement.

Added import for HwMeasurementAcc to support hardware metrics tracking in collection state management.

---

210-211: Added hardware measurement to payload index creation.

Modified create_payload_index call to include HwMeasurementAcc::disposable() as a parameter. The comment explains that this is an unmeasured internal operation used only in consensus, which is good documentation.

lib/segment/tests/integration/gpu_hnsw_test.rs (2)

76-77: Hardware counter initialization added.

Created a new HardwareCounterCell instance at the function level rather than locally within the loop, which is more efficient and allows for consistent hardware usage tracking across all operations in the test.

---

115-119: Hardware counter added to set_indexed method.

Updated the set_indexed method call to include the hardware counter parameter, enabling resource tracking during payload index creation in the test.

lib/collection/src/collection_manager/optimizers/segment_optimizer.rs (6)

505-506: Good addition of a hardware counter for tracking operations

This change adds a disposable hardware counter that will be used for internal operations that don't require measurement. This aligns with the PR objective of improving hardware usage tracking.

---

517-522: Consistent approach for passing hardware metrics to create_field_index

The hardware counter is now appropriately passed to the create_field_index method, allowing tracking of payload index IO writes when this method is executed. This change is consistent with the PR's goal of adding measurement for field index operations.

---

539-540: Hardware counter integration for delete_point method

Adding the hardware counter to the delete_point method ensures consistent tracking of hardware metrics across different operations. This is part of a systematic approach to hardware usage tracking throughout the codebase.

---

691-692: Reusing the hardware counter pattern for the second optimization phase

This follows the same pattern established earlier in the file, adding a disposable hardware counter for internal operations in a different part of the optimization workflow.

---

707-712: Consistent usage of hardware counter for create_field_index

This is a good implementation that maintains consistency with the earlier part of the code, ensuring that all calls to create_field_index include the hardware counter.

---

734-735: Hardware counter integration for second delete_point operation

This completes the pattern of adding hardware counters to all relevant operations in the optimization process, ensuring comprehensive tracking of hardware usage across the entire workflow.

lib/segment/src/segment_constructor/segment_builder.rs (1)

536-540: Appropriate hardware counter integration for payload index operations

The hardware counter is correctly added for the set_indexed method, allowing tracking of hardware metrics when setting indexed fields. This fits with the PR objective of measuring payload index IO writes.

Using a disposable counter here is appropriate since this is happening during the segment build process, which is an internal operation that doesn't need external metrics reporting.

lib/common/common/src/counter/referenced_counter.rs (1)

40-43: Well-designed addition of payload index IO write counter reference method

This new method follows the established pattern in the file, providing a way to access just the payload index IO write counter from a HardwareCounterCell. The method is appropriately marked with the #[inline] attribute for optimization.

This is an essential addition that enables granular hardware metrics tracking for payload index operations, directly supporting the PR's goal of measuring payload index IO writes.

src/tonic/api/collections_api.rs (2)

19-19: Appropriate import for hardware measurement

Adding the import for HwMeasurementAcc is necessary to support the changes in the perform_operation method.

---

54-59: Correct integration of hardware measurement for collection operations

The addition of HwMeasurementAcc::disposable() as an argument to submit_collection_meta_op aligns with the PR objectives. The comment "Collection ops not measured" clearly indicates the intent that these operations don't need hardware measurement tracking.

This implementation maintains consistency with other changes across the codebase where hardware measurements are being integrated.

lib/segment/src/index/plain_payload_index.rs (1)

78-85: Added hardware counter parameter to build_index method.

The change adds a _hw_counter parameter to the build_index method signature to match the updated trait definition. The underscore prefix indicates this parameter isn't used in this implementation, which makes sense as this is a "plain" index that doesn't actually build indexes.

lib/storage/src/content_manager/toc/collection_container.rs (2)

10-10: Added import for HwMeasurementAcc.

This import is needed to support the hardware measurement tracking being added to the collection operations.

---

21-27: Updated method signature to include hardware metrics tracking.

The perform_collection_meta_op method now accepts a hardware measurement accumulator as a parameter and passes it through to the sync implementation. This change ensures consistent hardware metrics collection across collection metadata operations, particularly for payload index creation.

lib/segment/tests/integration/multivector_hnsw_test.rs (2)

48-48: Created hardware counter earlier in the test function.

The hardware counter is now created immediately after segment initialization, making it available for the subsequent call to create_field_index. This change aligns with the PR's objective of adding hardware measurement for the field-index creation operations.

---

55-55: Updated create_field_index call to include hardware counter.

The hardware counter is now passed to the create_field_index method, allowing for consistent tracking of hardware resources during index creation operations. This is in line with the broader changes in this PR.

lib/segment/tests/integration/disbalanced_vectors_test.rs (3)

64-65: Removed hardware counter parameter from delete_vector call.

The hardware counter parameter has been removed from the delete_vector method call, which aligns with the PR objective of removing measurements for vector deletions. This change is consistent with the focused approach of measuring only specific operations of interest.

---

67-68: Removed hardware counter parameter from delete_vector call.

The hardware counter parameter has been removed from the second delete_vector method call, maintaining consistency with the previous change.

---

72-73: Removed hardware counter parameter from delete_vector call.

The hardware counter parameter has been removed from the third delete_vector method call, completing the consistent removal across all vector deletion operations in this test.

lib/segment/src/index/field_index/full_text_index/tests/mod.rs (1)

180-180: Properly added hardware counter to the text index operation.

The new parameter consistently extends the method call to include hardware usage measurement, which aligns with the PR objective of tracking payload index I/O operations.

lib/segment/tests/integration/multivector_quantization_test.rs (1)

269-273: Added hardware counter parameter to set_indexed method appropriately.

This change properly integrates the hardware counter measurement for payload index operations, consistent with similar changes across the codebase. The formatting with line breaks maintains good readability.

lib/segment/tests/integration/sparse_vector_index_search_tests.rs (2)

392-393: Appropriately added hardware counter for payload index measurement.

Correctly creates a new hardware counter for tracking the upcoming indexed operations.

---

397-397: Properly updated set_indexed method signature.

The set_indexed method signature is updated to include the hardware counter as a parameter, maintaining consistency with other payload index operations throughout the codebase.

lib/collection/tests/integration/collection_test.rs (4)

544-549: Properly cloned hardware counter for asynchronous operation.

Cloning the hardware counter ensures each asynchronous operation has its own independent measurement accumulator, which is necessary when operations may execute concurrently.

---

558-558: Hardware counter properly cloned for payload index creation.

Consistent with earlier changes, the hardware counter is cloned to ensure independent measurement for this asynchronous index creation operation.

---

568-568: Hardware counter properly cloned for second index creation.

Correctly cloned the hardware counter for independent measurement tracking during this asynchronous operation.

---

578-578: Hardware counter properly cloned for third index creation.

Maintains consistency by cloning the hardware counter for this asynchronous index creation operation, ensuring accurate and independent measurement.

lib/segment/tests/integration/batch_search_test.rs (2)

41-42: LGTM: Hardware counter moved to a better location

Moving the hw_counter initialization outside the loop is a good refactoring. Now a single counter instance is shared across both the field index creation and point operations.

---

48-48: LGTM: Hardware counter parameter added for field index creation

Good addition of the hardware counter to track resource usage during field indexing operations, which aligns with the PR objective of implementing measurement for the "create field-index" endpoint.

lib/storage/src/content_manager/snapshots/recover.rs (2)

9-9: LGTM: Added necessary import for hardware measurement

The import for HwMeasurementAcc is properly added to support the measurement of hardware metrics.

---

153-158: LGTM: Added hardware measurement for collection recovery

Good addition of HwMeasurementAcc::disposable() to measure hardware usage during snapshot recovery operations. The disposable() method is appropriate here since this is a one-time operation.

lib/segment/tests/integration/exact_search_test.rs (1)

82-89: LGTM: Added hardware counter for payload index tracking

Good addition of the hardware counter parameter to the set_indexed method to measure resource usage during payload indexing operations. This change is consistent with the PR objective of tracking index IO writes.

lib/segment/src/index/field_index/numeric_index/tests.rs (7)

36-41: Method signature updated to include hardware counter tracking.

The add_point method now accepts a hw_counter parameter of type &HardwareCounterCell, which aligns with the hardware counter tracking pattern implemented across other index types. This is a consistent change that enables tracking I/O operations in field index implementations.

---

43-45: Hardware counter correctly passed to builder implementations.

The hardware counter is properly forwarded to each specific builder type, ensuring consistent measurement tracking across all index implementations.

---

90-91: Added hardware counter initialization in test method.

The hw_counter is now properly initialized and passed to the add_point method calls in the random_index function, enabling hardware usage tracking in tests.

Also applies to: 98-99

---

144-145: Hardware counter properly integrated in test_set_empty_payload test.

The counter is correctly initialized and passed to the add_point method, maintaining consistency with the updated method signature.

Also applies to: 147-147

---

285-286: Hardware counter added to test_payload_blocks_small.

Proper initialization and usage of the hardware counter in this test function ensures consistent monitoring of I/O operations during index building.

Also applies to: 291-291

---

322-323: Hardware counter integrated in test_numeric_index_load_from_disk.

The hardware counter is correctly initialized and passed to the add_point method calls, enabling accurate measurement of I/O operations during the test.

Also applies to: 328-328

---

384-385: Hardware counter added to test_numeric_index.

The hardware counter is properly integrated in this test function, maintaining consistency with the updated method signature and enabling performance monitoring.

Also applies to: 390-390

lib/common/common/src/counter/hardware_counter.rs (4)

17-17: Added new counter field for tracking payload index write operations.

The new field payload_index_io_write_counter follows the established naming convention and complements the existing payload_index_io_read_counter field, providing a complete picture of payload index I/O operations.

---

32-32: Counter field initialized in all constructor methods.

The new payload_index_io_write_counter field is properly initialized across all constructor methods (new, disposable, new_with_accumulator, and fork), ensuring consistent behavior regardless of how the counter cell is created.

Also applies to: 49-49, 63-63, 84-84

---

129-132: Added accessor method for the new counter field.

The implementation of payload_index_io_write_counter() follows the established pattern for accessor methods in this class, providing a consistent API for accessing the counter.

---

172-172: Updated hw_data collection to include the new counter.

The get_hw_data method now correctly includes the value from the new payload index write counter in the returned HardwareData structure, ensuring that these metrics are properly reported.

Also applies to: 183-183

lib/segment/src/index/field_index/numeric_index/mutable_numeric_index.rs (2)

263-263: Method signature updated to include hardware counter parameter.

The add_many_to_list method now accepts a hw_counter parameter, aligning with the pattern implemented across other index operations for consistent hardware usage tracking.

---

265-273: Added instrumentation to track write operations size.

This implementation correctly:

1. Initializes a counter to track the total bytes written
2. Accumulates the size of each key and index value during the write loop
3. Updates the hardware counter with the total bytes written after all operations complete

This approach efficiently captures the I/O write metrics with minimal overhead.

lib/segment/tests/integration/segment_builder_test.rs (1)

107-108: Hardware counter added to field index creation in tests.

The hardware counter is properly initialized and passed to the create_field_index method calls for both segments, ensuring consistent tracking of payload index operations during testing. This change follows the pattern established in the PR to measure payload index I/O writes.

Also applies to: 111-111, 116-116

src/actix/api/collections_api.rs (5)

8-8: Import added for new hardware measurement functionality.

This import is necessary to support the newly added hardware measurement parameters in API calls.

---

126-131: Consistent integration of hardware measurement in collection operations.

The added parameter HwMeasurementAcc::disposable() provides a disposable hardware measurement accumulator for the collection creation operation. The comment "API unmeasured" indicates that this operation doesn't need actual measurement tracking but requires the parameter to satisfy the method signature.

---

147-155: Hardware measurement tracking added to update operation.

Similar to other collection operations, the update collection operation now includes a disposable hardware measurement accumulator to maintain consistent method signatures across the codebase.

---

168-176: Hardware measurement added to delete collection operation.

The delete collection operation now includes hardware measurement tracking, consistent with other collection operations. The disposable accumulator is used since this is an API-level operation that doesn't need detailed hardware metrics.

---

189-195: Hardware measurement tracking added to alias operations.

The aliases update operation now includes hardware measurement tracking to maintain consistent method signatures across the codebase.

src/migrations/single_to_cluster.rs (3)

6-6: Import added for hardware measurement functionality.

This import is necessary to support the hardware measurement parameters in collection operations.

---

121-127: Hardware measurement added to collection meta operations.

The submit_collection_meta_op method now requires a hardware measurement accumulator parameter, which has been added consistently across the codebase. This change maintains the same functionality while incorporating hardware usage tracking.

---

133-145: Consistent hardware measurement tracking for replica state operations.

Similar to other collection operations, the set shard replica state operation now includes a disposable hardware measurement accumulator. The comment "Operation unmeasured" indicates that this migration operation doesn't need detailed hardware metrics.

lib/segment/src/segment/segment_ops.rs (1)

548-554: Hardware counter added to field index creation.

The create_field_index method now requires a hardware counter parameter, which has been added consistently across all method calls. The comment indicates that this function is used in Segment::load which is unmeasured, explaining why a disposable counter is sufficient here.

lib/segment/tests/integration/hnsw_quantized_search_test.rs (1)

78-85: Hardware counter parameter added to field index creation in tests.

The test now passes the hardware counter (hw_counter) to the create_field_index method, ensuring compatibility with the updated method signature. This change reflects the integration of hardware measurement capabilities in the field index creation process.

lib/collection/src/collection_manager/optimizers/vacuum_optimizer.rs (2)

442-443: Added hardware counter tracking for index operations.

The introduction of the hardware counter enables tracking payload index I/O operations, which is consistent with the PR objective of measuring payload index writes.

---

449-449: Hardware counter added to create_field_index method.

The hardware counter is now passed to the create_field_index method, allowing proper measurement of the payload index I/O operations during field indexing.

lib/segment/tests/integration/hnsw_discover_test.rs (1)

205-209: Updated set_indexed method call to include hardware counter.

The set_indexed method call now includes the hardware counter parameter, which aligns with the broader implementation of payload index I/O write measurements throughout the codebase.

lib/segment/tests/integration/byte_storage_quantization_test.rs (1)

304-308: Added hardware counter to set_indexed method call.

The method signature has been updated to include hardware counter, allowing for measurement of payload index I/O writes during indexing operations in quantization tests.

lib/segment/src/fixtures/payload_context_fixture.rs (2)

269-270: Initialized hardware counter for payload indexing operations.

Added hardware counter to track I/O operations in the fixture creation, which is consistent with the PR's goal of measuring payload index writes.

---

272-311: Updated all set_indexed calls to include hardware counter.

All six set_indexed method calls have been consistently updated to include the hardware counter parameter, ensuring proper measurement of payload index I/O operations across different payload types (string, integer, float, geo, text, boolean).

lib/segment/src/index/query_optimization/rescore_formula/value_retriever.rs (6)

198-199: LGTM: Correctly imported the HardwareCounterCell module

The addition of the HardwareCounterCell import is needed to support the hardware counter functionality being added to the tests.

---

301-301: LGTM: Properly instantiated hardware counter

Creating a new HardwareCounterCell instance to track performance metrics during test execution.

---

306-310: LGTM: Updated method call to include hardware counter

The NumericIndex builder's add_point method now receives the hardware counter parameter to track resource usage during index operations.

---

318-319: LGTM: Consistent parameter passing for hardware counter

Correctly passing the hardware counter to the GeoMapIndex builder's add_point method.

---

320-321: LGTM: Properly integrated hardware counter

Hardware counter is consistently passed to the add_point method for geo index operations.

---

323-324: LGTM: Complete hardware counter integration

The hardware counter parameter is properly passed to all add_point operations, ensuring consistent measurement across the test.

lib/segment/src/index/field_index/full_text_index/mmap_inverted_index/mod.rs (1)

144-149: LGTM: Added hardware counter parameter for consistency

The _hw_counter parameter has been added to match the trait interface, while maintaining the original functionality. The underscore prefix correctly indicates that the parameter is intentionally unused, as this method always returns an error.

lib/segment/src/segment/tests.rs (3)

494-496: LGTM: Removed hardware counter parameter from delete_vector call

The hardware counter parameter has been removed from the delete_vector method call, aligning with the method's updated signature.

---

500-502: LGTM: Consistently updated delete_vector call

Hardware counter parameter has been removed from this delete_vector call, maintaining consistency with the previous change.

---

670-671: LGTM: Consistently updated method call in test case

All instances of delete_vector have been updated to no longer include the hardware counter parameter, ensuring consistency throughout the test file.

lib/segment/src/index/field_index/full_text_index/mmap_text_index.rs (4)

5-5: Import added for hardware counter tracking.

The addition of the HardwareCounterCell import is appropriate for implementing the hardware usage measurements for payload index I/O operations.

---

93-98: Method signature updated with hardware counter parameter.

The add_many method signature has been updated to include the hardware counter parameter, which is essential for tracking I/O operations during payload indexing.

---

111-114: Hardware counter now passed to index_document.

The hardware counter is now correctly forwarded to the index_document method, enabling measurement of I/O writes during document indexing.

---

132-139: Method signature of add_point updated with hardware counter parameter.

The add_point method has been updated to accept the hardware counter parameter and passes it correctly to the underlying ValueIndexer::add_point method, maintaining consistency in hardware metric tracking across the indexing operations.

lib/segment/src/vector_storage/dense/dynamic_mmap_flags.rs (3)

8-8: Import added for hardware metrics reference counter.

The addition of the HwMetricRefCounter import is appropriate for implementing hardware usage measurements in the DynamicMmapFlags struct.

---

230-236: Method signature updated to include hardware counter reference.

The set_with_resize method now accepts a hardware counter reference, enabling tracking of I/O operations during flag setting operations.

---

246-247: Added measurement for array resize operation.

The implementation correctly measures the additional I/O write size when resizing the array, incrementing the counter by the difference between the new length and the current length.

src/common/update.rs (2)

704-711: Hardware measurement accumulator now passed to submit_collection_meta_op.

The submit_collection_meta_op call in the do_create_index function now correctly passes the hardware measurement accumulator, ensuring that hardware metrics are tracked when creating payload indices.

---

783-790: Hardware measurement accumulator now passed to submit_collection_meta_op in do_delete_index.

The hardware measurement accumulator is now correctly passed to the submit_collection_meta_op call in the do_delete_index function, ensuring consistent tracking of hardware metrics across different index operations.

lib/storage/src/dispatcher.rs (3)

9-9: Updated import for hardware measurement accumulator.

The import statement has been updated to include HwMeasurementAcc and HwSharedDrain, which are essential for implementing hardware usage tracking in the dispatcher.

---

66-72: Method signature updated to accept hardware measurement accumulator.

The submit_collection_meta_op method now accepts a hardware measurement accumulator parameter, enabling the tracking of hardware metrics during collection metadata operations.

---

222-223: Hardware measurement accumulator passed to perform_collection_meta_op.

The hw_acc parameter is now correctly passed to the perform_collection_meta_op method, ensuring that hardware metrics are tracked during collection metadata operations in non-distributed deployments.

lib/segment/src/index/field_index/map_index/mutable_map_index.rs (3)

47-47: Parameter addition to track hardware metrics.

The hw_counter parameter is being added to enable tracking of hardware metrics related to I/O operations for the payload index. This is part of a broader change to measure payload index I/O writes.

---

67-70: Payload index I/O write measurement added.

This new code increments the write counter after inserting a database record, tracking the exact size of data written to the payload index. This provides valuable metrics for monitoring I/O performance.

---

141-159: Optimized hardware counter measurement.

The implementation has been improved to batch hardware counter measurements. Instead of incrementing the read counter for each item size individually, the code now accumulates all sizes and performs a single counter increment at the end, which is more efficient.

lib/storage/src/content_manager/consensus_manager.rs (5)

17-17: New hardware measurement import.

Added import for HwMeasurementAcc from the hardware accumulator module, enabling hardware performance tracking for consensus operations.

---

495-495: Added hardware measurement for internal operations.

Introduced a disposable hardware measurement accumulator for tracking internal operations. This complements the payload index I/O write tracking added in other parts of the codebase.

---

498-498: Updated method call to include hardware counter.

The perform_collection_meta_op method call now includes the hardware accumulator parameter, enabling consistent hardware metric tracking across the system.

---

1115-1115: Updated test imports.

Added import for HwMeasurementAcc to the test module to ensure that tests remain consistent with the implementation changes.

---

1230-1230: Updated test implementation signature.

The test implementation of perform_collection_meta_op now includes the _hw_acc parameter to match the updated method signature in the main code.

lib/segment/benches/range_filtering.rs (2)

76-80: Updated method call to include hardware counter.

The set_indexed method call for the float key now includes the hardware counter parameter, allowing the benchmark to properly measure hardware performance metrics during indexing operations.

---

85-89: Updated method call to include hardware counter.

Similar to the float key change, the set_indexed method call for the integer key now includes the hardware counter parameter, ensuring consistent measurement of indexing operations across different data types.

src/actix/api/update_api.rs (4)

405-405: Added service configuration parameter.

The service_config parameter enables access to hardware reporting configuration, allowing the API to control when hardware metrics should be reported based on service settings.

---

411-416: Implemented structured hardware metrics collection.

Replaced the previous ad-hoc hardware measurement with a more structured approach using get_request_hardware_counter. This provides consistent hardware metric collection that respects service configuration settings and wait parameters.

---

425-425: Updated hardware counter usage in index creation.

The do_create_index call now uses the request-specific hardware counter instead of a disposable one, enabling proper tracking and attribution of hardware metrics to specific requests.

---

429-429: Added hardware usage reporting to response.

The response now includes hardware usage metrics when available, providing clients with valuable information about resource utilization during the index creation operation.

lib/segment/tests/integration/nested_filtering_test.rs (1)

83-119: Added hardware counter to all set_indexed calls

The changes consistently pass the hardware counter reference to all set_indexed method calls for proper IO write measurement. This aligns with the objective of adding measurement for payload index IO writes.

lib/segment/src/index/field_index/full_text_index/mutable_inverted_index.rs (3)

82-95: Properly measuring memory allocations for document indexing

The implementation correctly measures the memory overhead of allocating new Document instances when resizing the vector. The comment clearly explains that only the document overhead is measured here, as token measurements are handled separately.

---

98-118: Efficiently tracking postings memory usage with batched counter updates

Great approach using a local accumulator variable hw_counter_val to batch the counter updates instead of incrementing on each iteration. This reduces the overhead of accessing the counter cell too frequently.

---

121-122: Single counter increment with accumulated value

Efficiently updating the hardware counter once with the accumulated value from all operations rather than multiple small increments.

lib/segment/src/index/payload_index_base.rs (2)

24-29: Added hardware counter to the build_index trait method

The hardware counter parameter has been appropriately added to the build_index method signature in the PayloadIndex trait to enable performance measurements during index building.

---

40-48: Updated set_indexed method to include hardware counter

The method signature has been updated to include the hardware counter parameter, and it's correctly passed to the build_index call within the method implementation.

lib/segment/src/index/field_index/bool_index/mod.rs (2)

189-199: Added hardware counter parameter to add_many method

The add_many method now includes the hardware counter parameter and properly passes it to the underlying implementation based on the enum variant.

---

300-302: Updated all test cases to use hardware counter

All test cases have been consistently updated to create and use a hardware counter instance, ensuring the test suite verifies the new measurement capabilities.

Also applies to: 352-353, 398-402, 436-443, 458-465, 485-492

lib/segment/src/index/field_index/full_text_index/text_index.rs (2)

274-281: Properly added hardware counter to the add_point method

The method now correctly accepts and passes the hardware counter to track resource usage during indexing operations, maintaining consistency with other index implementations.

---

291-299: Hardware counter integration looks good

The addition of the hardware counter parameter to the add_many method and its propagation to the underlying implementation is correct. This change will enable tracking of I/O operations during payload indexing.

lib/collection/src/collection_manager/holders/segment_holder.rs (1)

1115-1119: Properly implemented hardware counter for field index creation

The addition of a disposable hardware counter for the internal field index creation operation is appropriate. This change enables tracking of payload index I/O write operations without affecting the public API.

lib/collection/src/collection/payload_index_schema.rs (2)

36-46: Good propagation of hardware measurement accumulator

The hw_acc parameter is properly added to the method signature and correctly passed to the create_payload_index_with_wait method, maintaining the chain of hardware performance tracking.

---

48-74: Hardware counter integration properly implemented

The hardware measurement accumulator is now correctly passed through to the update_all_local method rather than creating a disposable one. This allows for accurate tracking of resource usage across the entire operation chain.

lib/segment/src/index/field_index/null_index/mmap_null_index.rs (5)

101-106: Hardware counter parameter correctly added to method signature

The method signature has been properly updated to include the hardware counter parameter, consistent with other index implementations.

---

140-146: Good implementation of hardware counter tracking

The implementation correctly gets a reference to the payload index I/O write counter and uses it in the set_with_resize method calls for both slices, enabling proper tracking of write operations.

---

150-157: Appropriate use of disposable hardware counter for unmeasured operations

Using a disposable hardware counter for delete operations is consistent with the codebase's approach to unmeasured operations. The code correctly creates and uses the counter reference.

---

352-359: Hardware counter properly propagated in builder implementation

The builder implementation correctly passes the hardware counter parameter to the underlying add_point method, maintaining the chain of performance tracking.

---

386-397: Tests updated correctly for hardware counter usage

The test cases have been properly updated to create and pass hardware counters to the methods under test, ensuring that the new functionality is properly exercised.

lib/api/src/grpc/proto/points.proto (1)

1172-1174: Addition of payload_index_io_write field looks good

The new field has been correctly added to the HardwareUsage message in the Proto definition. The field numbering is sequential and the field is properly documented with its type.

lib/storage/src/content_manager/toc/collection_meta_ops.rs (7)

11-11: Import for HwMeasurementAcc added correctly

The import for the hardware measurement accumulator has been properly added.

---

25-25: Parameter addition for hardware measurement is appropriate

Adding the hw_acc parameter to this method ensures that hardware measurements can be properly tracked and accumulated during collection metadata operations.

---

28-28: Parameter correctly passed to async implementation

The synchronous wrapper properly forwards the hw_acc parameter to the async implementation.

---

34-34: Parameter added to async implementation signature

Adding the hw_acc parameter to the async implementation ensures consistent tracking of hardware metrics through the call chain.

---

107-107: Hardware counter parameter correctly passed to create_payload_index function

The hw_acc parameter is properly forwarded to the create_payload_index method which implements the measurement for the "create field-index" endpoint.

---

631-631: Parameter addition to create_payload_index method signature

This parameter addition enables hardware metrics collection during payload index creation operations.

---

635-635: Hardware measurement parameter correctly forwarded to collection method

The hardware accumulator is properly passed to the collection's create_payload_index method, allowing for measurement of payload index IO operations at the appropriate level.

src/common/metrics.rs (2)

335-335: New hardware metric field added correctly

The payload_index_io_write field has been properly added to the destructuring of hw_info in the HardwareTelemetry implementation.

---

357-372: Updated metrics type and added new metric for payload index IO write

The changes include:

1. Changing payload_index_io_read from a GAUGE to a COUNTER type for consistent metric types
2. Adding a new payload_index_io_write metric as a COUNTER to track total IO write operations for payload indexes

These changes properly integrate the new metric within the existing metrics infrastructure.

lib/common/common/src/counter/hardware_accumulator.rs (6)

16-16: Added new counter field for payload index IO write operations

The new field payload_index_io_write_counter has been properly added to the HwSharedDrain struct, following the established naming pattern.

---

38-40: Added accessor method for the new counter

The get_payload_index_io_write method follows the existing pattern for counter accessors, correctly using Ordering::Relaxed which is appropriate for metrics that don't require strong ordering guarantees.

---

57-57: Updated accumulation logic to include new counter

The accumulate_from_hw_data method has been properly updated to include the new payload_index_io_write counter, ensuring it's correctly accumulated from the source data.

Also applies to: 66-66

---

74-82: Updated Clone implementation to match new field structure

The Clone implementation has been refactored to use direct field initialization instead of the removed new method, properly including the new field.

---

92-92: Added new counter to Default implementation

The Default implementation has been updated to initialize the new counter field with a zeroed AtomicUsize.

---

177-179: Added accessor method to HwMeasurementAcc for the new counter

The get_payload_index_io_write method has been added to the HwMeasurementAcc struct, maintaining API consistency by delegating to the corresponding method in request_drain.

lib/segment/src/index/field_index/full_text_index/inverted_index.rs (3)

86-91: Added hardware counter parameter to InvertedIndex trait method

The index_document method has been updated to include a hw_counter parameter, allowing the method to track hardware usage during document indexing operations. This change aligns with the PR objective to track payload index IO writes.

---

246-247: Hardware counter added to test function

A new HardwareCounterCell instance is appropriately created in the test module to support the updated method signature.

---

253-253: Updated test call to include hardware counter

The call to index_document has been correctly updated to include the new hw_counter parameter, ensuring the test continues to work with the updated method signature.

lib/segment/src/index/field_index/geo_index/mutable_geo_index.rs (5)

140-145: Added hardware counter parameter to MutableGeoMapIndex

The add_many_geo_points method signature now includes the hw_counter parameter, consistent with other index implementations in the codebase.

---

155-157: Updated method call to pass hardware counter

The call to the underlying in_memory_index.add_many_geo_points has been updated to pass the new hw_counter parameter, ensuring proper propagation of the hardware metrics tracking.

---

310-315: Added hardware counter parameter to InMemoryGeoMapIndex

The add_many_geo_points method signature in the InMemoryGeoMapIndex implementation has been updated to include the hw_counter parameter, consistent with other implementations.

---

329-336: Added payload index IO write tracking

The hardware counter is now used to track the size of each added geo hash, measuring the precise amount of data being written to the payload index.

---

349-350: Added tracking for point offset writes

The hardware counter now also tracks the memory used for storing point offsets, providing more comprehensive IO write measurements.

lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs (6)

3-3: Added hardware counter import

The necessary import for HardwareCounterCell has been added to support the new parameter.

---

50-55: Updated add_many method signature

The add_many method now includes the hw_counter parameter, consistent with the pattern used across the project for measuring hardware usage.

---

69-70: Pass hardware counter to inverted index

The call to index_document now includes the hw_counter parameter, ensuring that hardware usage is tracked during document indexing.

---

153-159: Updated test to use hardware counter

A new hw_cell instance is created in the test and passed to the add_point call, making the test compatible with the updated method signatures.

---

205-205: Updated add_point test call

Another call to add_point has been properly updated to include the hardware counter parameter.

---

210-210: Consistently updated all test calls

All instances of add_point in the test have been updated to pass the hardware counter, ensuring comprehensive test coverage of the new functionality.

lib/segment/src/index/field_index/bool_index/simple_bool_index.rs (3)

334-341: Updated BoolIndexBuilder add_point signature

The add_point method implementation now includes the hw_counter parameter and correctly passes it to the underlying implementation.

---

463-468: Updated SimpleBoolIndex add_many signature

The add_many method signature has been updated to include the hw_counter parameter, consistent with the pattern used across the project.

---

480-487: Added payload index I/O write tracking for boolean indexes

The hardware counter is now used to track both the point ID size and the serialized boolean item size, providing accurate measurements of I/O write operations for boolean indexes.

lib/segment/tests/integration/payload_index_test.rs (3)

98-104: Hardware counter integration looks good

The addition of hardware counter parameters to create_field_index calls allows proper tracking of payload index I/O operations, which aligns well with the PR objective.

---

290-291: Good initialization of hardware counter for mmap segment

Properly initialized hardware counter before using it in the subsequent field index operations on the mmap segment.

---

402-403: Hardware counter properly initialized for nested payload tests

Good placement of the hardware counter initialization at the beginning of the nested payload test function.

lib/collection/src/collection_manager/holders/proxy_segment.rs (4)

102-103: Good use of disposable hardware counter for internal operations

Using HardwareCounterCell::disposable() for internal operations is appropriate since these operations should be tracked separately from client-initiated operations.

---

580-581: Improved measurement approach for move operations

The change correctly adds hardware measurement to the move_if_exists operation using a disposable counter, which helps track internal payload index I/O operations.

---

584-585: Hardware counter parameter removal is correct

Removing the hardware counter parameter from delete_vector while adding it to move_if_exists properly refactors where measurements are taken - now at the higher-level operation rather than the individual vector operation.

---

1172-1173: Good method signature update

Properly updated the build_field_index method signature to include hardware counter tracking, which ensures consistent measurement throughout the index building process.

lib/collection/src/collection_manager/collection_updater.rs (3)

58-59: Correct integration of hardware counter for field index operations

The update properly passes the hardware counter to process_field_index_operation, enabling measurement of payload index I/O writes during field indexing operations.

---

362-363: Good test refactoring

Moved the hardware counter initialization to the beginning of the test function, improving code organization and ensuring the counter is available for all operations in the test.

---

370-371: Consistent hardware counter usage in tests

The test now correctly passes hardware counters to field index operations, ensuring that the measurement functionality is properly tested.

Also applies to: 380-381

lib/segment/src/index/field_index/bool_index/mmap_bool_index.rs (5)

85-91: Enhanced tracking of hardware resource usage for bool index operations.

The signature of set_or_insert method has been updated to include hw_counter parameter, allowing measurement of payload index I/O write operations.

---

93-96: Propagating hardware counter to set_or_insert_flag calls.

The calls to set_or_insert_flag have been updated to pass the hardware counter parameter, maintaining the measurement chain.

---

271-281: Proper implementation of hardware counter for bool operations.

The implementation tracks both the boolean value write and potential memory resizing operations, providing accurate metrics on payload index I/O usage.

---

287-288: Tracking memory resizing operations with hardware counter.

The counter is correctly incremented with the exact size difference when expanding the flags array.

---

351-353: Appropriate use of disposable hardware counter for remove operations.

Using a disposable hardware counter for the remove_point operation ensures that these operations don't count toward the global hardware metrics, which aligns with the PR objective of specifically measuring index write operations.

lib/segment/src/entry/entry_point.rs (2)

276-282: Hardware metrics now available for field index building.

The build_field_index method signature has been updated to include hardware counter tracking, which enables measuring I/O operations during index creation.

---

294-303: Measurement integration for field index creation.

The create_field_index method now accepts and forwards the hardware counter to the underlying build_field_index call, completing the measurement chain for payload index operations.

This change implements the measurement for the "create field-index" endpoint as mentioned in the PR objectives.

lib/segment/src/index/field_index/field_index_base.rs (4)

77-82: Hardware counter added to ValueIndexer trait.

The add_many method signature now includes a hardware counter parameter, providing a consistent interface for tracking I/O operations across all value indexers.

---

96-117: ValueIndexer's add_point now propagates hardware usage metrics.

The method has been updated to include and forward the hardware counter to the add_many method, ensuring consistent measurement across the indexing operations.

---

284-325: Consistent hardware counter propagation across all field index types.

The add_point method in FieldIndex now properly passes the hardware counter to all index implementations, ensuring comprehensive tracking regardless of the index type.

---

433-532: Hardware metrics now tracked for all field index builders.

All field index builder implementations now accept and propagate the hardware counter, creating a consistent tracking system across the entire indexing subsystem.

lib/segment/src/index/field_index/geo_index/mod.rs (6)

364-371: Hardware counter added to GeoMapIndexBuilder.

The add_point implementation now includes hardware resource tracking, consistent with other index types.

---

401-408: Hardware measurement support for GeoMapImmutableIndexBuilder.

The immutable geo index builder now properly tracks hardware usage during indexing operations.

---

431-443: MmapBuilder for geo indices now includes resource measurement.

The implementation properly forwards the hardware counter to the underlying in-memory index's add operation.

---

457-464: ValueIndexer implementation for GeoMapIndex now tracks hardware usage.

This change ensures consistent hardware measurement across both normal and geo-specific indexing operations.

---

675-687: Test utility updated to support hardware metrics.

The IndexBuilder test helper now includes hardware counter propagation, ensuring the tests properly validate the real implementation.

---

772-776: Test cases updated to include hardware counter instances.

All test cases that call indexing operations have been updated to provide hardware counter instances, allowing proper testing of the measurement infrastructure.

Also applies to: 1134-1135, 1218-1219, 1265-1266, 1324-1327, 1461-1463, 1472-1482

lib/collection/src/collection_manager/segments_updater.rs (4)

144-144: Confirm removal of hardware counter usage.

Previously, this call could have included hw_counter. If you do not need to track hardware usage for vector deletions, this is fine. Otherwise, ensure that removing hardware counter measurement here is intentional.

---

612-614: No hardware usage measurement for vector deletion.

The DeleteVectors operation no longer passes hw_counter. Verify that you no longer need hardware usage metrics for vector deletion, as this change may cause missing measurements in your telemetry.

---

344-349: Retaining hardware counter in create_field_index.

You still pass hw_counter here while removing it in other operations like delete_vectors. If this is intended for measuring field-index creation only, confirm consistency with your overall hardware usage tracking strategy.

---

690-699: Hardware counter usage consistency.

process_field_index_operation still receives and uses hw_counter. Ensure that removing the hardware counter in other vector-related functions is justified and that your telemetry remains consistent across different operations.

lib/segment/src/index/field_index/map_index/mod.rs (8)

2-2: New import of hash_map::Entry.

Good call on adding Entry import. It can simplify insertion and lookup logic by leveraging the entry API for more efficient updates.

---

455-461: Adding hardware counter to add_point in MapIndexBuilder.

Including hw_counter ensures that index-building operations are captured in hardware usage metrics, improving observability.

---

487-525: Double-check size accounting for each flattened value.

The code increments hw_counter_val by size_of_val(&id) for every value in flatten_values. Verify that you expect the ID overhead to be counted once per value rather than once per unique point. If correct, all good; otherwise, consider adjusting the accounting.

---

1065-1072: Added hw_counter in add_many for MapIndex<str>.

This integration further aligns hardware usage tracking with string-based map indexing, ensuring consistency throughout.

---

1097-1112: Hardware counter added for MapIndex<IntPayloadType>'s add_many.

Looks consistent with other indexers. Good job ensuring comprehensive telemetry on integer payload insertions.

---

1130-1144: Hardware counter added for MapIndex<UuidIntType>'s add_many.

Maintaining uniform hardware usage tracking across all payload types is commendable.

---

1185-1209: Added hw_counter usage in save_map_index tests.

Incorporating hardware counter checks in test code can help validate that indexing operations are measured correctly.

---

1255-1263: Hardware counter usage in test_index_non_ascending_insertion.

Providing a counter in this test helps confirm that insertion order does not affect hardware measurement logic. Nicely done.

lib/segment/src/index/struct_payload_index.rs (8)

7-8: Import of HwMeasurementIteratorExt for hardware measurements

The addition of this import enables hardware measurement for iterators through the measure_hw_with_cell extension method used later in the code to track resource usage.

---

148-152: Good internal hardware measurement initialization

You've appropriately added hardware measurement to the internal build_field_indexes call, using a disposable counter since this is an internal operation where the measurements don't need to be propagated upward.

---

216-221: Hardware counter parameter added consistently

The hw_counter parameter addition to build_field_indexes signature ensures consistent hardware metric collection throughout the codebase, following the pattern established in the PR dependencies.

---

234-235: Propagating the hardware counter to add_point

Good propagation of the hardware counter to the add_point calls, ensuring that payload index operations are properly measured at each level of the operation.

---

388-395: Added hardware measurement for fallback full scan

Great implementation of hardware measurement for the fallback case when an index is not built. The use of measure_hw_with_cell ensures that CPU usage is properly tracked even when performing a full scan of IDs.

---

452-454: Hardware counter properly propagated to build_index

The hardware counter parameter has been correctly added to the build_index method signature and is properly forwarded to the build_field_indexes method, ensuring consistent measurement throughout the index building process.

Also applies to: 463-464

---

570-575: Consistent hardware counter propagation in overwrite_payload

The hardware counter is correctly passed to the inner add_point method during payload overwrite operations, maintaining the consistent tracking of resource usage.

Also applies to: 584-585

---

595-601: Hardware counter properly used in set_payload

The hardware counter is correctly propagated to inner operations in the set_payload method, ensuring consistent resource tracking across the payload modification operations.

Also applies to: 620-621

lib/segment/src/index/field_index/numeric_index/mod.rs (7)

529-536: Hardware counter propagation in NumericIndexBuilder

The add_point method signature has been correctly updated to include the hardware counter parameter, which is then properly passed to the underlying implementation.

---

570-577: Hardware counter added to NumericIndexImmutableBuilder

The add_point method in NumericIndexImmutableBuilder now accepts and properly forwards the hardware counter to the underlying implementation.

---

614-637: Hardware measurements for payload index I/O writes

Excellent implementation of hardware resource tracking in the NumericIndexMmapBuilder. The code now correctly increments the payload index I/O write counter based on the size of the flattened values, providing accurate measurement of index write operations.

---

826-833: Hardware counter for integer payload indexing

The add_many method for IntPayloadType now consistently passes the hardware counter to the underlying implementation, ensuring proper tracking of resource usage during index operations.

---

862-874: Hardware counter for datetime payload indexing

The addition of the hardware counter to the add_many method for DateTimePayloadType and its propagation to the add_many_to_list call ensures consistent tracking of performance metrics during datetime indexing operations.

---

903-911: Hardware counter for float payload indexing

The hardware counter parameter has been properly added to the add_many method for FloatPayloadType and is correctly passed to the underlying implementation.

---

940-950: Hardware counter for UUID payload indexing

The hardware counter is now properly tracked in UUID payload indexing operations, maintaining the consistency of the performance measurement infrastructure across different payload types.

lib/segment/src/segment/entry.rs (3)

746-752: Added hardware counter parameter to build_field_index signature

The method signature has been updated to include hw_counter parameter, aligning with PR's goal to measure payload index IO operations. This change allows hardware usage tracking for the "create field-index" endpoint.

---

763-764: Added hardware counter to first build_index call

The hw_counter parameter is now passed to the first build_index method call, allowing for tracking of hardware resource usage during index building operations.

---

778-779: Added hardware counter to second build_index call

The hw_counter parameter is now passed to the second build_index method call in the type inference branch, ensuring consistent tracking of hardware usage regardless of which branch executes.

Reviewing files that changed from the base of the PR and between 2caa6e1 and 6550930.

• docs/grpc/docs.md (1 hunks)
• docs/redoc/master/openapi.json (2 hunks)
• lib/api/src/grpc/conversions.rs (3 hunks)
• lib/api/src/grpc/proto/points.proto (1 hunks)
• lib/api/src/grpc/qdrant.rs (1 hunks)
• lib/api/src/rest/models.rs (1 hunks)
• lib/collection/src/collection/payload_index_schema.rs (3 hunks)
• lib/collection/src/collection/state_management.rs (2 hunks)
• lib/collection/src/collection_manager/collection_updater.rs (3 hunks)
• lib/collection/src/collection_manager/holders/proxy_segment.rs (9 hunks)
• lib/collection/src/collection_manager/holders/segment_holder.rs (1 hunks)
• lib/collection/src/collection_manager/optimizers/indexing_optimizer.rs (1 hunks)
• lib/collection/src/collection_manager/optimizers/segment_optimizer.rs (6 hunks)
• lib/collection/src/collection_manager/optimizers/vacuum_optimizer.rs (1 hunks)
• lib/collection/src/collection_manager/segments_updater.rs (5 hunks)
• lib/collection/src/operations/verification/mod.rs (2 hunks)
• lib/collection/src/shards/local_shard/shard_ops.rs (1 hunks)
• lib/collection/src/tests/points_dedup.rs (1 hunks)
• lib/collection/tests/integration/collection_test.rs (4 hunks)
• lib/common/common/src/counter/hardware_accumulator.rs (6 hunks)
• lib/common/common/src/counter/hardware_counter.rs (8 hunks)
• lib/common/common/src/counter/hardware_data.rs (1 hunks)
• lib/common/common/src/counter/referenced_counter.rs (1 hunks)
• lib/segment/benches/boolean_filtering.rs (1 hunks)
• lib/segment/benches/range_filtering.rs (1 hunks)
• lib/segment/benches/segment_info.rs (1 hunks)
• lib/segment/benches/sparse_index_search.rs (1 hunks)
• lib/segment/src/entry/entry_point.rs (2 hunks)
• lib/segment/src/fixtures/payload_context_fixture.rs (1 hunks)
• lib/segment/src/index/field_index/bool_index/mmap_bool_index.rs (6 hunks)
• lib/segment/src/index/field_index/bool_index/mod.rs (7 hunks)
• lib/segment/src/index/field_index/bool_index/simple_bool_index.rs (3 hunks)
• lib/segment/src/index/field_index/field_index_base.rs (6 hunks)
• lib/segment/src/index/field_index/full_text_index/immutable_inverted_index.rs (1 hunks)
• lib/segment/src/index/field_index/full_text_index/inverted_index.rs (2 hunks)
• lib/segment/src/index/field_index/full_text_index/mmap_inverted_index/mod.rs (1 hunks)
• lib/segment/src/index/field_index/full_text_index/mmap_text_index.rs (4 hunks)
• lib/segment/src/index/field_index/full_text_index/mutable_inverted_index.rs (2 hunks)
• lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs (6 hunks)
• lib/segment/src/index/field_index/full_text_index/tests/mod.rs (1 hunks)
• lib/segment/src/index/field_index/full_text_index/text_index.rs (2 hunks)
• lib/segment/src/index/field_index/geo_index/mod.rs (13 hunks)
• lib/segment/src/index/field_index/geo_index/mutable_geo_index.rs (5 hunks)
• lib/segment/src/index/field_index/map_index/immutable_map_index.rs (1 hunks)
• lib/segment/src/index/field_index/map_index/mod.rs (11 hunks)
• lib/segment/src/index/field_index/map_index/mutable_map_index.rs (3 hunks)
• lib/segment/src/index/field_index/null_index/mmap_null_index.rs (4 hunks)
• lib/segment/src/index/field_index/numeric_index/mod.rs (8 hunks)
• lib/segment/src/index/field_index/numeric_index/mutable_numeric_index.rs (1 hunks)
• lib/segment/src/index/field_index/numeric_index/tests.rs (6 hunks)
• lib/segment/src/index/payload_index_base.rs (2 hunks)
• lib/segment/src/index/plain_payload_index.rs (1 hunks)
• lib/segment/src/index/query_optimization/rescore_formula/value_retriever.rs (2 hunks)
• lib/segment/src/index/struct_payload_index.rs (9 hunks)
• lib/segment/src/segment/entry.rs (3 hunks)
• lib/segment/src/segment/segment_ops.rs (1 hunks)
• lib/segment/src/segment/tests.rs (2 hunks)
• lib/segment/src/segment_constructor/segment_builder.rs (1 hunks)
• lib/segment/src/vector_storage/dense/dynamic_mmap_flags.rs (2 hunks)
• lib/segment/tests/integration/batch_search_test.rs (1 hunks)
• lib/segment/tests/integration/byte_storage_hnsw_test.rs (1 hunks)
• lib/segment/tests/integration/byte_storage_quantization_test.rs (1 hunks)
• lib/segment/tests/integration/disbalanced_vectors_test.rs (1 hunks)
• lib/segment/tests/integration/exact_search_test.rs (1 hunks)
• lib/segment/tests/integration/filtrable_hnsw_test.rs (1 hunks)
• lib/segment/tests/integration/gpu_hnsw_test.rs (2 hunks)
• lib/segment/tests/integration/hnsw_discover_test.rs (1 hunks)
• lib/segment/tests/integration/hnsw_quantized_search_test.rs (1 hunks)
• lib/segment/tests/integration/multivector_filtrable_hnsw_test.rs (1 hunks)
• lib/segment/tests/integration/multivector_hnsw_test.rs (1 hunks)
• lib/segment/tests/integration/multivector_quantization_test.rs (1 hunks)
• lib/segment/tests/integration/nested_filtering_test.rs (1 hunks)
• lib/segment/tests/integration/payload_index_test.rs (15 hunks)
• lib/segment/tests/integration/segment_builder_test.rs (1 hunks)
• lib/segment/tests/integration/segment_on_disk_snapshot.rs (2 hunks)
• lib/segment/tests/integration/sparse_vector_index_search_tests.rs (1 hunks)
• lib/storage/src/content_manager/consensus_manager.rs (4 hunks)
• lib/storage/src/content_manager/mod.rs (2 hunks)
• lib/storage/src/content_manager/snapshots/recover.rs (2 hunks)
• lib/storage/src/content_manager/toc/collection_container.rs (2 hunks)
• lib/storage/src/content_manager/toc/collection_meta_ops.rs (4 hunks)
• lib/storage/src/content_manager/toc/mod.rs (1 hunks)
• lib/storage/src/content_manager/toc/request_hw_counter.rs (2 hunks)
• lib/storage/src/dispatcher.rs (3 hunks)
• lib/storage/tests/integration/alias_tests.rs (4 hunks)
• src/actix/api/collections_api.rs (5 hunks)
• src/actix/api/update_api.rs (1 hunks)
• src/common/collections.rs (14 hunks)
• src/common/metrics.rs (2 hunks)
• src/common/update.rs (2 hunks)
• src/consensus.rs (2 hunks)
• src/migrations/single_to_cluster.rs (3 hunks)
• src/tonic/api/collections_api.rs (2 hunks)

• GitHub Check: test-snapshot-operations-s3-minio
• GitHub Check: test-consensus
• GitHub Check: test (windows-latest)
• GitHub Check: test
• GitHub Check: test (ubuntu-latest)

docs/redoc/master/openapi.json (1)

12668-12672: Addition of payload_index_io_write field enhances I/O metrics tracking

The new payload_index_io_write field is a well-structured addition to the HardwareUsage schema, used to track write operations performed on the payload index during request processing. This metric complements the existing payload_index_io_read field and follows the established pattern of I/O metrics in the system.

The field is correctly defined as an unsigned integer with a minimum value of 0, consistent with other hardware counter fields in the schema.

lib/common/common/src/counter/hardware_data.rs (1)

10-10: Consistent addition of hardware metric

Good addition of payload_index_io_write field to track write operations on the payload index, which complements the existing payload_index_io_read field. This is a logical extension to complete the monitoring of both read and write operations.

lib/api/src/rest/models.rs (1)

60-60: Field addition aligns with data model changes

The new payload_index_io_write field in the HardwareUsage struct correctly mirrors the change made to the HardwareData struct, ensuring consistency across the API and internal implementations.

lib/segment/tests/integration/segment_on_disk_snapshot.rs (1)

83-83: Hardware counter parameter added consistently

Correctly added the hardware counter parameter to both create_field_index method calls, which allows for proper measurement of payload index I/O operations during index creation. This change aligns with the purpose of this PR to measure payload index I/O writes.

Also applies to: 99-99

lib/segment/benches/segment_info.rs (1)

29-29: Hardware counter added to benchmark

Good addition of the hardware counter parameter to the create_field_index method call in the benchmark, ensuring that the benchmark code reflects the updated method signature. This maintains consistency with the implementation code.

Note that you're creating a new counter instance here rather than reusing the existing one declared at line 35. This is appropriate since this call occurs before the main benchmark loop.

lib/storage/src/content_manager/toc/mod.rs (1)

684-684: Adding payload_index_io_write field to hardware metrics

This change adds the new payload index IO write measurement field to the hardware usage metrics collection, ensuring write operations on the payload index are properly tracked alongside other hardware metrics.

lib/collection/src/collection_manager/optimizers/indexing_optimizer.rs (2)

515-516: Hardware counter added for field index creation

Adding a hardware counter here allows for measuring resources used during field index creation operations in tests.

---

524-524: Passing hardware counter to field index operation

The hardware counter is now passed to the process_field_index_operation function, enabling it to track IO operations during index creation - consistent with the PR objective to measure "create field-index" operations.

lib/collection/src/shards/local_shard/shard_ops.rs (1)

91-91: Using clone to preserve hardware measurements

Changed to use a clone of the hardware measurements accumulator rather than moving ownership. This ensures the hardware counter can be safely used elsewhere if needed after this point.

lib/storage/src/content_manager/toc/request_hw_counter.rs (2)

41-41: Adding payload_index_io_write to REST API metrics

This change exposes the new payload index IO write metric through the REST API, completing the hardware metrics exposure for this interface.

---

57-57: Adding payload_index_io_write to gRPC API metrics

This change exposes the new payload index IO write metric through the gRPC API with appropriate type conversion, maintaining consistency with other metrics in the interface.

src/consensus.rs (2)

1414-1417: New hardware measurement integration for testing

The addition of the HwMeasurementAcc import allows for hardware measurement tracking in the consensus tests, which supports the PR objective of measuring payload index IO writes.

---

1543-1544: Collection creation test now leverages hardware measurement

Passing HwMeasurementAcc::new() to the submit_collection_meta_op call properly integrates hardware performance measurement into the consensus test, aligning with the new function signature requirements.

lib/common/common/src/counter/referenced_counter.rs (1)

40-43: New payload index IO write counter reference method

This new method provides access to the payload index IO write counter, consistent with the existing counter reference methods. The implementation follows the established pattern and is properly marked with #[inline] for optimization.

lib/segment/tests/integration/multivector_hnsw_test.rs (5)

48-49: Added hardware counter initialization

The hardware counter is now initialized at the beginning of the test function, which allows it to be used consistently throughout the test for performance measurement.

---

55-56: Hardware counter integration with field index creation

The hardware counter is now properly passed to the create_field_index method, enabling measurement of performance metrics during index creation.

---

98-99: Hardware counter integration with point upsertion

The hardware counter is now properly passed to the upsert_point method, enabling measurement of payload operations during vector insertions.

---

102-103: Hardware counter integration with payload operations

The hardware counter is now properly passed to the set_full_payload method, enabling measurement of payload operations.

---

110-111: Hardware counter integration with vector storage operations

The hardware counter is now properly passed to the insert_vector method, enabling comprehensive tracking of vector storage operations.

lib/storage/src/content_manager/snapshots/recover.rs (2)

9-9: Added hardware measurement accumulator import

This import supports the integration of hardware measurement capabilities in the snapshot recovery process.

---

153-158: Using disposable hardware counter for collection meta operations

The submit_collection_meta_op call now includes a disposable hardware measurement accumulator as required by the updated method signature. The comment indicates this is for internal API use, suggesting metrics from this operation may not need to be persisted or reported in the same way as regular operations.

lib/segment/tests/integration/batch_search_test.rs (2)

41-42: Good addition of hardware counter initialization

Creating a single HardwareCounterCell instance at the beginning of the test function is a good approach for consistent hardware usage tracking across different operations within the test.

---

48-48: Added hardware counter parameter to create_field_index call

This change correctly passes the hardware counter to the field index creation method, allowing for the measurement of hardware resource usage during this operation.

src/common/collections.rs (2)

23-23: Good import addition for hardware measurement

The import for HwMeasurementAcc is appropriately added to support the new hardware measurement capabilities.

---

284-284: Consistent implementation of disposable hardware counters across cluster operations

All cluster operations have been consistently updated to use a disposable hardware measurement accumulator, appropriately indicating that these operations aren't being measured for hardware usage. The consistent approach across all operation types demonstrates good attention to detail.

Also applies to: 321-321, 353-353, 381-381, 455-455, 494-494, 539-539, 661-661, 688-688, 709-709, 761-761, 791-791, 821-821

lib/segment/tests/integration/multivector_quantization_test.rs (1)

269-273: Added hardware counter parameter to set_indexed call

This change properly updates the set_indexed method call to include the hardware counter parameter, ensuring consistency with the other changes in the codebase related to hardware measurement for indexing operations.

lib/api/src/grpc/qdrant.rs (1)

6521-6525:

❓ Verification inconclusive

Addition of payload_index_io_write looks good, but verify tag changes

The addition of the payload_index_io_write field is well-aligned with the PR objectives and maintains a consistent naming pattern with the existing payload_index_io_read field.

I notice that adding this new field has caused the tag numbers for vector_io_read and vector_io_write to be changed (from what were likely tags 5 and 6 to tags 6 and 7 respectively). Changing Protobuf tag numbers can potentially break backward compatibility for existing clients.

Based on the retrieved learning, we know that payload_index_io_read was introduced in PR #5951 and was not yet used in production. If vector_io_read and vector_io_write are in a similar situation (not yet used in production), then these tag changes are acceptable.

---

🏁 Script executed:

#!/bin/bash
# Check when vector_io_read and vector_io_write were introduced
git log -p -- lib/api/src/grpc/qdrant.rs | grep -A 5 -B 5 "vector_io_"

Length of output: 0

---

Action Required: Verify Protobuf Tag Changes for Qdrant Fields

The addition of payload_index_io_write aligns well with our objectives and follows a consistent naming pattern with the existing payload_index_io_read field. However, the resulting change in tag numbers for vector_io_read (now tag 6) and vector_io_write (now tag 7) raises some concerns about backward compatibility.

Since the automated git log check didn't reveal any commit history for vector_io_read and vector_io_write, please manually confirm that these fields are in a similar situation as payload_index_io_read (introduced in PR #5951 and not used in production). Specifically, verify that:

• The previous tag numbers for vector_io_read and vector_io_write (likely tags 5 and 6) have not been relied upon by any production client.
• The change in tags does not break compatibility with existing services.

Once you’ve confirmed these points manually, the tag changes can be safely accepted.

lib/segment/tests/integration/hnsw_discover_test.rs (1)

205-209: Added hw_counter parameter to set_indexed method

The added &hw_counter parameter enables hardware counter tracking during payload index creation, aligning with the PR's objective to measure payload index I/O operations. This change ensures that hardware performance metrics are properly measured during indexing operations in test scenarios.

lib/collection/src/tests/points_dedup.rs (1)

97-102: Added HwMeasurementAcc parameter to create_payload_index

The addition of HwMeasurementAcc::new() parameter to the create_payload_index call implements measurement for the "create field-index" endpoint, which directly aligns with the PR objectives. This change makes the method call consistent with other operations in the file that already use hardware measurement accumulators (e.g., line 136).

src/tonic/api/collections_api.rs (2)

19-19: Added HwMeasurementAcc import

Added import for hardware measurement accumulator to support payload index I/O tracking.

---

54-59: Added disposable hardware counter to collection operations

The addition of HwMeasurementAcc::disposable() parameter integrates with the hardware measurement system while the comment indicates these operations aren't currently measured. This maintains API consistency across the codebase.

Is there a plan to measure collection operations in the future, or is there a specific reason why these operations don't need measurement? The comment "Collection ops not measured" suggests this might be intentional.

lib/segment/src/index/field_index/full_text_index/mmap_inverted_index/mod.rs (1)

144-149: Added hw_counter parameter to index_document method

The addition of the _hw_counter parameter (with underscore prefix) maintains interface consistency with other implementations, even though this immutable index implementation doesn't use the parameter since it always returns an error. This change is part of the systematic integration of hardware counter tracking throughout the codebase.

lib/segment/tests/integration/byte_storage_hnsw_test.rs (3)

190-191: New hardware counter integration looks good

The hardware counter is properly initialized before being used with the set_indexed method calls, which aligns with the PR objective of measuring payload index IO operations.

---

195-199: Parameter addition to set_indexed is properly implemented

Adding the hardware counter parameter to the set_indexed method call maintains consistency with the API changes being made across the codebase.

---

204-208: Consistent implementation across multiple calls

The hardware counter is consistently passed to both instances of the set_indexed method call, ensuring uniform tracking behavior.

lib/segment/src/index/plain_payload_index.rs (1)

78-83: Interface update for hardware counter tracking

The method signature has been appropriately updated to include the hardware counter parameter, maintaining consistency with the trait contract. Note that the parameter is prefixed with an underscore indicating it's not used within this implementation, which is acceptable for a base implementation.

lib/segment/benches/sparse_index_search.rs (1)

185-185: Hardware counter parameter added to method call

The set_indexed method call has been updated to include the hardware counter reference, properly reusing the counter initialized at line 89. This change ensures consistency with the updated API for tracking payload index operations.

lib/segment/tests/integration/sparse_vector_index_search_tests.rs (2)

392-393: Hardware counter initialization added

A new hardware counter is properly initialized before being used with the set_indexed method call, consistent with the pattern used in other test files.

---

397-397: Method call updated with hardware counter parameter

The set_indexed method call now includes the hardware counter parameter, ensuring payload index operations can be properly measured in this test scenario.

lib/segment/src/index/field_index/numeric_index/mutable_numeric_index.rs (1)

259-276: Good implementation of payload index IO write monitoring

The added hardware counter implementation correctly measures the amount of data written to the payload index storage. The counter is incremented with the exact size of the key and index value in bytes, ensuring accurate measurement of write operations.

The implementation follows good practices by:

1. Accepting a hardware counter reference parameter
2. Tracking the size with size_of_val to get precise byte counts
3. Incrementing the appropriate counter with the accumulated value

lib/segment/benches/boolean_filtering.rs (1)

110-116: Updated method call to match new signature

The set_indexed method call has been properly updated to include the hardware counter parameter, maintaining consistency with the interface changes across the codebase.

lib/segment/tests/integration/gpu_hnsw_test.rs (2)

76-77: Good enhancement: shared hardware counter instance

Creating a single hardware counter instance that's used throughout the test function is a good practice. This ensures consistent measurement across all operations and avoids the overhead of creating multiple counter instances.

---

113-120: Updated method call with hardware counter parameter

The set_indexed call has been properly updated to include the hardware counter parameter, matching the interface changes in the StructPayloadIndex implementation.

lib/segment/src/index/field_index/full_text_index/tests/mod.rs (1)

179-181: Updated method call to include hardware counter

The add_many method call has been correctly updated to pass the hardware counter parameter, maintaining consistency with the interface changes in the text indexing implementation.

lib/segment/tests/integration/multivector_filtrable_hnsw_test.rs (1)

173-177: Hardware counter appropriately added to set_indexed method.

The addition of &hw_counter parameter to the set_indexed method aligns with the PR objective of adding measurements for payload index IO operations. This change extends hardware counter tracking to payload index operations, following the pattern already established for other operations in the file (like upsert_point at line 156 and set_full_payload at line 159).

lib/segment/src/segment_constructor/segment_builder.rs (2)

536-537: Appropriate use of disposable hardware counter.

The creation of a disposable hardware counter is appropriate for internal operations where metrics aren't intended to persist beyond the immediate operation.

---

539-540: Successfully updated set_indexed to include hardware counter.

The method call now correctly includes the hardware counter as its third parameter, consistent with the changes made to the method signature. This modification enables tracking of IO write operations during payload indexing.

lib/storage/tests/integration/alias_tests.rs (2)

9-9: Added necessary import for hardware measurement.

The import for HwMeasurementAcc enables the use of hardware measurement capabilities in the test file, supporting the PR's objective of adding metrics for payload index operations.

---

127-127: Consistently added hardware measurement to collection metadata operations.

Hardware measurement accumulator has been consistently added to all three submit_collection_meta_op method calls, enabling proper tracking of hardware resources during collection and alias operations in the tests.

Also applies to: 143-143, 169-169

lib/segment/tests/integration/exact_search_test.rs (1)

84-88: Hardware counter parameter added to set_indexed with improved formatting.

The set_indexed method call has been updated to include the hardware counter parameter and is now formatted across multiple lines for better readability. This change matches the implementation pattern seen in other files and supports the PR's objective of measuring payload index IO operations.

lib/common/common/src/counter/hardware_counter.rs (4)

17-17: New field added to track payload index writes.

The addition of payload_index_io_write_counter extends the hardware counter to include writes to the payload index, complementing the existing payload_index_io_read_counter.

---

32-32: Consistent initialization of the new counter in all constructors.

The new payload_index_io_write_counter is properly initialized in all constructor methods, ensuring consistent behavior regardless of how the HardwareCounterCell is created.

Also applies to: 49-49, 63-63, 84-84

---

129-132: Added accessor method for the new counter.

This accessor method follows the established pattern for other counters, providing read-only access to the payload_index_io_write_counter.

---

172-172: Updated get_hw_data to include the new counter.

The get_hw_data method now correctly includes the new counter in both the destructuring pattern and the returned HardwareData struct, ensuring it's properly reported.

Also applies to: 183-183

lib/collection/src/operations/verification/mod.rs (2)

305-305: Added import for hardware counter support.

The import of HwMeasurementAcc from the common::counter::hardware_accumulator module is necessary for passing hardware measurement parameters to collection operations.

---

520-520: Updated create_payload_index call to include hardware measurement.

The test now correctly passes a new instance of HwMeasurementAcc to the create_payload_index method, adapting to the updated method signature that now measures payload index operations.

lib/storage/src/content_manager/mod.rs (2)

2-2: Added import for hardware measurement accumulator.

This import is required for the updated method signature that now includes hardware measurement capabilities.

---

215-215:

✅ Verification successful

Updated trait method to include hardware measurement.

The perform_collection_meta_op method in the CollectionContainer trait now takes an additional hw_acc parameter of type HwMeasurementAcc to track hardware resource usage during collection operations.

Ensure that all implementations of the CollectionContainer trait have been updated to match this new method signature. If there are any implementations that haven't been updated, they will cause compilation errors.

---

🏁 Script executed:

#!/bin/bash
# Find all implementations of CollectionContainer trait
echo "Searching for implementations of CollectionContainer trait..."
rg -l "impl\s+.*CollectionContainer\s+for" --type rust

Length of output: 302

---

🏁 Script executed:

#!/bin/bash
echo "Checking for updated perform_collection_meta_op signature in toc/collection_container.rs:"
rg "fn\s+perform_collection_meta_op" -A 3 lib/storage/src/content_manager/toc/collection_container.rs

echo -e "\nChecking for updated perform_collection_meta_op signature in consensus_manager.rs:"
rg "fn\s+perform_collection_meta_op" -A 3 lib/storage/src/content_manager/consensus_manager.rs

Length of output: 891

---

Hardware measurement parameter integrated across all implementations.
The perform_collection_meta_op method now consistently includes the hw_acc: HwMeasurementAcc parameter. Both implementations in:

• lib/storage/src/content_manager/toc/collection_container.rs
• lib/storage/src/content_manager/consensus_manager.rs

have been updated accordingly, ensuring there are no signature mismatches causing compilation issues.

lib/api/src/grpc/proto/points.proto (1)

1172-1174: Added payload index I/O write field to HardwareUsage message.

The HardwareUsage message has been updated to include the payload_index_io_write field, with fields vector_io_read and vector_io_write shifted to positions 6 and 7. This change reflects the addition of the new hardware counter in the system.

Important notes:

1. The Protocol Buffers field numbering has been updated properly
2. This change will require updates to clients consuming the API

lib/segment/tests/integration/hnsw_quantized_search_test.rs (1)

78-85: Correct implementation of hardware counter usage in create_field_index

The addition of the hardware counter parameter (&hw_counter) to the create_field_index method call is consistent with the PR's goal of measuring payload index IO writes. This change properly integrates the hardware counter that was initialized on line 66, enabling performance metrics tracking during the indexing process.

lib/segment/tests/integration/byte_storage_quantization_test.rs (1)

302-309: Hardware counter properly integrated in set_indexed method call

The addition of the hardware counter parameter to the set_indexed method call is consistent with other changes in the PR. The hardware counter (initialized on line 279) is now properly passed to track payload index IO writes during the indexing operation.

lib/segment/tests/integration/filtrable_hnsw_test.rs (1)

145-152: Hardware counter correctly integrated in set_indexed method call

The modification to include the hardware counter parameter in the set_indexed method call is consistent with the PR's objective of measuring payload index IO writes. The hardware counter (initialized on line 109) is properly passed to enable metric tracking during indexing operations.

lib/segment/src/index/field_index/full_text_index/immutable_inverted_index.rs (1)

28-33: Interface update to include hardware counter in index_document

The addition of the _hw_counter parameter to the index_document method signature properly aligns the interface with other methods in the codebase. The parameter is prefixed with an underscore since it's not used within this method implementation, which is appropriate as this method immediately returns an error for immutable indices.

lib/collection/tests/integration/collection_test.rs (4)

544-549: Consistent hardware counter sharing through cloning

The addition of .clone() on the hw_counter ensures each method gets its own copy of the hardware measurement accumulator, which is necessary since these operations may execute in parallel or need independent tracking.

---

558-559: Hardware counter propagation to index creation

Good practice to propagate the hardware counter to the payload index creation operation, allowing proper tracking of resource usage during index building.

---

568-569: Consistent hardware counter approach

The cloning pattern is consistently applied across all calls to create_payload_index_with_wait, which maintains uniformity in how performance metrics are tracked.

---

578-579: Hardware counter for multi-value index

Properly extending the hardware counter to the multi-value field index creation, completing the coverage of performance tracking across all index types.

lib/segment/src/index/query_optimization/rescore_formula/value_retriever.rs (3)

199-199: Added hardware counter dependency for performance tracking

Importing HardwareCounterCell to enable monitoring IO operations in the test module.

---

301-311: Hardware counter integration for numeric index testing

Properly initializes a hardware counter and passes it to the add_point method calls for the numeric index, enabling performance measurement during point additions in tests.

---

318-324: Hardware counter for geo index operations

Extended hardware counter usage to geo index operations, ensuring consistent performance tracking across different index types.

src/actix/api/collections_api.rs (3)

8-8: Added hardware measurement capability

Importing the necessary hardware measurement accumulator for tracking performance metrics.

---

130-131: Disposable hardware counter for collection creation

Adds a disposable (unmeasured) hardware counter for consistency with the updated API signatures, while indicating that this specific API operation isn't being measured for performance.

---

154-155: Consistent hardware counter pattern across API operations

All collection meta operations consistently use the same disposable hardware counter approach, maintaining uniformity across the API surface.

Also applies to: 175-176, 194-195

lib/segment/src/index/field_index/numeric_index/tests.rs (4)

36-47: Updated IndexBuilder interface for hardware counter support

The add_point method signature has been updated to include the hw_counter parameter, which is then properly passed to the underlying builder implementations. This change ensures that hardware metrics can be collected during index operations across all builder types.

---

90-99: Hardware counter for random index generation

Added hardware counter initialization and usage in the random_index function, ensuring that performance metrics are collected during test index creation.

---

144-147: Hardware counter for payload modification tests

Added hardware counter when testing the effects of empty payload operations, maintaining consistent measurement across all test scenarios.

---

285-292: Consistent hardware counter usage across all test cases

Each test function properly initializes and uses the hardware counter when adding points to indices, ensuring comprehensive and consistent performance measurement across different index types and test scenarios.

Also applies to: 322-329, 384-391

lib/segment/src/index/field_index/map_index/immutable_map_index.rs (1)

263-275: Good optimization for hardware counter metrics collection

The change improves efficiency by accumulating the total size of all checked values first, then incrementing the hardware counter once outside the closure, rather than incrementing it repeatedly inside the closure. This reduces atomic counter contention when processing multiple values.

lib/collection/src/collection/state_management.rs (1)

209-211: Well-documented hardware measurement handling

Good addition of the HwMeasurementAcc::disposable() parameter with a clear comment explaining why this particular operation uses a disposable (unmeasured) counter. The comment effectively clarifies that this is an internal consensus operation that doesn't need performance tracking.

lib/segment/benches/range_filtering.rs (2)

76-80: Proper integration of hardware metrics in benchmarks

The hardware counter is now correctly passed to the set_indexed method, allowing for accurate tracking of hardware usage during benchmark execution. This aligns with the improvements to hardware metric collection throughout the codebase.

---

85-89: Consistent hardware counter integration

Hardware counter is properly included in the integer index creation, maintaining consistency with the float index implementation above.

src/actix/api/update_api.rs (1)

405-429: Implement hardware metrics for create-field-index endpoint

Excellent implementation that adds hardware performance measurement to the create-field-index endpoint. The code:

1. Gets a hardware counter configured according to service settings
2. Passes the counter to the underlying operation for tracking
3. Includes the hardware metrics in the API response

This implementation fulfills the PR objective of measuring the "create field-index" endpoint operations.

lib/api/src/grpc/conversions.rs (2)

2731-2731: Adding new hardware counter metric for payload index I/O writes

This adds the payload_index_io_write field to track write operations for payload indexes, complementing the existing payload_index_io_read field introduced in PR #5951.

---

2745-2745: Proper integration of the new hardware counter field

The field is correctly destructured and passed to the HardwareData constructor with appropriate type conversion from u64 to usize.

Also applies to: 2755-2755

lib/segment/src/index/field_index/full_text_index/mmap_text_index.rs (4)

5-5: Added missing import for hardware counter

Correctly added the import for the HardwareCounterCell needed for the method signature changes.

---

97-97: Added hardware counter parameter to add_many method

This change aligns with the pattern seen in other implementations of ValueIndexer to track hardware resources during indexing operations.

---

111-113: Updated index_document call to include hardware counter

Now passing the hardware counter to the index_document method, allowing proper measurement of I/O writes during document indexing.

---

132-139: Updated add_point method signature to include hardware counter

The method signature and implementation has been properly updated to include the hardware counter parameter and correctly forwards it to the underlying ValueIndexer::add_point method.

lib/collection/src/collection_manager/optimizers/segment_optimizer.rs (6)

505-506: Created disposable hardware counter for internal operations

Using a disposable hardware counter is appropriate here since this is an internal operation where measurement isn't required. The counter is properly constructed and documented.

---

517-522: Updated create_field_index call to include hardware counter

Now passing the hardware counter to track resource usage during field index creation.

---

539-540: Updated delete_point call to include hardware counter

The hardware counter is now passed to delete_point, allowing proper tracking of I/O operations during point deletion.

---

691-692: Created second disposable hardware counter instance

Another disposable hardware counter is created for the second part of the optimization process. The comment clearly explains why a disposable counter is appropriate here.

---

707-712: Consistently passing hardware counter to create_field_index

This maintains consistency with the earlier part of the method by passing the hardware counter to the field index creation.

---

734-735: Consistent use of hardware counter for point deletion

The hardware counter is consistently passed to delete_point in both places where it's called.

lib/segment/tests/integration/disbalanced_vectors_test.rs (1)

64-65: Removed hardware counter parameter from delete_vector calls

The delete_vector method signature has been changed to no longer require a hardware counter parameter, while maintaining the hardware counter for delete_point calls (as seen on line 77). This suggests an architectural decision to track I/O operations differently for vector deletion vs point deletion.

Also applies to: 67-68, 72-73

lib/segment/src/segment/tests.rs (3)

495-495: Removed hardware counter parameter from delete_vector call

This change aligns with the PR objective of removing measurements related to vector deletions. The delete_vector method signature has been updated to no longer require the hardware counter parameter.

---

501-501: Removed hardware counter parameter from delete_vector call

Consistent with the earlier change, the hardware counter parameter is removed from this delete_vector call as well.

---

670-671: Removed hardware counter parameter from delete_vector call

The hardware counter parameter is removed from the delete_vector call while preserving the error checking logic. This is consistent with the other similar changes in this file.

lib/segment/src/segment/segment_ops.rs (1)

548-553: Added hardware counter parameter to create_field_index method call

This change adds a hardware counter parameter to the create_field_index method call, using a disposable counter since this function is only used in an unmeasured context (Segment::load). This modification supports the PR's objective of adding a new measurement field for index IO writes.

lib/storage/src/content_manager/toc/collection_container.rs (2)

10-10: Added import for hardware accumulator

Added import for HwMeasurementAcc from the hardware accumulator module to support hardware metric tracking.

---

21-27: Updated method signature to include hardware accumulator parameter

The perform_collection_meta_op method signature has been updated to include a hardware measurement accumulator parameter. This parameter is then correctly passed to the perform_collection_meta_op_sync method. This change is consistent with the PR's objective of adding hardware measurement tracking capabilities for collection operations.

lib/segment/src/vector_storage/dense/dynamic_mmap_flags.rs (2)

8-8: Added import for hardware metric reference counter

Added import for HwMetricRefCounter to support tracking hardware metrics for I/O operations.

---

230-255: Enhanced set_with_resize method to measure I/O writes

The set_with_resize method has been enhanced to measure I/O write operations by:

1. Adding a hw_counter_ref parameter of type HwMetricRefCounter
2. Measuring the write of a single boolean value
3. Measuring additional writes when resizing the underlying storage

This implementation ensures that both individual writes and bulk operations during resizing are properly tracked, which directly supports the PR's objective of adding payload index I/O write measurements.

lib/segment/tests/integration/segment_builder_test.rs (3)

107-108: Properly added hardware counter for field index creation.

Adding the hardware counter at the beginning of the test function is a good approach, as it ensures consistent metric tracking throughout the test.

---

111-112: Correctly updated method call with hardware counter parameter.

The hardware counter parameter is correctly passed to the create_field_index method call, aligning with the PR objective of measuring payload index IO operations.

---

116-117: Consistently applied hardware counter across all field index creations.

The hardware counter is consistently passed to all field index creation operations, ensuring that metrics are captured uniformly throughout the test.

lib/storage/src/dispatcher.rs (3)

9-9: Updated import statement to include necessary hardware counter types.

The import has been expanded to include both HwMeasurementAcc and HwSharedDrain from the hardware accumulator module, providing access to the required types for the method signature update.

---

71-71: Added hardware measurement accumulator parameter to method signature.

The hw_acc parameter has been added to the submit_collection_meta_op method, which allows tracking hardware metrics during collection metadata operations. This aligns with the PR's objective of measuring payload index IO writes.

---

222-222: Properly propagated hardware accumulator to perform_collection_meta_op method.

The hardware accumulator is correctly passed to the underlying method call, ensuring that metrics are properly tracked throughout the operation chain.

lib/segment/tests/integration/nested_filtering_test.rs (2)

84-91: Updated set_indexed call with hardware counter parameter.

The hardware counter is now correctly passed to the set_indexed method call, allowing for proper tracking of payload index IO operations when creating an index for the "f" field.

---

94-98: Consistently added hardware counter to all set_indexed method calls.

The hardware counter parameter has been consistently added to all set_indexed calls for various fields in the test. This ensures uniform tracking of payload index operations across all indexed fields.

Also applies to: 101-105, 108-112, 115-119

lib/collection/src/collection_manager/optimizers/vacuum_optimizer.rs (2)

442-443: Added hardware counter at the beginning of test function.

Initializing the hardware counter at the beginning of the test function is a good practice, ensuring it's available for all operations that require performance metrics tracking.

---

449-450: Updated create_field_index call with hardware counter parameter.

The hardware counter is correctly passed to the create_field_index method, allowing for accurate measurement of payload index IO write operations during index creation.

lib/storage/src/content_manager/consensus_manager.rs (5)

17-17: Added hardware counter for payload index I/O measurement.

The import of HwMeasurementAcc is necessary for tracking hardware usage metrics during operations.

---

495-496: Good use of disposable accumulator for internal operations.

The disposable() method creates a measurement accumulator that won't contribute to metrics, which is appropriate for internal operations where we don't want to account for the hardware usage.

---

498-499: Updated method call to include hardware metrics.

The perform_collection_meta_op now includes the hardware accumulator parameter to track resource usage during the operation.

---

1115-1115: Appropriate test update for new dependence.

Updated the import in the test module to match the changes in the main code.

---

1230-1231: Updated trait method signature for hardware measurement.

The CollectionContainer trait's perform_collection_meta_op now requires a hardware measurement accumulator parameter, completing the propagation of this capability throughout the system.

lib/common/common/src/counter/hardware_accumulator.rs (6)

16-16: Added payload index I/O write counter field.

New counter field for tracking write operations to the payload index, which complements the existing read counter.

---

38-40: Added getter method for payload index I/O write counter.

The method allows retrieving the current value of the newly added counter, maintaining API consistency with other counters.

---

57-57: Updated accumulation logic to include new counter.

The destructuring pattern and accumulation call now properly handle the new payload index I/O write counter, ensuring it's correctly updated when hardware data is accumulated.

Also applies to: 66-66

---

74-82: Updated Clone implementation for HwSharedDrain.

The Clone implementation has been properly updated to include the new counter field, ensuring that clones maintain all necessary state.

---

92-92: Initialized new counter in Default implementation.

The Default implementation now properly initializes the new counter field to zero, ensuring proper behavior when creating new instances.

---

177-179: Added accessor method to HwMeasurementAcc.

This method completes the API for accessing the new counter, delegating to the underlying request_drain as expected.

src/migrations/single_to_cluster.rs (3)

6-6: Added hardware accumulator import for measurement capabilities.

Necessary import to support using hardware measurement during migration operations.

---

122-127: Added hardware measurement to collection meta operations.

Now properly measures hardware usage during collection operations in migrations, using a disposable accumulator to indicate these are unmeasured operations.

---

144-145: Added hardware measurement for shard replica state operation.

The operation now includes a hardware accumulator with a helpful comment explaining it's unmeasured, maintaining consistency with other operation calls.

lib/segment/src/index/payload_index_base.rs (3)

28-29: Added hardware counter parameter to build_index method.

This allows tracking hardware resource usage during index building operations, which is important for performance monitoring.

---

44-45: Added hardware counter parameter to set_indexed method.

Ensures hardware usage is tracked when setting fields as indexed, maintaining consistency with the build_index method.

---

48-48: Updated internal method call to pass hardware counter.

The implementation of set_indexed now correctly passes the hardware counter to build_index, ensuring that resource usage is properly tracked through the call chain.

lib/segment/src/index/field_index/bool_index/mod.rs (7)

189-198: Add hw_counter parameter to add_many method

The new hw_counter parameter has been added to the add_many method and properly passed through to both BoolIndex variants. This change enables tracking of payload index I/O write operations.

---

300-302: Add hardware counter initialization in test function

A new HardwareCounterCell instance is created for testing, which will allow proper tracking of hardware metrics during the tests.

---

352-353: Add hardware counter initialization in load_from_disk test

Proper initialization of the hardware counter in this test function ensures consistent usage of the hardware measurement feature across all test scenarios.

---

398-399: Add hardware counter for modify_value test

The hardware counter is appropriately initialized for the modify_value test case, maintaining consistency with the other test functions.

---

436-437: Add hardware counter for indexed_count test

Hardware counter initialization is added for the indexed_count test function, consistent with other test implementations.

---

458-459: Add hardware counter for payload_blocks test

Hardware counter initialization is properly added for the payload_blocks test function, maintaining a consistent testing approach across all test cases.

---

485-486: Add hardware counter for estimate_cardinality test

Consistent with other test functions, the hardware counter is appropriately initialized for the estimate_cardinality test.

lib/segment/src/index/field_index/full_text_index/text_index.rs (2)

274-281: Add hw_counter parameter to add_point method

The hw_counter parameter is now properly included in the add_point method signature and passed to the underlying implementation. This change enables tracking of hardware metrics for full-text indexing operations.

---

291-306: Update add_many method to include hw_counter parameter

The add_many method has been updated to include the hw_counter parameter and properly pass it to the mutable index implementation. This change ensures hardware metrics are consistently tracked across different index types.

lib/segment/src/index/field_index/bool_index/simple_bool_index.rs (2)

334-341: Add hw_counter parameter to add_point method

The hw_counter parameter has been added to the add_point method in the BoolIndexBuilder implementation, ensuring hardware metrics are tracked during point addition operations.

---

463-489: Implement payload index I/O write tracking in add_many method

The add_many method now tracks I/O write operations by incrementing the payload_index_io_write_counter with the size of the data being written. This implementation properly measures the payload index I/O writes, which is the main objective of this PR.

The implementation correctly:

1. Calculates the size of data being written (id + item_bytes)
2. Increments the counter before performing the actual database write
3. Maintains the existing logic for adding boolean values to the index

src/common/update.rs (2)

704-711: Pass hardware measurement accumulator to submit_collection_meta_op

The hardware measurement accumulator is now properly passed to the submit_collection_meta_op method, enabling metric tracking for collection metadata operations related to index creation.

---

783-790: Pass hardware measurement accumulator to submit_collection_meta_op for delete operation

Similar to the index creation operation, the hardware measurement accumulator is now passed to submit_collection_meta_op for index deletion operations, ensuring consistent tracking of hardware metrics across all index-related operations.

lib/segment/src/index/field_index/full_text_index/mutable_inverted_index.rs (6)

78-83: Method signature correctly updated to include hardware counter parameter.

The index_document method signature has been properly updated to include the hardware counter parameter, aligning with the trait definition changes in the inverted_index.rs file.

---

85-87: Good initialization of hardware counter.

The code correctly retrieves the payload index write counter from the hardware counter cell, making it available for efficient usage throughout the method.

---

88-96: Hardware counter implementation for document sizing is correct.

The calculation properly measures only the overhead of Document objects during vector resizing, as mentioned in the comment. The approach correctly accounts for the size difference between the new and old lengths.

---

98-106: Efficient counter value accumulation.

Good implementation of accumulating the hardware counter values locally before incrementing the counter. This avoids excessive counter cell access during the loop iteration.

---

116-118: Proper accounting for point_id size.

The code correctly accounts for each point_id's memory usage by accumulating its size into the hardware counter value.

---

120-122: Efficient hardware counter update with accumulated value.

The implementation correctly performs a single increment with the accumulated value after processing all tokens, which is more efficient than incrementing for each token individually.

lib/segment/src/index/field_index/full_text_index/inverted_index.rs (2)

86-91: Correctly updated trait method signature.

The InvertedIndex trait has been properly updated to include the hardware counter parameter, ensuring consistency across all implementers.

---

246-254: Test function correctly updated to use hardware counter.

The test function now properly initializes a hardware counter and passes it to the index_document method call, ensuring the tests properly exercise the new functionality.

src/common/metrics.rs (3)

334-336: Added new payload_index_io_write field to HardwareUsage struct.

This addition ensures that write operations on the payload index are now tracked alongside other hardware counters, completing the metrics collection for this component.

---

356-362: Metrics type changed from GAUGE to COUNTER for consistency.

The metric type for payload_index_io_read has been correctly changed from GAUGE to COUNTER, making it consistent with other I/O metrics and ensuring proper accumulation of values.

---

364-372: New metric family added for payload_index_io_write.

The implementation correctly adds a new metric family for tracking payload index write operations, with appropriate name, description, and type definition.

lib/collection/src/collection/payload_index_schema.rs (4)

36-41: Added hardware accumulator parameter to create_payload_index method.

The method now accepts a hardware accumulator parameter, allowing for consistent measurement of hardware usage during index creation operations.

---

44-45: Now passing the hardware accumulator to the create_payload_index_with_wait method.

Instead of using a disposable accumulator, the code now correctly passes the provided hardware accumulator to the underlying method, ensuring measurements are properly tracked.

---

48-54: Added hardware accumulator parameter to create_payload_index_with_wait method.

The method properly accepts the hardware accumulator parameter that will be used in downstream operations.

---

72-74: Using provided hardware accumulator in update_all_local call.

The code now correctly uses the provided hardware accumulator instead of creating a disposable one, ensuring hardware measurements are consistently tracked during index creation.

lib/collection/src/collection_manager/holders/segment_holder.rs (1)

1115-1119: Hardware counter added for index creation operations

The addition of the hardware counter (hw_counter) for field index creation is appropriate for measuring payload index I/O operations. Using a disposable counter for internal operations matches the pattern used elsewhere in the codebase.

lib/storage/src/content_manager/toc/collection_meta_ops.rs (5)

11-11: Added hardware accumulator import

The import of the HwMeasurementAcc type is needed for the new hardware counter parameters added to various methods in this file.

---

25-25: Added hardware measurement accumulator parameter

Adding the hw_acc parameter to perform_collection_meta_op_sync allows the method to track hardware usage metrics when performing collection metadata operations.

---

34-34: Consistently added hw_acc parameter to corresponding async method

The hardware measurement accumulator parameter is consistently added to the async version of perform_collection_meta_op as well, maintaining the same signature pattern.

---

107-108: Propagated hw_acc parameter to create_payload_index call

The hardware counter is correctly passed through to the create_payload_index method, ensuring that payload index I/O write operations are measured.

---

631-636: Added hw_acc parameter to create_payload_index method

The hardware measurement accumulator is added to the create_payload_index method and correctly propagated to the collection's create_payload_index method. This completes the chain of passing the hardware counter from the API layer down to the actual index creation implementation.

lib/segment/src/index/field_index/null_index/mmap_null_index.rs (5)

101-105: Added hardware counter parameter to add_point method

The hardware counter parameter has been added to the add_point method, allowing the measurement of I/O operations when adding points to the null index.

---

140-146: Improved I/O measurement for bit flag operations

The implementation now captures payload index I/O write operations by getting a reference to the counter and using it when setting bit flags.

---

151-156: Using disposable hardware counter for remove operations

The remove_point method now uses a disposable hardware counter for unmeasured delete operations, which is appropriate since deletions don't need to be tracked in the same way as additions.

---

352-359: Updated FieldIndexBuilderTrait implementation consistently

The add_point method in the builder trait implementation has been updated to include and pass along the hardware counter parameter, maintaining consistency with the main implementation.

---

386-397: Updated tests to include hardware counter

The test code has been properly updated to initialize a hardware counter and pass it to the various add_point calls, ensuring that the tests remain valid with the new parameter requirements.

lib/segment/src/index/field_index/geo_index/mutable_geo_index.rs (5)

140-145: Added hardware counter to MutableGeoMapIndex.add_many_geo_points

The hardware counter parameter has been added to the add_many_geo_points method in the MutableGeoMapIndex struct, allowing for measurement of I/O operations when adding geographic points.

---

155-157: Correctly propagated hardware counter to InMemoryGeoMapIndex

The hardware counter is properly passed to the underlying in_memory_index.add_many_geo_points method, ensuring consistent tracking of I/O operations.

---

310-315: Added hardware counter to InMemoryGeoMapIndex.add_many_geo_points

The hardware counter parameter has been added to the corresponding method in the InMemoryGeoMapIndex struct, maintaining consistent signatures across related implementations.

---

329-336: Added I/O measurement for geo hash operations

The implementation now captures payload index I/O write operations by getting a reference to the counter and incrementing it by the size of each added geo hash. This provides accurate measurement of the storage impact of geo index operations.

---

349-350: Added bulk I/O measurement for all geo hashes

The code now also captures the total I/O cost of storing point IDs for all the geo hashes, providing a comprehensive measurement of the index write operations.

lib/segment/src/index/field_index/full_text_index/mutable_text_index.rs (5)

3-3: Added hardware counter import to track payload index operations.

The import of HardwareCounterCell is added to enable tracking of payload index I/O operations, which aligns with the PR objective of measuring payload index IO writes.

---

50-55: Method signature updated to include hardware counter tracking.

The add_many function signature now includes a hw_counter parameter that will be used to measure performance metrics during text index operations, specifically payload index IO writes. This change is consistent with the interfaces in other field index implementations as shown in the relevant code snippets.

---

69-70: Hardware counter properly passed to underlying inverted index.

The change properly propagates the hardware counter to the inverted index implementation, ensuring that IO write operations are measured at the appropriate level of abstraction.

---

114-114: Updated test implementation to include hardware counters.

The test code now includes the required hardware counter imports and initializes a counter instance for use in tests, maintaining consistency with the implementation changes.

Also applies to: 153-154

---

156-158: Hardware counter properly integrated into test point operations.

The test code has been updated to pass the hardware counter to all relevant add_point operations, ensuring that the tests properly validate the new behavior. This is a thorough implementation that ensures test coverage of the new functionality.

Also applies to: 205-205, 210-210

lib/collection/src/collection_manager/collection_updater.rs (3)

58-58: Hardware counter passed to field index operation processing.

The field index operation is now properly instrumented with hardware counter tracking. This ensures that payload index IO metrics will be collected during field index creation operations.

---

362-363: Added hardware counter in test fixture.

The hardware counter is now initialized once at the beginning of the test, which is a good practice to reduce code duplication. This single instance is then reused throughout the test.

---

370-370: Hardware counter properly passed to field index creation in tests.

The tests now correctly pass the hardware counter to create_field_index operations, ensuring that the test properly exercises the hardware counter functionality in a realistic scenario.

Also applies to: 380-380

lib/collection/src/collection_manager/segments_updater.rs (5)

108-133: Removed hardware counter from delete_vectors function.

The delete_vectors function no longer requires a hardware counter parameter. This suggests that vector deletion operations are not relevant for measuring payload index IO, which aligns with the PR objective of focusing specifically on payload index write operations.

---

144-144: Updated call site to match new function signature.

The call to delete_vectors has been updated to remove the hardware counter argument, maintaining consistency with the updated function signature.

---

344-344: Added hardware counter parameter to create_field_index.

The create_field_index function now accepts a hardware counter parameter, allowing it to track payload index IO write operations during index creation.

---

349-349: Hardware counter properly passed to build_field_index.

The hardware counter is now correctly passed to the lower-level build_field_index function, ensuring that IO operations are tracked at the appropriate level of abstraction.

---

690-691: Hardware counter added to field index operation processing.

The process_field_index_operation function now accepts and properly passes the hardware counter to the create_field_index function. This change ensures consistent measurement of payload index operations throughout the call stack.

Also applies to: 698-698

lib/segment/src/fixtures/payload_context_fixture.rs (2)

269-270: Added hardware counter to test fixtures.

A hardware counter is now initialized in the create_struct_payload_index function to enable tracking of index operations in tests, ensuring that test environments match production behavior.

---

272-311: Hardware counter passed to all set_indexed operations.

All calls to set_indexed now include the hardware counter, ensuring consistent tracking of index operations across all field types (string, integer, float, geo, text, boolean). This comprehensive approach ensures that all payload index write operations will be measured.

lib/segment/src/segment/entry.rs (3)

751-751: Consistent hardware counter parameter addition.

The addition of the hw_counter parameter to the build_field_index method follows the pattern of tracking hardware metrics across the codebase and aligns with the PR objective to measure payload index IO writes.

---

763-764: Hardware counter correctly passed to build_index.

The hw_counter is properly integrated here to measure index building operations.

---

778-779: Consistent hardware counter usage.

The second instance of build_index properly receives the hardware counter, maintaining consistent measurement throughout all code paths.

lib/segment/src/index/field_index/map_index/mutable_map_index.rs (4)

47-47: Added hardware counter parameter for IO tracking.

The hw_counter parameter has been added to the add_many_to_map method signature to enable tracking of payload index write operations.

---

67-70: Correctly increments payload index write counter.

The hardware counter is now used to measure and track the size of database records being written. This implements the core functionality required for the PR objective of measuring payload index IO writes.

---

141-142: Improved measurement implementation for accumulated IO.

The code now accumulates the size of all values being checked before incrementing the counter once, which is more efficient than incrementing for each individual value.

---

155-159: Measurement logic for read operations optimized.

The accumulated hardware counter values are applied after the loop completes, improving performance by reducing the number of counter increment operations.

lib/segment/tests/integration/payload_index_test.rs (4)

290-291: Hardware counter added to test.

A HardwareCounterCell is now properly initialized for tests to ensure the updated method signatures work correctly.

---

402-403: Hardware counter initialized for nested payload tests.

Another instance of HardwareCounterCell initialization to support the updated method signatures.

---

101-103: Consistently updated method calls with hardware counter parameters.

All calls to create_field_index have been properly updated to include the hardware counter parameter, ensuring test coverage of the new measurement functionality.

Also applies to: 129-134, 137-138, 151-153, 168-169, 176-177, 192-193, 303-304, 319-320, 335-336, 351-352, 363-364, 375-376

---

1193-1193: Hardware counter added to set_indexed method calls.

The set_indexed method calls are updated to include the hardware counter parameter, maintaining consistency across the test suite.

Also applies to: 1203-1203, 1212-1212

lib/segment/src/index/field_index/bool_index/mmap_bool_index.rs (7)

85-91: Updated method signature with hardware counter parameter.

The set_or_insert method signature has been updated to include the hardware counter parameter, consistent with changes across the codebase.

---

94-96: Hardware counter properly passed to helper function.

The hardware counter is correctly passed to the set_or_insert_flag function, ensuring consistent measurement throughout the call chain.

---

275-281: Added hardware counter measurement for boolean writes.

The function now measures the size of boolean values being written and increments the payload index IO write counter accordingly.

---

287-287: Tracks size changes during resize operations.

The code correctly accounts for additional writes when resizing the flags slice by incrementing the counter with the difference between the new length and the current length.

---

315-318: Builder method updated with hardware counter parameter.

The add_point method in MmapBoolIndexBuilder now correctly passes the hardware counter to the underlying implementation.

---

332-332: ValueIndexer implementation updated with hardware counter.

The add_many method and its call to set_or_insert have been updated to include the hardware counter parameter, maintaining measurement consistency.

Also applies to: 341-341

---

351-352: Uses disposable hardware counter for remove operations.

A disposable hardware counter is used for the remove_point method, which is appropriate since these measurements don't need to be persisted.

lib/collection/src/collection_manager/holders/proxy_segment.rs (4)

102-103: Internal operations now use disposable hardware counters

The change uses HardwareCounterCell::disposable() for internal field index creation operations. This is consistent with the pattern of using disposable counters for internal operations that shouldn't affect external metrics.

---

279-280: Consistent use of disposable hardware counters for internal index operations

Using HardwareCounterCell::disposable() for index creation operations in the wrapped segment continues the pattern of isolating internal operations from external metrics.

---

580-584: Hardware counter parameter removed from delete_vector method

The delete_vector method signature has been modified to remove the hw_counter parameter, while the internal call to move_if_exists now uses a disposable counter. This ensures consistency with how vector deletions are measured across the system.

---

1172-1182: Added hardware counter parameter to build_field_index method

The build_field_index method now accepts a hardware counter parameter and passes it through to the underlying implementation. This enables proper tracking of hardware resources consumed during field index building.

lib/segment/src/index/field_index/geo_index/mod.rs (5)

364-371: Hardware counter parameter added to GeoMapIndexBuilder's add_point method

The add_point method now accepts a hardware counter parameter which is passed through to the underlying implementation. This enables tracking of resource usage during point addition operations.

---

401-408: Hardware counter parameter added to GeoMapImmutableIndexBuilder's add_point method

Similar to other builder implementations, the immutable index builder now accepts and forwards the hardware counter parameter for resource tracking.

---

431-443: Hardware counter parameter added to GeoMapIndexMmapBuilder's add_point method

The Mmap builder implementation now passes the hardware counter to add_many_geo_points, ensuring consistent resource tracking across all index implementations.

---

457-462: Hardware counter parameter added to GeoMapIndex's add_many method

The add_many method in the ValueIndexer implementation for GeoMapIndex now includes the hardware counter parameter, which is passed to the mutable index implementation.

---

676-687: Updated test implementation with hardware counter parameter

The test implementation correctly passes the hardware counter through to the implementations being tested, ensuring that tests properly validate the hardware counter functionality.

lib/segment/src/index/field_index/field_index_base.rs (5)

77-82: Added hardware counter parameter to ValueIndexer trait's add_many method

The base add_many method in the ValueIndexer trait now includes a hardware counter parameter, which standardizes hardware resource tracking across all indexing implementations.

---

96-117: Updated ValueIndexer trait's add_point method with hardware counter parameter

The add_point method now accepts and forwards the hardware counter to the add_many method, ensuring consistent resource tracking throughout the indexing pipeline.

---

284-325: Updated FieldIndex enum's add_point method with hardware counter parameter

Each variant of the FieldIndex enum now forwards the hardware counter to the respective implementation's add_point method, providing uniform resource tracking across all index types.

---

433-438: Added hardware counter parameter to FieldIndexBuilderTrait's add_point method

The trait definition for all field index builders now requires implementations to accept a hardware counter parameter, ensuring consistent hardware resource tracking across the builder pattern.

---

503-532: Updated FieldIndexBuilder's add_point method implementation with hardware counter parameter

Each variant in the implementation of add_point for FieldIndexBuilder now forwards the hardware counter to the respective builder's implementation, providing consistent resource tracking for all index builder types.

lib/segment/src/entry/entry_point.rs (2)

281-281: Added hw_counter parameter in build_field_index
Including the hardware counter parameter aligns with the broader approach of measuring I/O and CPU usage during index building. Ensure that all call sites correctly supply a valid counter.

---

299-303: Extended create_field_index to accept hw_counter
Passing the hardware counter into create_field_index maintains consistency with other index-building methods. This allows meaningful measurement of payload index writes.

lib/segment/src/index/field_index/map_index/mod.rs (6)

2-2: Importing Entry is necessary for proper map entry handling.
This import is required to leverage hash_map::Entry, which is used in the changed logic below.

---

455-462: Propagating hw_counter in add_point (MapIndexBuilder)
By forwarding the hardware counter to the underlying map index, you ensure hardware usage is tracked during point addition.

---

487-523: Measuring memory usage in add_point (MapIndexMmapBuilder)
This block correctly accumulates the size of new entries and updates the payload_index_io_write_counter. Grouping the increment in one final call to incr_delta is efficient.

---

1065-1080: Adding hw_counter parameter in add_many (MapIndex)
The hw_counter parameter ensures string payload insertion is counted toward I/O metrics. This change is consistent with other map index types.

---

1101-1105: Extending add_many for IntPayloadType with hw_counter
Including hw_counter maintains uniform measurement support for integer payload additions, matching other index implementations.

---

1133-1136: Adding hw_counter to add_many for UuidIntType
Tracking I/O writes for UUID-valued payloads continues the consistent hardware usage measurement across all indexable data types.

lib/segment/src/index/struct_payload_index.rs (4)

8-8: Importing HwMeasurementIteratorExt
This extension trait supports measuring iteration overhead with the supplied hardware counter, fitting well with the new measurement logic.

---

389-395: Applying hardware measurements to fallback iter_ids
When no index is available, the fallback to iter_ids().measure_hw_with_cell(...) ensures that iteration overhead is consistently captured in usage metrics.

---

584-584: Injecting hw_counter in overwrite_payload
This ensures any reindexing triggered by overwriting payload also records the additional I/O writes.

---

620-620: Tracking add_point calls in set_payload
Passing hw_counter here ensures newly set payload values are consistently tracked for I/O operations.

lib/segment/src/index/field_index/numeric_index/mod.rs (7)

529-536: Changed signature to include hardware counter parameter

The add_point method now correctly takes a hardware counter parameter and passes it through to the inner implementation. This change maintains consistency with other field index implementations.

---

570-577: Updated NumericIndexImmutableBuilder::add_point to include hw_counter

This change correctly propagates the hardware counter parameter to the underlying implementation, maintaining consistency across builder implementations.

---

614-636: Added hardware metrics tracking for payload index writes

The implementation now tracks the size of payload data being written to the index by incrementing the payload_index_io_write_counter with the size of flattened values. This is a well-implemented approach to measuring index IO write operations.

 self.in_memory_index.remove_point(id);
 let mut flatten_values: Vec<_> = vec![];
 for value in payload.iter() {
     let payload_values = <NumericIndex<T, P> as ValueIndexer>::get_values(value);
     flatten_values.extend(payload_values);
 }
 let flatten_values = flatten_values
     .into_iter()
     .map(NumericIndex::into_inner_value)
     .collect();
+
+hw_counter
+    .payload_index_io_write_counter()
+    .incr_delta(size_of_val(&flatten_values));
+
 self.in_memory_index.add_many_to_list(id, flatten_values);

---

826-830: Updated IntPayloadType ValueIndexer implementation to use hardware counter

The add_many method now appropriately takes and forwards the hardware counter to the underlying index implementation, ensuring consistent metrics tracking.

Also applies to: 832-832

---

862-874: Updated DateTimePayloadType ValueIndexer implementation to use hardware counter

The hardware counter is correctly propagated to the inner implementation, maintaining consistent tracking of write operations across different payload types.

---

903-911: Updated FloatPayloadType ValueIndexer implementation to use hardware counter

This implementation correctly passes the hardware counter to the inner index, providing consistent metrics collection for float payload types.

---

940-950: Updated UuidPayloadType ValueIndexer implementation to use hardware counter

The UUID indexer implementation now correctly passes the hardware counter to the inner implementation, completing the consistent tracking of write operations across all supported payload types.