dd/fix-negative-tests - Branch - schemathesis/schemathesis

schemathesis

Blog Docs Changelog

fix: False positive test cases on path, query, header, and cookie parameters of type "string"

#2929

Comparing

dd/fix-negative-tests

(

2aec074

) with

master

(

be95ed2

)

+13%

Improvements: 1

Regressions: 0

Untouched: 97

New: 0

Dropped: 0

Ignored: 0

Benchmarks

Improved

test_array_constraints[array-uniqueItems-negative]

benches/coverage_phase.py::test_array_constraints[array-uniqueItems-negative]

+13%

2.2 ms

1.9 ms

Passed

test_array_constraints[array-uniqueItems-positive]

benches/coverage_phase.py::test_array_constraints[array-uniqueItems-positive]

+7%

1.3 ms

1.2 ms

test_length[universe]

benches/schema.py::test_length[universe]

+7%

596.7 µs

558.6 µs

test_array_constraints[array-minItems-negative]

benches/coverage_phase.py::test_array_constraints[array-minItems-negative]

+6%

3 ms

2.8 ms

test_length[bbci]

benches/schema.py::test_length[bbci]

+1%

786.5 µs

778.6 µs

test_complex_nested_schemas[complex-object-positive]

benches/coverage_phase.py::test_complex_nested_schemas[complex-object-positive]

+1%

3.7 ms

test_array_constraints[array-minItems-positive]

benches/coverage_phase.py::test_array_constraints[array-minItems-positive]

+1%

3.7 ms

test_string_constraints[string-maxLength-positive]

benches/coverage_phase.py::test_string_constraints[string-maxLength-positive]

+1%

226.3 µs

225 µs

test_string_constraints[string-minLength-positive]

benches/coverage_phase.py::test_string_constraints[string-minLength-positive]

+1%

205.7 µs

204.6 µs

test_length[ml_webservices]

benches/schema.py::test_length[ml_webservices]

542.4 µs

539.8 µs

test_string_constraints[string-pattern-positive]

benches/coverage_phase.py::test_string_constraints[string-pattern-positive]

147.4 µs

146.8 µs

test_get_operation_by_id_single[bbci]

benches/schema.py::test_get_operation_by_id_single[bbci]

2.8 ms

test_links_count[bbci]

benches/schema.py::test_links_count[bbci]

763.8 µs

762 µs

test_basic_types[basic-null-positive]

benches/coverage_phase.py::test_basic_types[basic-null-positive]

72.8 µs

72.7 µs

test_array_constraints[array-items-positive]

benches/coverage_phase.py::test_array_constraints[array-items-positive]

224.7 µs

224.3 µs

test_get_operation_single[bbci]

benches/schema.py::test_get_operation_single[bbci]

2.7 ms

test_get_operation_by_id_repeatedly[vmware]

benches/schema.py::test_get_operation_by_id_repeatedly[vmware]

82.1 µs

82 µs

test_number_constraints[number-exclusiveMinimum-positive]

benches/coverage_phase.py::test_number_constraints[number-exclusiveMinimum-positive]

92.7 µs

92.6 µs

test_get_operation_by_reference_repeatedly[vmware]

benches/schema.py::test_get_operation_by_reference_repeatedly[vmware]

101.8 µs

101.7 µs

test_inlining_during_resolution

benches/references.py::test_inlining_during_resolution

2.6 ms

test_get_operation_by_reference_single[bbci]

benches/schema.py::test_get_operation_by_reference_single[bbci]

2.8 ms

test_length[appveyor]

benches/schema.py::test_length[appveyor]

3.9 ms

test_links_count[vmware]

benches/schema.py::test_links_count[vmware]

1.9 ms

test_get_operation_single[vmware]

benches/schema.py::test_get_operation_single[vmware]

10.5 ms

test_get_operation_single[universe]

benches/schema.py::test_get_operation_single[universe]

70.7 ms

test_basic_types[basic-boolean-positive]

benches/coverage_phase.py::test_basic_types[basic-boolean-positive]

75.1 µs

test_combined_schemas[combined-anyOf-positive]

benches/coverage_phase.py::test_combined_schemas[combined-anyOf-positive]

143.8 µs

test_deepclone[evetech]

benches/schema.py::test_deepclone[evetech]

44.9 ms

test_basic_types[basic-integer-positive]

benches/coverage_phase.py::test_basic_types[basic-integer-positive]

95.1 µs

test_get_operation_by_reference_repeatedly[bbci]

benches/schema.py::test_get_operation_by_reference_repeatedly[bbci]

102.5 µs

test_get_operation_by_id_single[vmware]

benches/schema.py::test_get_operation_by_id_single[vmware]

11 ms

test_cli_startup

benches/cli.py::test_cli_startup

357.3 ms

357.2 ms

test_basic_types[basic-number-positive]

benches/coverage_phase.py::test_basic_types[basic-number-positive]

91.7 µs

test_iter_operations[universe]

benches/schema.py::test_iter_operations[universe]

72.7 ms

test_length[azure_network]

benches/schema.py::test_length[azure_network]

1.2 ms

test_links_count[stripe]

benches/schema.py::test_links_count[stripe]

4.1 ms

test_get_operation_by_reference_single[vmware]

benches/schema.py::test_get_operation_by_reference_single[vmware]

10.7 ms

test_length[osisoft]

benches/schema.py::test_length[osisoft]

3.6 ms

test_object_constraints[object-properties-positive]

benches/coverage_phase.py::test_object_constraints[object-properties-positive]

225 µs

225.1 µs

test_deepclone[stripe]

benches/schema.py::test_deepclone[stripe]

115.6 ms

test_get_operation_by_id_repeatedly[bbci]

benches/schema.py::test_get_operation_by_id_repeatedly[bbci]

83 µs

test_iter_operations[azure_network]

benches/schema.py::test_iter_operations[azure_network]

178 ms

test_iter_operations[bbci]

benches/schema.py::test_iter_operations[bbci]

9.6 ms

test_deepclone[universe]

benches/schema.py::test_deepclone[universe]

29.6 ms

29.7 ms

test_deepclone[osisoft]

benches/schema.py::test_deepclone[osisoft]

27.8 ms

27.9 ms

test_iter_operations[vmware]

benches/schema.py::test_iter_operations[vmware]

72.6 ms

test_combined_schemas[combined-allOf-positive]

benches/coverage_phase.py::test_combined_schemas[combined-allOf-positive]

1.1 ms

test_object_constraints[object-required-positive]

benches/coverage_phase.py::test_object_constraints[object-required-positive]

158.9 µs

159 µs

test_length[vmware]

benches/schema.py::test_length[vmware]

1.9 ms

test_iter_operations[ml_webservices]

benches/schema.py::test_iter_operations[ml_webservices]

15.1 ms

test_deepclone[bbci]

benches/schema.py::test_deepclone[bbci]

3.1 ms

test_deepclone[azure_network]

benches/schema.py::test_deepclone[azure_network]

6.6 ms

test_deepclone[vmware]

benches/schema.py::test_deepclone[vmware]

12.3 ms

test_string_constraints[string-format-positive]

benches/coverage_phase.py::test_string_constraints[string-format-positive]

96.4 µs

96.5 µs

test_number_constraints[number-minimum-positive]

benches/coverage_phase.py::test_number_constraints[number-minimum-positive]

100.5 µs

100.7 µs

test_combined_schemas[combined-allOf-negative]

benches/coverage_phase.py::test_combined_schemas[combined-allOf-negative]

1.1 ms

test_iter_operations[evetech]

benches/schema.py::test_iter_operations[evetech]

135.8 ms

136.1 ms

test_length[stripe]

benches/schema.py::test_length[stripe]

4.1 ms

test_get_operation_repeatedly[bbci]

benches/schema.py::test_get_operation_repeatedly[bbci]

92.3 µs

92.5 µs

test_events

benches/schema.py::test_events

1.8 s

test_deepclone[ml_webservices]

benches/schema.py::test_deepclone[ml_webservices]

2.2 ms

test_rewritten_components[vmware]

benches/schema.py::test_rewritten_components[vmware]

10.9 ms

test_iter_operations[appveyor]

benches/schema.py::test_iter_operations[appveyor]

45 ms

45.1 ms

test_rewritten_components[stripe]

benches/schema.py::test_rewritten_components[stripe]

167.9 ms

168.3 ms

test_get_operation_repeatedly[universe]

benches/schema.py::test_get_operation_repeatedly[universe]

87.4 µs

87.6 µs

test_rewritten_components[bbci]

benches/schema.py::test_rewritten_components[bbci]

8.6 ms

8.7 ms

test_deepclone[appveyor]

benches/schema.py::test_deepclone[appveyor]

5.6 ms

test_array_constraints[array-items-negative]

benches/coverage_phase.py::test_array_constraints[array-items-negative]

390.6 µs

391.7 µs

test_iter_operations[osisoft]

benches/schema.py::test_iter_operations[osisoft]

172.4 ms

172.9 ms

test_basic_types[basic-string-positive]

benches/coverage_phase.py::test_basic_types[basic-string-positive]

106.8 µs

107.2 µs

test_resolve_pointer

benches/references.py::test_resolve_pointer

102.3 µs

102.6 µs

test_length[evetech]

benches/schema.py::test_length[evetech]

2.9 ms

test_basic_types[basic-string-negative]

benches/coverage_phase.py::test_basic_types[basic-string-negative]

162.8 µs

163.4 µs

test_number_constraints[number-exclusiveMinimum-negative]

benches/coverage_phase.py::test_number_constraints[number-exclusiveMinimum-negative]

199.7 µs

200.6 µs

test_string_constraints[string-minLength-negative]

benches/coverage_phase.py::test_string_constraints[string-minLength-negative]

264 µs

265.3 µs

test_basic_types[basic-number-negative]

benches/coverage_phase.py::test_basic_types[basic-number-negative]

155 µs

155.8 µs

test_object_constraints[object-required-negative]

benches/coverage_phase.py::test_object_constraints[object-required-negative]

-1%

387.5 µs

389.8 µs

test_basic_types[basic-null-negative]

benches/coverage_phase.py::test_basic_types[basic-null-negative]

-1%

156.4 µs

157.4 µs

test_number_constraints[number-minimum-negative]

benches/coverage_phase.py::test_number_constraints[number-minimum-negative]

-1%

200.7 µs

202.3 µs

test_basic_types[basic-boolean-negative]

benches/coverage_phase.py::test_basic_types[basic-boolean-negative]

-1%

156.1 µs

157.4 µs

test_string_constraints[string-maxLength-negative]

benches/coverage_phase.py::test_string_constraints[string-maxLength-negative]

-1%

234.8 µs

237.2 µs

test_complex_nested_schemas[complex-array-positive]

benches/coverage_phase.py::test_complex_nested_schemas[complex-array-positive]

-1%

52.2 ms

52.7 ms

test_as_json_schema[vmware]

benches/schema.py::test_as_json_schema[vmware]

-1%

8.2 ms

8.3 ms

test_get_operation_repeatedly[vmware]

benches/schema.py::test_get_operation_repeatedly[vmware]

-1%

91.4 µs

92.6 µs

test_as_json_schema[bbci]

benches/schema.py::test_as_json_schema[bbci]

-1%

4.1 ms

test_object_constraints[object-additionalProperties-negative]

benches/coverage_phase.py::test_object_constraints[object-additionalProperties-negative]

-2%

16.7 ms

17 ms

test_complex_nested_schemas[complex-array-negative]

benches/coverage_phase.py::test_complex_nested_schemas[complex-array-negative]

-2%

46.2 ms

47.3 ms

test_combined_schemas[combined-oneOf-negative]

benches/coverage_phase.py::test_combined_schemas[combined-oneOf-negative]

-3%

23.2 ms

23.9 ms

test_combined_schemas[combined-oneOf-positive]

benches/coverage_phase.py::test_combined_schemas[combined-oneOf-positive]

-3%

9.9 ms

10.2 ms

test_complex_nested_schemas[complex-object-negative]

benches/coverage_phase.py::test_complex_nested_schemas[complex-object-negative]

-3%

14.3 ms

14.8 ms

test_object_constraints[object-additionalProperties-positive]

benches/coverage_phase.py::test_object_constraints[object-additionalProperties-positive]

-3%

8.7 ms

9 ms

test_string_constraints[string-pattern-negative]

benches/coverage_phase.py::test_string_constraints[string-pattern-negative]

-4%

5 ms

5.2 ms

test_number_constraints[number-multipleOf-positive]

benches/coverage_phase.py::test_number_constraints[number-multipleOf-positive]

-4%

5.2 ms

5.4 ms

test_number_constraints[number-multipleOf-negative]

benches/coverage_phase.py::test_number_constraints[number-multipleOf-negative]

-4%

15.3 ms

16 ms

test_string_constraints[string-format-negative]

benches/coverage_phase.py::test_string_constraints[string-format-negative]

-5%

3.8 ms

4 ms

test_object_constraints[object-properties-negative]

benches/coverage_phase.py::test_object_constraints[object-properties-negative]

-7%

5.1 ms

5.4 ms

test_basic_types[basic-integer-negative]

benches/coverage_phase.py::test_basic_types[basic-integer-negative]

-7%

4.8 ms

5.1 ms

test_combined_schemas[combined-anyOf-negative]

benches/coverage_phase.py::test_combined_schemas[combined-anyOf-negative]

-7%

4.8 ms

5.2 ms

Commits

Click on a commit to change the comparison range

Base

master

be95ed2

+13%

fix: False positive test cases on path, query, header, and cookie parameters of type "string" Signed-off-by: Dmitry Dygalo <dmitry@dygalo.dev>

2aec074

4 days ago

by Stranger6667

Home Terms Privacy Docs